From d10107ff269c8cab60b9af763e979f12cc434815 Mon Sep 17 00:00:00 2001 From: Alessandro Berti Date: Thu, 11 Apr 2024 14:36:09 +0200 Subject: [PATCH 1/5] refactor(pm4py): refactoring Dockerfile --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5cb569893..994d032d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,10 @@ RUN apt-get -y install libxml2-dev libxslt-dev libfreetype6-dev libsuitesparse-d RUN pip install -U wheel six pytest RUN pip install -U meson-python>=0.13.1 Cython>=3.0.6 ninja spin==0.8 build RUN pip install deprecation==2.1.0 graphviz==0.20.3 intervaltree==3.1.0 networkx==3.3 packaging==24.0 python-dateutil==2.9.0.post0 pytz==2024.1 six==1.16.0 sortedcontainers==2.4.0 tzdata==2024.1 -RUN pip install colorama==0.4.6 contourpy==1.2.1 cycler==0.12.1 fonttools==4.51.0 kiwisolver==1.4.5 lxml==5.2.1 matplotlib==3.8.4 numpy==1.26.4 pandas==2.2.2 pillow==10.3.0 pydotplus==2.0.2 pyparsing==3.1.2 scipy==1.13.0 tqdm==4.66.2 +RUN pip install colorama==0.4.6 contourpy==1.2.1 cycler==0.12.1 fonttools==4.51.0 kiwisolver==1.4.5 lxml==5.2.1 matplotlib==3.8.4 numpy==1.26.4 pandas==2.2.2 pillow==10.3.0 pydotplus==2.0.2 pyparsing==3.1.2 scipy==1.13.0 tqdm==4.66.2 + +#RUN cd / && git clone https://github.com/numpy/numpy.git && cd /numpy && git submodule update --init && pip3 install . +#RUN cd / && git clone https://github.com/pandas-dev/pandas/ && cd /pandas && pip3 install . COPY . /app RUN cd /app && python setup.py install From 8f9ed77f44a71ca35f793f0fec0cefcf80f1792b Mon Sep 17 00:00:00 2001 From: Alessandro Berti Date: Thu, 11 Apr 2024 14:51:27 +0200 Subject: [PATCH 2/5] docs(pm4py): updating Dockerfile --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 994d032d0..435a0b6cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,8 @@ RUN pip install colorama==0.4.6 contourpy==1.2.1 cycler==0.12.1 fonttools==4.51. #RUN cd / && git clone https://github.com/numpy/numpy.git && cd /numpy && git submodule update --init && pip3 install . #RUN cd / && git clone https://github.com/pandas-dev/pandas/ && cd /pandas && pip3 install . +#RUN cd / && git clone https://github.com/scipy/scipy.git && cd /scipy && git submodule update --init && pip3 install . +#RUN cd / && git clone https://github.com/lxml/lxml.git && cd /lxml && pip3 install . COPY . /app RUN cd /app && python setup.py install From 7cae1475d4dc402323cab34a96feb46c26ece2ca Mon Sep 17 00:00:00 2001 From: Alessandro Berti Date: Thu, 11 Apr 2024 14:58:21 +0200 Subject: [PATCH 3/5] refactor(pm4py): updating dependencies generation script. updating Dockerfile --- Dockerfile | 3 ++- third_party/generate_dependencies.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 435a0b6cb..bb503ee64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,8 @@ RUN apt-get -y install libxml2-dev libxslt-dev libfreetype6-dev libsuitesparse-d RUN pip install -U wheel six pytest RUN pip install -U meson-python>=0.13.1 Cython>=3.0.6 ninja spin==0.8 build RUN pip install deprecation==2.1.0 graphviz==0.20.3 intervaltree==3.1.0 networkx==3.3 packaging==24.0 python-dateutil==2.9.0.post0 pytz==2024.1 six==1.16.0 sortedcontainers==2.4.0 tzdata==2024.1 -RUN pip install colorama==0.4.6 contourpy==1.2.1 cycler==0.12.1 fonttools==4.51.0 kiwisolver==1.4.5 lxml==5.2.1 matplotlib==3.8.4 numpy==1.26.4 pandas==2.2.2 pillow==10.3.0 pydotplus==2.0.2 pyparsing==3.1.2 scipy==1.13.0 tqdm==4.66.2 +RUN pip install colorama==0.4.6 cycler==0.12.1 pydotplus==2.0.2 pyparsing==3.1.2 tqdm==4.66.2 +RUN pip install contourpy==1.2.1 fonttools==4.51.0 kiwisolver==1.4.5 lxml==5.2.1 matplotlib==3.8.4 numpy==1.26.4 pandas==2.2.2 pillow==10.3.0 scipy==1.13.0 #RUN cd / && git clone https://github.com/numpy/numpy.git && cd /numpy && git submodule update --init && pip3 install . #RUN cd / && git clone https://github.com/pandas-dev/pandas/ && cd /pandas && pip3 install . diff --git a/third_party/generate_dependencies.py b/third_party/generate_dependencies.py index a6e0ee26c..cd7e19891 100644 --- a/third_party/generate_dependencies.py +++ b/third_party/generate_dependencies.py @@ -110,14 +110,19 @@ def get_version(package): F.close() first_line_packages = ["deprecation", "packaging", "networkx", "graphviz", "six", "python-dateutil", "pytz", "tzdata", "intervaltree", "sortedcontainers"] +second_line_packages = ["pydotplus", "pyparsing", "tqdm", "colorama", "cycler"] + first_packages_line = "" second_packages_line = "" +third_packages_line = "" for x in packages: cont = x[0] + "==" + x[2] + " " if x[0] in first_line_packages: first_packages_line += cont - else: + elif x[0] in second_line_packages: second_packages_line += cont + else: + third_packages_line += cont F = open("../Dockerfile", "r") dockerfile_contents = F.readlines() @@ -137,7 +142,7 @@ def get_version(package): before_lines.append(dockerfile_contents[i]) i = i + 1 -stru = "".join(before_lines + ["RUN pip install " + x + "\n" for x in [first_packages_line, second_packages_line]] + after_lines) +stru = "".join(before_lines + ["RUN pip install " + x + "\n" for x in [first_packages_line, second_packages_line, third_packages_line]] + after_lines) stru = stru.strip() + "\n" if UPDATE_DOCKERFILE: From 20bd03384f499034499c24be173c65349e75d063 Mon Sep 17 00:00:00 2001 From: Alessandro Berti Date: Thu, 11 Apr 2024 15:11:39 +0200 Subject: [PATCH 4/5] refactor(pm4py): updating dependencies generation script. updating Dockerfile --- Dockerfile | 2 +- third_party/generate_dependencies.py | 119 +++++++++++++++------------ 2 files changed, 69 insertions(+), 52 deletions(-) diff --git a/Dockerfile b/Dockerfile index bb503ee64..c1c7d324b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN apt-get -y install libxml2-dev libxslt-dev libfreetype6-dev libsuitesparse-d RUN pip install -U wheel six pytest RUN pip install -U meson-python>=0.13.1 Cython>=3.0.6 ninja spin==0.8 build RUN pip install deprecation==2.1.0 graphviz==0.20.3 intervaltree==3.1.0 networkx==3.3 packaging==24.0 python-dateutil==2.9.0.post0 pytz==2024.1 six==1.16.0 sortedcontainers==2.4.0 tzdata==2024.1 -RUN pip install colorama==0.4.6 cycler==0.12.1 pydotplus==2.0.2 pyparsing==3.1.2 tqdm==4.66.2 +RUN pip install colorama==0.4.6 cycler==0.12.1 joblib==1.4.0 pydotplus==2.0.2 pyparsing==3.1.2 threadpoolctl==3.4.0 tqdm==4.66.2 RUN pip install contourpy==1.2.1 fonttools==4.51.0 kiwisolver==1.4.5 lxml==5.2.1 matplotlib==3.8.4 numpy==1.26.4 pandas==2.2.2 pillow==10.3.0 scipy==1.13.0 #RUN cd / && git clone https://github.com/numpy/numpy.git && cd /numpy && git submodule update --init && pip3 install . diff --git a/third_party/generate_dependencies.py b/third_party/generate_dependencies.py index cd7e19891..5a1d2f31a 100644 --- a/third_party/generate_dependencies.py +++ b/third_party/generate_dependencies.py @@ -34,57 +34,75 @@ def get_version(package): return package, url, version, license -if not os.path.exists("deps.txt"): - os.system("pipdeptree -p pm4py >deps.txt") +def elaborate_single_python_package(package_name, deps): + if not os.path.exists("deps.txt"): + os.system("pipdeptree -p "+package_name+" >deps.txt") -F = open("deps.txt", "r") -content = F.readlines() -F.close() -G = nx.DiGraph() -i = 1 -dep_level = {} -blocked = False -blocked_level = -1 -while i < len(content): - row = content[i].split("- ") - level = round(len(row[0]) / 2) - dep = row[1].split(" ")[0] - if blocked and blocked_level == level: - blocked = False - if dep == "pm4pycvxopt": - blocked = True - blocked_level = level - if not blocked: - dep_level[level] = dep - if level > 1: - G.add_edge(dep_level[level - 1], dep_level[level]) - else: - G.add_node(dep_level[level]) - i = i + 1 -edges = list(G.edges) -deps = [] -while len(edges) > 0: - left = set(x[0] for x in edges) - right = set(x[1] for x in edges) - diff = sorted([x for x in right if x not in left]) - for x in diff: + F = open("deps.txt", "r") + content = F.readlines() + F.close() + + if REMOVE_DEPS_AT_END: + os.remove("deps.txt") + + G = nx.DiGraph() + i = 1 + dep_level = {} + blocked = False + blocked_level = -1 + while i < len(content): + row = content[i].split("- ") + level = round(len(row[0]) / 2) + dep = row[1].split(" ")[0] + if blocked and blocked_level == level: + blocked = False + if dep == "pm4pycvxopt": + blocked = True + blocked_level = level + if not blocked: + dep_level[level] = dep + if level > 1: + G.add_edge(dep_level[level - 1], dep_level[level]) + else: + G.add_node(dep_level[level]) + i = i + 1 + edges = list(G.edges) + while len(edges) > 0: + left = set(x[0] for x in edges) + right = set(x[1] for x in edges) + diff = sorted([x for x in right if x not in left]) + for x in diff: + if not x in deps: + deps.append(x) + G.remove_node(x) + edges = list(G.edges) + nodes = sorted(list(G.nodes)) + for x in nodes: if not x in deps: deps.append(x) - G.remove_node(x) - edges = list(G.edges) -nodes = sorted(list(G.nodes)) -for x in nodes: - if not x in deps: - deps.append(x) -deps = sorted(deps, key=lambda x: x.lower()) -if "cvxopt" in deps: - del deps[deps.index("cvxopt")] -if "pm4py" in deps: - del deps[deps.index("pm4py")] -packages = [] -for x in deps: - packages.append(get_version(x)) + if "cvxopt" in deps: + del deps[deps.index("cvxopt")] + if "pm4py" in deps: + del deps[deps.index("pm4py")] + deps = sorted(deps, key=lambda x: x.lower()) + + return deps + + +def get_all_third_party_dependencies(package_name, deps, packages_dictio): + deps = elaborate_single_python_package(package_name, deps) + packages = [] + for x in deps: + if x not in packages_dictio: + packages_dictio[x] = get_version(x) + packages.append(packages_dictio[x]) + return deps, packages + + +deps = [] +packages_dictio = {} +deps, packages = get_all_third_party_dependencies("pm4py", deps, packages_dictio) if UPDATE_OTHER_FILES: F = open("../requirements_complete.txt", "w") @@ -109,8 +127,10 @@ def get_version(package): F.write("| %s | %s | %s | %s |\n" % (x[0].strip(), x[1].strip(), x[3].strip(), x[2].strip())) F.close() +deps, packages = get_all_third_party_dependencies("scikit-learn", deps, packages_dictio) + first_line_packages = ["deprecation", "packaging", "networkx", "graphviz", "six", "python-dateutil", "pytz", "tzdata", "intervaltree", "sortedcontainers"] -second_line_packages = ["pydotplus", "pyparsing", "tqdm", "colorama", "cycler"] +second_line_packages = ["pydotplus", "pyparsing", "tqdm", "colorama", "cycler", "joblib", "threadpoolctl"] first_packages_line = "" second_packages_line = "" @@ -151,6 +171,3 @@ def get_version(package): F.close() else: print(stru) - -if REMOVE_DEPS_AT_END: - os.remove("deps.txt") From fa936f160274fd90fd0fd928103c501a467a7ae3 Mon Sep 17 00:00:00 2001 From: Alessandro Berti Date: Thu, 11 Apr 2024 15:15:13 +0200 Subject: [PATCH 5/5] refactor(pm4py): updating dependencies generation script. updating Dockerfile --- Dockerfile | 3 ++- third_party/generate_dependencies.py | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c1c7d324b..84bc8b954 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,8 @@ RUN pip install -U wheel six pytest RUN pip install -U meson-python>=0.13.1 Cython>=3.0.6 ninja spin==0.8 build RUN pip install deprecation==2.1.0 graphviz==0.20.3 intervaltree==3.1.0 networkx==3.3 packaging==24.0 python-dateutil==2.9.0.post0 pytz==2024.1 six==1.16.0 sortedcontainers==2.4.0 tzdata==2024.1 RUN pip install colorama==0.4.6 cycler==0.12.1 joblib==1.4.0 pydotplus==2.0.2 pyparsing==3.1.2 threadpoolctl==3.4.0 tqdm==4.66.2 -RUN pip install contourpy==1.2.1 fonttools==4.51.0 kiwisolver==1.4.5 lxml==5.2.1 matplotlib==3.8.4 numpy==1.26.4 pandas==2.2.2 pillow==10.3.0 scipy==1.13.0 +RUN pip install lxml==5.2.1 numpy==1.26.4 pandas==2.2.2 scipy==1.13.0 +RUN pip install contourpy==1.2.1 fonttools==4.51.0 kiwisolver==1.4.5 matplotlib==3.8.4 pillow==10.3.0 #RUN cd / && git clone https://github.com/numpy/numpy.git && cd /numpy && git submodule update --init && pip3 install . #RUN cd / && git clone https://github.com/pandas-dev/pandas/ && cd /pandas && pip3 install . diff --git a/third_party/generate_dependencies.py b/third_party/generate_dependencies.py index 5a1d2f31a..9a8c7141a 100644 --- a/third_party/generate_dependencies.py +++ b/third_party/generate_dependencies.py @@ -131,18 +131,23 @@ def get_all_third_party_dependencies(package_name, deps, packages_dictio): first_line_packages = ["deprecation", "packaging", "networkx", "graphviz", "six", "python-dateutil", "pytz", "tzdata", "intervaltree", "sortedcontainers"] second_line_packages = ["pydotplus", "pyparsing", "tqdm", "colorama", "cycler", "joblib", "threadpoolctl"] +third_line_packages = ["lxml", "numpy", "pandas", "scipy"] first_packages_line = "" second_packages_line = "" third_packages_line = "" +fourth_package_line = "" + for x in packages: cont = x[0] + "==" + x[2] + " " if x[0] in first_line_packages: first_packages_line += cont elif x[0] in second_line_packages: second_packages_line += cont - else: + elif x[0] in third_line_packages: third_packages_line += cont + else: + fourth_package_line += cont F = open("../Dockerfile", "r") dockerfile_contents = F.readlines() @@ -162,7 +167,7 @@ def get_all_third_party_dependencies(package_name, deps, packages_dictio): before_lines.append(dockerfile_contents[i]) i = i + 1 -stru = "".join(before_lines + ["RUN pip install " + x + "\n" for x in [first_packages_line, second_packages_line, third_packages_line]] + after_lines) +stru = "".join(before_lines + ["RUN pip install " + x + "\n" for x in [first_packages_line, second_packages_line, third_packages_line, fourth_package_line]] + after_lines) stru = stru.strip() + "\n" if UPDATE_DOCKERFILE: