Merge remote-tracking branch 'upstream/hotfixes' into release

Dockerfile

@@ -15,7 +15,14 @@ RUN apt-get -y install libxml2-dev libxslt-dev libfreetype6-dev libsuitesparse-d
 RUN pip install -U wheel six pytest
 RUN pip install -U meson-python>=0.13.1 Cython>=3.0.6 ninja spin==0.8 build
 RUN pip install deprecation==2.1.0 graphviz==0.20.3 intervaltree==3.1.0 networkx==3.3 packaging==24.0 python-dateutil==2.9.0.post0 pytz==2024.1 six==1.16.0 sortedcontainers==2.4.0 tzdata==2024.1 
-RUN pip install colorama==0.4.6 contourpy==1.2.1 cycler==0.12.1 fonttools==4.51.0 kiwisolver==1.4.5 lxml==5.2.1 matplotlib==3.8.4 numpy==1.26.4 pandas==2.2.2 pillow==10.3.0 pydotplus==2.0.2 pyparsing==3.1.2 scipy==1.13.0 tqdm==4.66.2 
+RUN pip install colorama==0.4.6 cycler==0.12.1 joblib==1.4.0 pydotplus==2.0.2 pyparsing==3.1.2 threadpoolctl==3.4.0 tqdm==4.66.2 
+RUN pip install lxml==5.2.1 numpy==1.26.4 pandas==2.2.2 scipy==1.13.0 
+RUN pip install contourpy==1.2.1 fonttools==4.51.0 kiwisolver==1.4.5 matplotlib==3.8.4 pillow==10.3.0 
+
+#RUN cd / && git clone https://github.com/numpy/numpy.git && cd /numpy && git submodule update --init && pip3 install .
+#RUN cd / && git clone https://github.com/pandas-dev/pandas/ && cd /pandas && pip3 install .
+#RUN cd / && git clone https://github.com/scipy/scipy.git && cd /scipy && git submodule update --init && pip3 install .
+#RUN cd / && git clone https://github.com/lxml/lxml.git && cd /lxml && pip3 install .
 
 COPY . /app
 RUN cd /app && python setup.py install

third_party/generate_dependencies.py

@@ -34,58 +34,76 @@ def get_version(package):
     return package, url, version, license
 
 
-if not os.path.exists("deps.txt"):
-    os.system("pipdeptree -p pm4py >deps.txt")
+def elaborate_single_python_package(package_name, deps):
+    if not os.path.exists("deps.txt"):
+        os.system("pipdeptree -p "+package_name+" >deps.txt")
 
-F = open("deps.txt", "r")
-content = F.readlines()
-F.close()
-G = nx.DiGraph()
-i = 1
-dep_level = {}
-blocked = False
-blocked_level = -1
-while i < len(content):
-    row = content[i].split("- ")
-    level = round(len(row[0]) / 2)
-    dep = row[1].split(" ")[0]
-    if blocked and blocked_level == level:
-        blocked = False
-    if dep == "pm4pycvxopt":
-        blocked = True
-        blocked_level = level
-    if not blocked:
-        dep_level[level] = dep
-        if level > 1:
-            G.add_edge(dep_level[level - 1], dep_level[level])
-        else:
-            G.add_node(dep_level[level])
-    i = i + 1
-edges = list(G.edges)
-deps = []
-while len(edges) > 0:
-    left = set(x[0] for x in edges)
-    right = set(x[1] for x in edges)
-    diff = sorted([x for x in right if x not in left])
-    for x in diff:
+    F = open("deps.txt", "r")
+    content = F.readlines()
+    F.close()
+
+    if REMOVE_DEPS_AT_END:
+        os.remove("deps.txt")
+
+    G = nx.DiGraph()
+    i = 1
+    dep_level = {}
+    blocked = False
+    blocked_level = -1
+    while i < len(content):
+        row = content[i].split("- ")
+        level = round(len(row[0]) / 2)
+        dep = row[1].split(" ")[0]
+        if blocked and blocked_level == level:
+            blocked = False
+        if dep == "pm4pycvxopt":
+            blocked = True
+            blocked_level = level
+        if not blocked:
+            dep_level[level] = dep
+            if level > 1:
+                G.add_edge(dep_level[level - 1], dep_level[level])
+            else:
+                G.add_node(dep_level[level])
+        i = i + 1
+    edges = list(G.edges)
+    while len(edges) > 0:
+        left = set(x[0] for x in edges)
+        right = set(x[1] for x in edges)
+        diff = sorted([x for x in right if x not in left])
+        for x in diff:
+            if not x in deps:
+                deps.append(x)
+            G.remove_node(x)
+        edges = list(G.edges)
+    nodes = sorted(list(G.nodes))
+    for x in nodes:
         if not x in deps:
             deps.append(x)
-        G.remove_node(x)
-    edges = list(G.edges)
-nodes = sorted(list(G.nodes))
-for x in nodes:
-    if not x in deps:
-        deps.append(x)
-deps = sorted(deps, key=lambda x: x.lower())
-if "cvxopt" in deps:
-    del deps[deps.index("cvxopt")]
-if "pm4py" in deps:
-    del deps[deps.index("pm4py")]
-packages = []
-for x in deps:
-    packages.append(get_version(x))
+
+    if "cvxopt" in deps:
+        del deps[deps.index("cvxopt")]
+    if "pm4py" in deps:
+        del deps[deps.index("pm4py")]
+    deps = sorted(deps, key=lambda x: x.lower())
+
+    return deps
 
 
+def get_all_third_party_dependencies(package_name, deps, packages_dictio):
+    deps = elaborate_single_python_package(package_name, deps)
+    packages = []
+    for x in deps:
+        if x not in packages_dictio:
+            packages_dictio[x] = get_version(x)
+        packages.append(packages_dictio[x])
+    return deps, packages
+
+
+deps = []
+packages_dictio = {}
+deps, packages = get_all_third_party_dependencies("pm4py", deps, packages_dictio)
+
 if UPDATE_OTHER_FILES:
     F = open("../requirements_complete.txt", "w")
     for x in packages:
@@ -109,15 +127,27 @@ def get_version(package):
         F.write("| %s | %s | %s | %s |\n" % (x[0].strip(), x[1].strip(), x[3].strip(), x[2].strip()))
     F.close()
 
+deps, packages = get_all_third_party_dependencies("scikit-learn", deps, packages_dictio)
+
 first_line_packages = ["deprecation", "packaging", "networkx", "graphviz", "six", "python-dateutil", "pytz", "tzdata", "intervaltree", "sortedcontainers"]
+second_line_packages = ["pydotplus", "pyparsing", "tqdm", "colorama", "cycler", "joblib", "threadpoolctl"]
+third_line_packages = ["lxml", "numpy", "pandas", "scipy"]
+
 first_packages_line = ""
 second_packages_line = ""
+third_packages_line = ""
+fourth_package_line = ""
+
 for x in packages:
     cont = x[0] + "==" + x[2] + " "
     if x[0] in first_line_packages:
         first_packages_line += cont
-    else:
+    elif x[0] in second_line_packages:
         second_packages_line += cont
+    elif x[0] in third_line_packages:
+        third_packages_line += cont
+    else:
+        fourth_package_line += cont
 
 F = open("../Dockerfile", "r")
 dockerfile_contents = F.readlines()
@@ -137,7 +167,7 @@ def get_version(package):
         before_lines.append(dockerfile_contents[i])
     i = i + 1
 
-stru = "".join(before_lines + ["RUN pip install " + x + "\n" for x in [first_packages_line, second_packages_line]] + after_lines)
+stru = "".join(before_lines + ["RUN pip install " + x + "\n" for x in [first_packages_line, second_packages_line, third_packages_line, fourth_package_line]] + after_lines)
 stru = stru.strip() + "\n"
 
 if UPDATE_DOCKERFILE:
@@ -146,6 +176,3 @@ def get_version(package):
     F.close()
 else:
     print(stru)
-
-if REMOVE_DEPS_AT_END:
-    os.remove("deps.txt")