Fix requirements

Dockerfile

@@ -1,21 +1,23 @@
-FROM python:3.8
+FROM python:3.10
 
 WORKDIR /usr/d2m
 
 RUN mkdir -p assets/data/raw
 
-COPY src ./src
-COPY dvc.yaml ./dvc.yaml
-COPY params_default.yaml ./params.yaml
-COPY params_default.yaml ./params_default.yaml
 COPY requirements.txt ./requirements.txt
 
 RUN pip3 install -r requirements.txt
 RUN pip3 install dvc
 RUN pip3 install flask flask-restful
 
+COPY src ./src
+COPY dvc.yaml ./dvc.yaml
+COPY params_default.yaml ./params.yaml
+COPY params_default.yaml ./params_default.yaml
+
 EXPOSE 5000
 
+RUN git init
 RUN dvc init --no-scm
 
 CMD ["python3", "src/api.py"]

assets/.gitignore

@@ -0,0 +1,2 @@
+/profile
+/models

assets/data/.gitignore

@@ -4,3 +4,4 @@
 /scaled
 /sequentialized
 /combined
+/cleaned

requirements.txt

@@ -1,330 +1,17 @@
-absl-py==1.4.0
-aiofiles==22.1.0
-aiohttp==3.8.4
-aiohttp-retry==2.8.3
-aiosignal==1.3.1
-aiosqlite==0.19.0
-amqp==5.1.1
-aniso8601==9.0.1
-antlr4-python3-runtime==4.9.3
-anyio==3.6.2
-appdirs==1.4.4
-argon2-cffi==21.3.0
-argon2-cffi-bindings==21.2.0
-arrow==1.2.3
-asgiref==3.7.2
-asttokens==2.2.1
-astunparse==1.6.3
-async-timeout==4.0.2
-asyncssh==2.13.1
-atpublic==3.1.1
-attrs==23.1.0
-Babel==2.12.1
-backcall==0.2.0
-beautifulsoup4==4.12.2
-billiard==3.6.4.0
-black==23.7.0
-boltons @ file:///croot/boltons_1677628692245/work
-brotlipy==0.7.0
-bs4==0.0.1
-cachetools==5.3.1
-celery==5.2.7
-certifi @ file:///croot/certifi_1671487769961/work/certifi
-cffi==1.15.1
-charset-normalizer==2.1.1
-click==8.1.7
-click-didyoumean==0.3.0
-click-plugins==1.1.1
-click-repl==0.2.0
-cloudpickle==2.2.1
-cmake==3.27.0
-codecarbon==2.3.1
-comm==0.1.3
-conda==23.3.1
-conda-content-trust @ file:///tmp/abs_5952f1c8-355c-4855-ad2e-538535021ba5h26t22e5/croots/recipe/conda-content-trust_1658126371814/work
-conda-package-handling @ file:///croot/conda-package-handling_1672865015732/work
-conda_package_streaming @ file:///croot/conda-package-streaming_1670508151586/work
-configobj==5.0.8
-contourpy==1.0.7
-cryptography==41.0.4
-cycler==0.11.0
-dash==2.10.2
-dash-bootstrap-components==1.4.1
-dash-core-components==2.0.0
-dash-html-components==2.0.0
-dash-table==5.0.0
-dataclasses-json==0.5.14
-dateparser==1.1.1
-debugpy==1.6.7
-decorator==5.1.1
-defusedxml==0.7.1
-dictdiffer==0.9.0
-diskcache==5.6.1
-dpath==2.1.5
-dulwich==0.21.3
-dvc==2.54.0
-dvc-data==0.47.2
-dvc-http==2.30.2
-dvc-objects==0.21.1
-dvc-render==0.3.1
-dvc-studio-client==0.7.0
-dvc-task==0.2.0
-exceptiongroup==1.1.2
-executing==1.2.0
-fairscale==0.4.13
-fastapi==0.77.1
-fastjsonschema==2.16.3
-filelock==3.12.0
-fire==0.5.0
-Flask==2.2.5
-Flask-RESTful==0.3.10
-flatbuffers==23.5.26
-flatten-dict==0.4.2
-flufl.lock==7.1.1
-flwr==1.5.0
-fonttools==4.39.3
-fqdn==1.5.1
-frozenlist==1.3.3
-fsspec==2023.4.0
-funcy==2.0
-future==0.18.3
-fuzzywuzzy==0.18.0
-gast==0.4.0
-gitdb==4.0.10
-GitPython==3.1.31
-google-auth==2.23.3
-google-pasta==0.2.0
-gpt4all==1.0.8
-grandalf==0.8
-greenlet==2.0.2
-grpcio==1.54.0
-h11==0.14.0
-h5py==3.8.0
-htmlmin==0.1.12
-huggingface-hub==0.16.4
-hydra-core==1.3.2
-hyperopt==0.2.7
-idna @ file:///croot/idna_1666125576474/work
-ImageHash==4.3.1
-imageio==2.31.1
-ipykernel==6.22.0
-ipython==8.12.0
-ipython-genutils==0.2.0
-isoduration==20.11.0
-isort==5.12.0
-iterative-telemetry==0.0.8
-iterators==0.0.2
-itsdangerous==2.1.2
-jax==0.4.8
-jedi==0.18.2
-Jinja2==3.1.2
-joblib==1.2.0
-json5==0.9.11
-jsonpatch @ file:///tmp/build/80754af9/jsonpatch_1615747632069/work
-jsonpointer==2.3
-jupyter-events==0.6.3
-jupyter-ydoc==0.2.4
-jupyter_client==8.2.0
-jupyter_core==5.3.0
-jupyter_server==2.5.0
-jupyter_server_fileid==0.9.0
-jupyter_server_terminals==0.4.4
-jupyter_server_ydoc==0.8.0
-jupyterlab==3.6.3
-jupyterlab-pygments==0.2.2
-jupyterlab_server==2.22.1
-kaleido==0.2.1
-keras==2.14.0
-keras-tuner==1.3.5
-khoj-assistant==0.10.0
-kiwisolver==1.4.4
-kombu==5.2.4
-kt-legacy==1.0.5
-langchain==0.0.247
-langsmith==0.0.15
-lazy_loader==0.3
-libclang==16.0.0
-lightgbm==3.3.5
-lime==0.2.0.1
-lit==16.0.6
-llvmlite==0.39.1
-Markdown==3.4.3
-markdown-footnotes==0.1.1
-markdown-it-py==2.2.0
-MarkupSafe==2.1.2
-marshmallow==3.20.1
-matplotlib==3.6.3
-matplotlib-inline==0.1.6
-mdurl==0.1.2
-mistune==2.0.5
-ml-dtypes==0.2.0
-mpmath==1.3.0
-multidict==6.0.4
-multimethod==1.9.1
-mypy-extensions==1.0.0
-nanotime==0.5.2
-nbclassic==0.5.5
-nbclient==0.7.3
-nbconvert==7.3.1
-nbformat==5.8.0
-nest-asyncio==1.5.6
-networkx==3.1
-notebook==6.5.4
-notebook_shim==0.2.2
-numba==0.56.4
-numexpr==2.8.4
-numpy==1.26.0
-nvidia-cublas-cu11==11.10.3.66
-nvidia-cuda-cupti-cu11==11.7.101
-nvidia-cuda-nvrtc-cu11==11.7.99
-nvidia-cuda-runtime-cu11==11.7.99
-nvidia-cudnn-cu11==8.5.0.96
-nvidia-cufft-cu11==10.9.0.58
-nvidia-curand-cu11==10.2.10.91
-nvidia-cusolver-cu11==11.4.0.1
-nvidia-cusparse-cu11==11.7.4.91
-nvidia-nccl-cu11==2.14.3
-nvidia-nvtx-cu11==11.7.91
-oauthlib==3.2.2
-omegaconf==2.3.0
-openai==0.27.4
-openapi-schema-pydantic==1.2.4
-opt-einsum==3.3.0
-orjson==3.8.10
-outcome==1.2.0
-packaging==23.1
-paho-mqtt==1.6.1
-pandas-profiling==3.6.6
-pandocfilters==1.5.0
-parso==0.8.3
-pathspec==0.11.1
-patsy==0.5.3
-phik==0.12.3
-pickleshare==0.7.5
-Pillow==9.4.0
-platformdirs==3.2.0
-plotly==5.14.1
-pluggy @ file:///tmp/build/80754af9/pluggy_1648024709248/work
-prometheus-client==0.16.0
-prompt-toolkit==3.0.38
-protobuf==3.20.3
-psutil==5.9.5
-pure-eval==0.2.2
-py-cpuinfo==9.0.0
-py4j==0.10.9.7
-pyasn1==0.5.0
-pyasn1-modules==0.3.0
-pycodestyle==2.11.0
-pycosat @ file:///croot/pycosat_1666805502580/work
-pycparser==2.21
-pycryptodome==3.19.0
-pydot==1.4.2
-pygit2==1.12.0
-Pygments==2.15.1
-pygtrie==2.5.0
-pynvml==11.5.0
-pyOpenSSL @ file:///opt/conda/conda-bld/pyopenssl_1643788558760/work
-pypdf==3.14.0
-PyPDF2==3.0.1
-pyrsistent==0.19.3
-PySide6==6.5.2
-PySide6-Addons==6.5.2
-PySide6-Essentials==6.5.2
-PySocks @ file:///home/builder/ci_310/pysocks_1640793678128/work
-pytz==2023.3.post1
-PyWavelets==1.4.1
-PyYAML==6.0
-pyzmq==25.0.2
-regex==2022.3.2
-requests @ file:///opt/conda/conda-bld/requests_1657734628632/work
-requests-oauthlib==1.3.1
-rfc3339-validator==0.1.4
-rfc3986-validator==0.1.1
-rich==13.3.4
-rsa==4.9
-ruamel.yaml==0.17.21
-ruamel.yaml.clib==0.2.7
-ruff==0.0.261
-safetensors==0.3.1
-schedule==1.1.0
-scikit-image==0.21.0
-scikit-learn==1.2.2
-scipy==1.9.3
-scmrepo==1.0.2
-seaborn==0.12.2
-selenium==4.11.2
-Send2Trash==1.8.0
-sentence-transformers==2.2.2
-sentencepiece==0.1.99
-shap==0.42.0
-shiboken6==6.5.2
-shortuuid==1.0.11
-shtab==1.6.1
-six @ file:///tmp/build/80754af9/six_1644875935023/work
-sklearn==0.0.post5
-slicer==0.0.7
-smmap==5.0.0
-sniffio==1.3.0
-sortedcontainers==2.4.0
-soupsieve==2.4.1
-SQLAlchemy==2.0.19
-sqltrie==0.3.1
-srt==3.5.3
-stack-data==0.6.2
-starlette==0.19.1
-statsmodels==0.13.5
-sympy==1.12
-tabulate==0.9.0
-tangled-up-in-unicode==0.2.0
-tenacity==8.2.2
-tensorboard==2.14.1
-tensorboard-plugin-wit==1.8.1
-tensorflow-cpu==2.14.0
-tensorflow-estimator==2.14.0
-tensorflow-io-gcs-filesystem==0.32.0
-termcolor==2.3.0
-terminado==0.17.1
-threadpoolctl==3.1.0
-tifffile==2023.7.4
-tiktoken==0.4.0
-tinycss2==1.2.1
-tokenizers==0.13.3
-tomli==2.0.1
-tomlkit==0.11.7
-toolz @ file:///croot/toolz_1667464077321/work
-torch==2.0.1
-torchvision==0.15.2
-tornado==6.3
-tqdm==4.64.1
-traitlets==5.9.0
-transformers==4.31.0
-trio==0.22.2
-trio-websocket==0.10.3
-triton==2.0.0
-typeguard==2.13.3
-typing-inspect==0.9.0
-typing_extensions==4.8.0
-tzdata==2023.3
-tzlocal==5.0.1
-UNKNOWN==0.0.0
-uri-template==1.2.0
-urllib3 @ file:///croot/urllib3_1673575502006/work
-uvicorn==0.17.6
-vine==5.0.0
-visions==0.7.5
-voluptuous==0.13.1
-vosk==0.3.45
-wcwidth==0.2.6
-webcolors==1.13
-webencodings==0.5.1
-websocket-client==1.5.1
-Werkzeug==2.2.3
-wrapt==1.14.1
-wsproto==1.2.0
-xgboost==1.7.6
-y-py==0.5.9
-yarl==1.8.2
-ydata-profiling==4.1.2
-ypy-websocket==0.8.2
-zc.lockfile==3.0.post1
-zstandard @ file:///opt/conda/conda-bld/zstandard_1663827383994/work
+numpy
+matplotlib
+plotly
+pyyaml
+tensorflow
+scikit-learn
+flask
+lime
+shap
+codecarbon
+pandas
+scipy
+ydata-profiling
+pydantic
+keras-tuner
+xgboost
+lightgbm

src/clean.py

@@ -211,7 +211,11 @@ def parse_profile_warnings(self):
             messages = profile_json["alerts"]
 
         variables = list(profile_json["variables"].keys())
-        correlations = profile_json["correlations"]["auto"]
+
+        try:
+            correlations = profile_json["correlations"]["auto"]
+        except:
+            correlations = None
 
         removable_features = []