From 4432046cefaf541a65de28eeb1c7e56a4432f738 Mon Sep 17 00:00:00 2001 From: Saurabh Kumar Date: Tue, 3 Jan 2023 16:03:21 +0530 Subject: [PATCH] feat: Use poetry everywhere, remove vagrant support - Update ansible to use poetry - Do not generate requirements.txt files from poetry - Upgrade poetry version to 1.3.1 --- .github/workflows/main.yml | 10 ++-- README.md | 28 +---------- hooks/post_gen_project.sh | 6 +-- .../.github/workflows/main.yml | 23 ++++----- {{cookiecutter.github_repository}}/Makefile | 28 +++-------- {{cookiecutter.github_repository}}/README.md | 16 ------- .../Vagrantfile | 48 ------------------- .../compose/dev/django/Dockerfile | 2 +- .../compose/local/Dockerfile | 2 +- .../docs/backend/server_config.md | 1 + .../provisioner/hosts | 19 -------- .../provisioner/roles/common/tasks/main.yml | 15 ------ .../provisioner/roles/nginx/tasks/main.yml | 7 ++- .../roles/nginx/templates/site.80.conf.j2 | 4 +- .../roles/project_data/defaults/main.yml | 1 - .../roles/project_data/tasks/main.yml | 35 ++++++++------ .../provisioner/site.yml | 20 -------- .../provisioner/vars.yml | 1 - .../runtime.txt | 1 - 19 files changed, 54 insertions(+), 213 deletions(-) delete mode 100644 {{cookiecutter.github_repository}}/Vagrantfile delete mode 100644 {{cookiecutter.github_repository}}/runtime.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4e00e7d..feac8db3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,13 +28,9 @@ jobs: with: python-version: "3.9" cache: "pip" - - name: Install cookiecutter + - name: Install cookiecutter and poetry run: | python -m pip install --upgrade pip - pip3 install cookiecutter==1.7.3 - - name: Install poetry - run: | - pip3 install poetry==1.2.0 + pip3 install cookiecutter==2.1.1 poetry==1.3.1 - name: Run tests - run: | - ./run_test.sh + run: bash run_test.sh diff --git a/README.md b/README.md index b8eb5b20..57e43dc1 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ If you opt to setup the project automatically, it will also: - initialize a git repo and bump initial tag and version. - create a virtualenv in the folder `venv` inside the project. - install all the python dependencies inside it. -- create `poetry.lock` file after resolving dependencies and then generate `requirements.txt` and `requirements_dev.txt` for production and dev use respectively, for backward-compatibility. +- create `poetry.lock` file after resolving dependencies. - create a postgres database and run the initial migration against it. then only thing you'll need to do is: @@ -65,32 +65,6 @@ Don't forget to carefully look at the generated README. Awesome, right? You can also explore the [wiki] section for details on advance setup and usages. -## Managing dependencies - -### Poetry - -To guarantee repeatable installations, all project dependencies are managed using [Poetry](https://python-poetry.org/). The project’s direct dependencies are listed in `pyproject.toml`. -Running `poetry lock` generates `poetry.lock` which has all versions pinned. - -You can install Poetry by using `pip install --pre poetry` or by following the official installation guide [here](https://github.com/python-poetry/poetry#installation). - -*Tip:* We recommend that you use this workflow and keep `pyproject.toml` as well as `poetry.lock` under version control to make sure all computers and environments run exactly the same code. - -### Other tools - -For compatibility, `requirements.txt` and `requirements_dev.txt` can be updated by running - -```bash -poetry export -f requirements.txt -o requirements.txt -poetry export -f requirements.txt -o requirements_dev.txt --dev -``` - -or - -```bash -make generate_requirements -``` - ## Articles - [Setting up Django projects in a breeze](https://medium.com/fueled-engineering/setting-up-django-projects-in-a-breeze-36c715cc9a6f) diff --git a/hooks/post_gen_project.sh b/hooks/post_gen_project.sh index af6c0a0a..40a77b5b 100755 --- a/hooks/post_gen_project.sh +++ b/hooks/post_gen_project.sh @@ -10,7 +10,7 @@ echo "${green}[Finished]${reset}" echo "==> Setup project dependencies? It will:" echo " - Create virtualenv at './{{ cookiecutter.github_repository }}/venv/'." -echo " - Install development requirements inside virtualenv." +echo " - Install development requirements using poetry" echo " - Create a postgres database named '{{ cookiecutter.main_module }}'." echo " - Run './manage.py migrate'." echo " - Initialize git." @@ -26,11 +26,11 @@ else fi if echo "{{ cookiecutter.add_heroku }}" | grep -iq "^n"; then - rm -rf uwsgi.ini Procfile runtime.txt bin/post_compile + rm -rf uwsgi.ini Procfile bin/post_compile fi if echo "{{ cookiecutter.add_ansible }}" | grep -iq "^n"; then - rm -rf provisioner Vagrantfile ansible.cfg + rm -rf provisioner ansible.cfg fi if echo "{{ cookiecutter.add_celery }}" | grep -iq "^n"; then diff --git a/{{cookiecutter.github_repository}}/.github/workflows/main.yml b/{{cookiecutter.github_repository}}/.github/workflows/main.yml index 32912a6a..b12d5776 100644 --- a/{{cookiecutter.github_repository}}/.github/workflows/main.yml +++ b/{{cookiecutter.github_repository}}/.github/workflows/main.yml @@ -33,21 +33,22 @@ jobs: sudo apt-get update sudo apt-get install postgresql-13-postgis-3-scripts {%- endif %} + - name: Set up Python 3.9 uses: actions/setup-python@v2 with: python-version: '3.9' cache: 'pip' - - name: Install poetry - run: | - pip3 install poetry==1.2.0 - - name: Install requirements + + - name: Install python dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip poetry wheel poetry install --with dev - - name: Run tests - run: | - poetry run pytest --cov -v --tb=native - - name: Linting - run: | - make lint + + - name: Run Linting + run: make lint + + - name: Run Python tests + run: poetry run pytest --cov -v --tb=native + + diff --git a/{{cookiecutter.github_repository}}/Makefile b/{{cookiecutter.github_repository}}/Makefile index b5a3e75b..cbc80bf8 100644 --- a/{{cookiecutter.github_repository}}/Makefile +++ b/{{cookiecutter.github_repository}}/Makefile @@ -4,8 +4,10 @@ SHELL := bash PROJECT_NAME={{cookiecutter.main_module}} DB_NAME=$(PROJECT_NAME) +{% if cookiecutter.add_ansible.lower() == 'y' -%} INVENTORY=provisioner/hosts PLAYBOOK=provisioner/site.yml +{%- endif %} ENV_PREFIX=$(shell echo 'poetry run ') @@ -36,32 +38,14 @@ run_all: ## Run all the servers in parallel, requires GNU Make make -j django docs{% if cookiecutter.add_celery == 'y' %} celery{% endif %} redis .PHONY: run_all -virtualenv: ## Create a virtual environment. - @echo "creating virtualenv using poetry..." - @pip install -U pip poetry - @poetry env use python3 - @echo - @echo "!!! Please run 'poetry shell' to enable the environment !!!" - - regenerate: ## Delete and create new database. -dropdb $(DB_NAME) createdb $(DB_NAME) ${ENV_PREFIX}python manage.py migrate .PHONY: regenerate -generate_requirements: - poetry export -f requirements.txt -o requirements_dev.txt --dev - poetry export -f requirements.txt -o requirements.txt - -update_libs: ## update libs + generate new lockfile & requirements - poetry update - make generate_requirements -.PHONY: update-libs - -install: virtualenv ## Install and setup project dependencies - python3 -m pip install --upgrade pip wheel - make generate_requirements +install: ## Install dependencies, create db and migrate + python3 -m pip install --upgrade pip wheel poetry poetry install ${ENV_PREFIX}pre-commit install ifneq ($(CI),True) @@ -99,7 +83,7 @@ djurls: ## Displays all the django urls shell: ## Enter the django shell ${ENV_PREFIX}python manage.py shell_plus -docs: virtualenv ## Start documentation server locally +docs: ## Start documentation server locally ${ENV_PREFIX}mkdocs serve {%- if cookiecutter.add_celery == 'y' %} @@ -110,6 +94,7 @@ celery: install ## Start celery worker redis: ## Start redis server redis-server +{% if cookiecutter.add_ansible.lower() == 'y' %} # Ansible related things # ------------------------------------------------------ # Usages: @@ -138,3 +123,4 @@ deploy_qa: deploy deploy_prod: ENV=prod ## Deploy to production server deploy_prod: deploy +{%- endif %} diff --git a/{{cookiecutter.github_repository}}/README.md b/{{cookiecutter.github_repository}}/README.md index 592b7e8c..4981f93b 100644 --- a/{{cookiecutter.github_repository}}/README.md +++ b/{{cookiecutter.github_repository}}/README.md @@ -45,22 +45,6 @@ You can install Poetry by using `pip install --pre poetry` or by following the o *Tip:* We recommend that you use this workflow and keep `pyproject.toml` as well as `poetry.lock` under version control to make sure all computers and environments run exactly the same code. -### Other tools - -For compatibility, `requirements.txt` and `requirements_dev.txt` can be updated by running - -```bash -poetry export --without-hashes -f requirements.txt -o requirements.txt -``` - -and - -```bash -poetry export --without-hashes -f requirements.txt -o requirements_dev.txt --with dev -``` - -, respectively. - ## Deploying Project diff --git a/{{cookiecutter.github_repository}}/Vagrantfile b/{{cookiecutter.github_repository}}/Vagrantfile deleted file mode 100644 index 6e3e5622..00000000 --- a/{{cookiecutter.github_repository}}/Vagrantfile +++ /dev/null @@ -1,48 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - - config.vm.box = "bento/ubuntu-16.04" - - # Make virtualbox use 1GB RAM. Compilation of libraries like lxml may fail for - # less than 1GB RAM. - config.vm.provider "virtualbox" do |v| - v.memory = 1024 - end - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - config.vm.network :forwarded_port, guest: 80, host: 8080 - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - config.vm.network :private_network, ip: "192.168.33.12" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network :public_network - - # If true, then any SSH connections made will enable agent forwarding. - # Default value: false - # config.ssh.forward_agent = true - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - config.vm.synced_folder "./", "/home/vagrant/{{ cookiecutter.github_repository }}/" - - config.vm.provision "ansible" do |ansible| - ansible.playbook = "provisioner/site.yml" - ansible.host_key_checking = "False" - ansible.inventory_path = "provisioner/hosts" - ansible.verbose = "v" - ansible.limit = "vagrant" - end -end diff --git a/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile b/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile index 72027a2a..016b993d 100644 --- a/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile +++ b/{{cookiecutter.github_repository}}/compose/dev/django/Dockerfile @@ -3,7 +3,7 @@ ARG PYTHON_VERSION=3.9-slim-buster # define an alias for the specfic python version used in this file. FROM python:${PYTHON_VERSION} as python -ENV POETRY_VERSION=1.2.0 +ENV POETRY_VERSION=1.3.1 ARG BUILD_ENVIRONMENT=dev ARG APP_HOME=/app diff --git a/{{cookiecutter.github_repository}}/compose/local/Dockerfile b/{{cookiecutter.github_repository}}/compose/local/Dockerfile index 29f5045d..b50cf68f 100644 --- a/{{cookiecutter.github_repository}}/compose/local/Dockerfile +++ b/{{cookiecutter.github_repository}}/compose/local/Dockerfile @@ -6,7 +6,7 @@ FROM python:${PYTHON_VERSION} as python ARG BUILD_ENVIRONMENT=local ARG APP_HOME=/app -ENV POETRY_VERSION=1.2.0 +ENV POETRY_VERSION=1.3.1 ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 ENV BUILD_ENV ${BUILD_ENVIRONMENT} diff --git a/{{cookiecutter.github_repository}}/docs/backend/server_config.md b/{{cookiecutter.github_repository}}/docs/backend/server_config.md index f99012b6..49c9782b 100644 --- a/{{cookiecutter.github_repository}}/docs/backend/server_config.md +++ b/{{cookiecutter.github_repository}}/docs/backend/server_config.md @@ -55,6 +55,7 @@ Run these commands to deploy this project on Heroku (substitue all references of ``` heroku create --ssh-git +heroku buildpacks:add https://github.com/moneymeets/python-poetry-buildpack.git --app= heroku buildpacks:set heroku/python --app= heroku addons:create heroku-postgresql{% if cookiecutter.add_postgis == 'y' %}:standard-0{% endif %} --app= diff --git a/{{cookiecutter.github_repository}}/provisioner/hosts b/{{cookiecutter.github_repository}}/provisioner/hosts index 36decf3c..d2ae7f42 100644 --- a/{{cookiecutter.github_repository}}/provisioner/hosts +++ b/{{cookiecutter.github_repository}}/provisioner/hosts @@ -1,31 +1,12 @@ [all:vars] -vm=0 user=ubuntu project_namespace={% raw %}{{ project_name }}-{{ deploy_env }}{% endraw %} project_path=/home/ubuntu/{% raw %}{{ deploy_env }}{% endraw %}/{{ cookiecutter.github_repository }} venv_path={% raw %}{{ project_path }}/venv{% endraw %} use_letsencrypt={{ 'True' if cookiecutter.letsencrypt.lower() == 'y' else 'False' }} letsencrypt_email={{ cookiecutter.letsencrypt_email }} -django_requirements_file=requirements.txt django_settings="settings.production" -[vagrant] -192.168.33.12 - -[vagrant:vars] -vm=1 -deploy_env=vagrant -user=vagrant -project_path=/home/vagrant/{{ cookiecutter.github_repository }} -venv_path=/home/vagrant/venv -django_requirements_file=requirements_dev.txt -django_settings="settings.development" -use_letsencrypt=False -pg_db={{ cookiecutter.main_module }} -pg_user=vagrant -pg_password=vagrant -domain_name=vagrant.{{ cookiecutter.main_module }}.com - [dev] dev.{{ cookiecutter.main_module }}.com diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml index 7daee0e7..a865304d 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/common/tasks/main.yml @@ -1,21 +1,6 @@ {% raw %}--- - -# Uncomment following if ipv6 is not available or not tunelled through ipv4 properly -# Disable ipv6 when running in VM(vagrant) -#- name: Disable ipv6 for all interfaces -# sysctl: name="net.ipv6.conf.all.disable_ipv6" value=1 state=present -# when: vm == 1 -# -#- name: Disable ipv6 for default interface -# sysctl: name="net.ipv6.conf.default.disable_ipv6" value=1 state=present -# when: vm == 1 -# -#- name: Disable ipv6 for local interface -# sysctl: name="net.ipv6.conf.lo.disable_ipv6" value=1 state=present -# when: vm == 1 - name: Set hostname action: shell hostnamectl set-hostname {{ domain_name }} - when: vm == 0 - name: set system locale command: update-locale LC_ALL={{ lc_all }} LANG={{ lc_lang }} LC_CTYPE={{ lc_ctype }} LC_COLLATE={{ lc_collate }} diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml index 5b869eff..260cb393 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/tasks/main.yml @@ -9,7 +9,7 @@ - name: make sure ssl directory exists file: path={{ ssl_cert_dir }} state=directory - when: vm == 0 and use_letsencrypt + when: use_letsencrypt - name: check {{ letsencrypt_ssl_cert_dir }} exists stat: path={{ letsencrypt_ssl_cert_dir }} @@ -18,7 +18,7 @@ - import_tasks: htpasswd.yml - import_tasks: letsencrypt.yml - when: vm == 0 and use_letsencrypt and letsencrypt_dir.stat.exists == false + when: use_letsencrypt and letsencrypt_dir.stat.exists == false - name: check ssl/nginx.crt exists stat: path={{ ssl_certificate }} @@ -29,11 +29,10 @@ register: nginx_key - fail: msg="Whoops! ssl certificate doesn't exist" - when: (vm == 0 and use_letsencrypt) == true and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) + when: use_letsencrypt and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) - name: generate ssl forward secrecy key command: openssl dhparam -out {{ ssl_forward_secrecy_key_path }} {{ ssl_forward_secrecy_key_length }} creates={{ ssl_forward_secrecy_key_path }} - when: vm == 0 - name: copy base nginx configuration. template: src=nginx.conf.j2 dest=/etc/nginx/nginx.conf diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 index 78bc0d98..31cb1b43 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 +++ b/{{cookiecutter.github_repository}}/provisioner/roles/nginx/templates/site.80.conf.j2 @@ -2,7 +2,7 @@ server { listen 80; listen [::]:80; - server_name {% if vm %}_{% else %}{{ domain_name }}{% endif %}; + server_name {{ domain_name }}; {% if use_letsencrypt %} location /.well-known/acme-challenge/ { @@ -11,7 +11,7 @@ server { } {% endif %} - {% if vm and (nginx_cert.stat.exists == false or nginx_key.stat.exists == false) %} + {% if nginx_cert.stat.exists == false or nginx_key.stat.exists == false %} location / { uwsgi_pass unix:///tmp/uwsgi-{{ project_namespace }}.sock; include /etc/nginx/uwsgi_params; diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml index 67012813..2c02b9c2 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/defaults/main.yml @@ -3,7 +3,6 @@ pg_hstore: False pg_db: "{{ project_namespace }}" pg_user: dev pg_password: password -django_requirements_file: requirements.txt # uwsgi related variables uwsgi_user: www-data diff --git a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml index ec32104e..ff5d9276 100644 --- a/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml +++ b/{{cookiecutter.github_repository}}/provisioner/roles/project_data/tasks/main.yml @@ -3,15 +3,13 @@ git: repo={{ project_repo_url }} dest={{ project_path }} version={{ repo_version }} accept_hostkey=true become: false register: gitresult - when: vm == 0 tags: ['always'] - debug: msg="Git SHA-1 before={{ gitresult.before }} after={{ gitresult.after }}" - when: vm == 0 tags: ['always'] -- name: Ensure python virtualenv folder exist - command: python3 -m venv {{ venv_path }} creates={{ venv_path }} +- name: Ensure poetry is installed + command: python3 -m pip install poetry become: false tags: ['always'] @@ -23,8 +21,10 @@ cache_valid_time: 300 when: pg_gis -- name: Install python dependencies - pip: requirements={{ project_path }}/{{ django_requirements_file }} executable={{ venv_path }}/bin/pip +- name: Install python dependencies with Poetry + command: "poetry install" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] @@ -38,20 +38,26 @@ pkg: gettext state: present -- name: Run collect static - django_manage: command=collectstatic app_path={{ project_path }} virtualenv={{ venv_path }} +- name: Run Django collectstatic + command: "poetry run python manage.py collectstatic --no-input" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] -- name: Run database migrations - django_manage: command=migrate app_path={{ project_path }} virtualenv={{ venv_path }} +- name: Run Django database migrations + command: "poetry run python manage.py migrate" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] - import_tasks: uwsgi-setup.yml - name: Run compilemessages for static translations - django_manage: command=compilemessages app_path={{ project_path }} virtualenv={{ venv_path }} + command: "poetry run python manage.py compilemessages" + args: + chdir: "{{ project_path }}" become: false tags: ['deploy'] @@ -67,16 +73,15 @@ {% raw %}- name: apt_get install graphviz for db schema generation apt: pkg=graphviz state=present -- name: Generate DB Schema. - shell: "source {{ venv_path }}/bin/activate && python bin/generate_db_schema.py" +- name: Generate DB Schema for documentation + command: "poetry run python bin/generate_db_schema.py" args: chdir: "{{ project_path }}" - executable: /bin/bash become: false tags: ['deploy', 'documentation'] - name: Build documentation for "/docs" url. - command: "{{ venv_path }}/bin/mkdocs build" + command: "poetry run mkdocs build" args: chdir: "{{ project_path }}" become: false diff --git a/{{cookiecutter.github_repository}}/provisioner/site.yml b/{{cookiecutter.github_repository}}/provisioner/site.yml index bfaa2cac..bec93a33 100644 --- a/{{cookiecutter.github_repository}}/provisioner/site.yml +++ b/{{cookiecutter.github_repository}}/provisioner/site.yml @@ -1,24 +1,4 @@ --- -- hosts: vagrant - vars_files: - - vars.yml - gather_facts: true - become: true - become_method: sudo - - roles: - - common - - nginx - - postgresql - - project_data -{%- if cookiecutter.add_celery.lower() == 'y' %} - - redis - - celery -{%- else %} - # - redis - # - celery -{%- endif %} - #= Dev #=================================================== - hosts: dev diff --git a/{{cookiecutter.github_repository}}/provisioner/vars.yml b/{{cookiecutter.github_repository}}/provisioner/vars.yml index b57c1e1e..4598924a 100644 --- a/{{cookiecutter.github_repository}}/provisioner/vars.yml +++ b/{{cookiecutter.github_repository}}/provisioner/vars.yml @@ -6,7 +6,6 @@ repo_version: master pg_gis: {{ 'True' if cookiecutter.add_postgis.lower() == 'y' else 'False' }} ansible_python_interpreter: /usr/bin/python3 django_settings: settings.production -django_requirements_file: requirements.txt python_version: 3.9 postgresql_version: 13 postgis_version: 3 diff --git a/{{cookiecutter.github_repository}}/runtime.txt b/{{cookiecutter.github_repository}}/runtime.txt deleted file mode 100644 index f72c5111..00000000 --- a/{{cookiecutter.github_repository}}/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.9.0