diff --git a/.coveragerc b/.coveragerc index bc952e6657..8b31df7753 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,7 @@ +[coverage:run] +relative_files = True +source = src/ +branch = True + [report] omit=src/tests/*,src/vendor/*,/usr/lib/moulinette/yunohost/*,/usr/lib/python3/dist-packages/yunohost/tests/*,/usr/lib/python3/dist-packages/yunohost/vendor/* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 748940b337..b7b9254299 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,11 +1,10 @@ --- stages: + - lint - build - install - test - - lint - - doc - - translation + - bot default: tags: @@ -47,7 +46,7 @@ workflow: variables: GIT_CLONE_PATH: '$CI_BUILDS_DIR/$CI_COMMIT_SHA/$CI_JOB_ID' YNH_SOURCE: "https://github.com/yunohost" - YNH_DEBIAN: "bullseye" + YNH_DEBIAN: "bookworm" YNH_SKIP_DIAGNOSIS_DURING_UPGRADE: "true" include: diff --git a/.gitlab/ci/bot.gitlab-ci.yml b/.gitlab/ci/bot.gitlab-ci.yml new file mode 100644 index 0000000000..0443ee6888 --- /dev/null +++ b/.gitlab/ci/bot.gitlab-ci.yml @@ -0,0 +1,53 @@ +generate-helpers-doc: + stage: bot + image: "build-and-lint" + needs: [] + before_script: + - git config --global user.email "yunohost@yunohost.org" + - git config --global user.name "$GITHUB_USER" + script: + - cd doc + - python3 generate_helper_doc.py 2 + - python3 generate_helper_doc.py 2.1 + - python3 generate_resource_doc.py > resources.md + - python3 generate_configpanel_and_formoptions_doc.py > forms.md + - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo + - cp helpers.v2.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md + - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/12.helpers21/packaging_app_scripts_helpers_v21.md + - cp resources.md doc_repo/pages/06.contribute/10.packaging_apps/10.manifest/10.appresources/packaging_app_manifest_resources.md + - cp forms.md doc_repo/pages/06.contribute/15.dev/03.forms/forms.md + - cd doc_repo + # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ? + - hub checkout -b "${CI_COMMIT_REF_NAME}" + - hub commit -am "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}" + - hub pull-request -m "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}" -p # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd + artifacts: + paths: + - doc/helpers.md + - doc/resources.md + only: + - tags + +autofix-translated-strings: + stage: bot + image: "build-and-lint" + needs: [] + before_script: + - git config --global user.email "yunohost@yunohost.org" + - git config --global user.name "$GITHUB_USER" + - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo + - cd github_repo + script: + # create a local branch that will overwrite distant one + - git checkout -b "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}" --no-track + - python3 maintenance/missing_i18n_keys.py --fix + - python3 maintenance/autofix_locale_format.py + - '[ $(git diff --ignore-blank-lines --ignore-all-space --ignore-space-at-eol --ignore-cr-at-eol | wc -l) != 0 ] || exit 0' # stop if there is nothing to commit + - git commit -am "[CI] Reformat / remove stale translated strings" || true + - git push -f origin "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}":"ci-remove-stale-translated-strings-${CI_COMMIT_REF_NAME}" + - hub pull-request -m "[CI] Reformat / remove stale translated strings" -b Yunohost:$CI_COMMIT_REF_NAME -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd + only: + variables: + - $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH + changes: + - locales/* diff --git a/.gitlab/ci/build.gitlab-ci.yml b/.gitlab/ci/build.gitlab-ci.yml index 422d8f74a9..aedef5ac8c 100644 --- a/.gitlab/ci/build.gitlab-ci.yml +++ b/.gitlab/ci/build.gitlab-ci.yml @@ -1,9 +1,14 @@ .build-stage: stage: build + needs: + - job: actionsmap + - job: invalidcode311 image: "build-and-lint" variables: YNH_BUILD_DIR: "$GIT_CLONE_PATH/build" before_script: + - echo $PWD + - echo $CI_PROJECT_DIR - mkdir -p $YNH_BUILD_DIR artifacts: paths: @@ -31,7 +36,7 @@ build-yunohost: - mkdir -p $YNH_BUILD_DIR/$PACKAGE - cat archive.tar.gz | tar -xz -C $YNH_BUILD_DIR/$PACKAGE - rm archive.tar.gz - - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE + - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE || { apt-get update && DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE; } - *build_script build-ssowat: @@ -40,7 +45,7 @@ build-ssowat: PACKAGE: "ssowat" script: - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $YNH_DEBIAN $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1 - - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE + - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE || { apt-get update && DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE; } - *build_script build-moulinette: @@ -49,5 +54,5 @@ build-moulinette: PACKAGE: "moulinette" script: - git clone $YNH_SOURCE/$PACKAGE -b $CI_COMMIT_REF_NAME $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE -b $YNH_DEBIAN $YNH_BUILD_DIR/$PACKAGE --depth 1 || git clone $YNH_SOURCE/$PACKAGE $YNH_BUILD_DIR/$PACKAGE --depth 1 - - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE + - DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE || { apt-get update && DEBIAN_FRONTEND=noninteractive apt --assume-yes -o Dpkg::Options::="--force-confold" build-dep $YNH_BUILD_DIR/$PACKAGE; } - *build_script diff --git a/.gitlab/ci/doc.gitlab-ci.yml b/.gitlab/ci/doc.gitlab-ci.yml deleted file mode 100644 index 1791901661..0000000000 --- a/.gitlab/ci/doc.gitlab-ci.yml +++ /dev/null @@ -1,31 +0,0 @@ -######################################## -# DOC -######################################## - -generate-helpers-doc: - stage: doc - image: "build-and-lint" - needs: [] - before_script: - - git config --global user.email "yunohost@yunohost.org" - - git config --global user.name "$GITHUB_USER" - script: - - cd doc - - python3 generate_helper_doc.py 2 - - python3 generate_helper_doc.py 2.1 - - python3 generate_resource_doc.py > resources.md - - hub clone https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/doc.git doc_repo - - cp helpers.v2.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/10.helpers/packaging_app_scripts_helpers.md - - cp helpers.v2.1.md doc_repo/pages/06.contribute/10.packaging_apps/20.scripts/12.helpers21/packaging_app_scripts_helpers_v21.md - - cp resources.md doc_repo/pages/06.contribute/10.packaging_apps/10.manifest/10.appresources/packaging_app_manifest_resources.md - - cd doc_repo - # replace ${CI_COMMIT_REF_NAME} with ${CI_COMMIT_TAG} ? - - hub checkout -b "${CI_COMMIT_REF_NAME}" - - hub commit -am "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}" - - hub pull-request -m "[CI] Update app helpers/resources for ${CI_COMMIT_REF_NAME}" -p # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd - artifacts: - paths: - - doc/helpers.md - - doc/resources.md - only: - - tags diff --git a/.gitlab/ci/lint.gitlab-ci.yml b/.gitlab/ci/lint.gitlab-ci.yml index ad1e46d078..6edaf2c98b 100644 --- a/.gitlab/ci/lint.gitlab-ci.yml +++ b/.gitlab/ci/lint.gitlab-ci.yml @@ -3,24 +3,39 @@ ######################################## # later we must fix lint and format-check jobs and remove "allow_failure" -lint39: +actionsmap: + stage: lint + image: "build-and-lint" + needs: [] + script: + - python3 -c 'import yaml; yaml.safe_load(open("share/actionsmap.yml"))' + - python3 -c 'import yaml; yaml.safe_load(open("share/actionsmap-portal.yml"))' + +lint311: stage: lint image: "build-and-lint" needs: [] allow_failure: true script: - - tox -e py39-lint + - tox -e py311-lint -invalidcode39: +invalidcode311: stage: lint image: "build-and-lint" needs: [] script: - - tox -e py39-invalidcode + - tox -e py311-invalidcode mypy: stage: lint image: "build-and-lint" needs: [] script: - - tox -e py39-mypy + - tox -e py311-mypy + +i18n-keys: + stage: lint + image: "build-and-lint" + needs: [] + script: + - python3 maintenance/missing_i18n_keys.py --check diff --git a/.gitlab/ci/test.gitlab-ci.yml b/.gitlab/ci/test.gitlab-ci.yml index 438098c14a..def3c2ab36 100644 --- a/.gitlab/ci/test.gitlab-ci.yml +++ b/.gitlab/ci/test.gitlab-ci.yml @@ -5,13 +5,11 @@ stage: test image: "core-tests" variables: - PYTEST_ADDOPTS: "--color=yes" + PYTEST_ADDOPTS: "--color=yes --cov=src" + COVERAGE_FILE: ".coverage_$CI_JOB_NAME" before_script: - *install_debs - cache: - paths: - - src/tests/apps - key: "$CI_JOB_STAGE-$CI_COMMIT_REF_SLUG" + - ln -s src yunohost needs: - job: build-yunohost artifacts: true @@ -20,42 +18,14 @@ - job: build-moulinette artifacts: true - job: upgrade + artifacts: + paths: + - ./.coverage_* ######################################## # TESTS ######################################## -full-tests: - stage: test - image: "before-install" - variables: - PYTEST_ADDOPTS: "--color=yes" - before_script: - - *install_debs - - pip install mock pip pyOpenSSL pytest pytest-cov pytest-mock pytest-sugar requests-mock "packaging<22" - - yunohost tools postinstall -d domain.tld -u syssa -F 'Syssa Mine' -p the_password --ignore-dyndns --force-diskspace - script: - - python3 -m pytest --cov=yunohost tests/ src/tests/ --junitxml=report.xml - needs: - - job: build-yunohost - artifacts: true - - job: build-ssowat - artifacts: true - - job: build-moulinette - artifacts: true - coverage: '/TOTAL.*\s+(\d+%)/' - artifacts: - reports: - junit: report.xml - -test-actionmap: - extends: .test-stage - script: - - python3 -m pytest tests/test_actionmap.py - only: - changes: - - share/actionsmap.yml - test-helpers2: extends: .test-stage script: @@ -72,129 +42,130 @@ test-domains: extends: .test-stage script: - python3 -m pytest src/tests/test_domains.py - only: - changes: - - src/domain.py test-dns: extends: .test-stage script: - python3 -m pytest src/tests/test_dns.py - only: - changes: - - src/dns.py - - src/utils/dns.py test-apps: extends: .test-stage script: - python3 -m pytest src/tests/test_apps.py - only: - changes: - - src/app.py test-appscatalog: extends: .test-stage script: - python3 -m pytest src/tests/test_app_catalog.py - only: - changes: - - src/app_calalog.py test-appurl: extends: .test-stage script: - python3 -m pytest src/tests/test_appurl.py - only: - changes: - - src/app.py test-questions: extends: .test-stage script: - python3 -m pytest src/tests/test_questions.py - only: - changes: - - src/utils/config.py test-app-config: extends: .test-stage script: - python3 -m pytest src/tests/test_app_config.py - only: - changes: - - src/app.py - - src/utils/config.py test-app-resources: extends: .test-stage script: - python3 -m pytest src/tests/test_app_resources.py - only: - changes: - - src/app.py - - src/utils/resources.py test-changeurl: extends: .test-stage script: - python3 -m pytest src/tests/test_changeurl.py - only: - changes: - - src/app.py test-backuprestore: extends: .test-stage script: - python3 -m pytest src/tests/test_backuprestore.py - only: - changes: - - src/backup.py test-permission: extends: .test-stage script: - python3 -m pytest src/tests/test_permission.py - only: - changes: - - src/permission.py test-settings: extends: .test-stage script: - python3 -m pytest src/tests/test_settings.py - only: - changes: - - src/settings.py test-user-group: extends: .test-stage script: - python3 -m pytest src/tests/test_user-group.py - only: - changes: - - src/user.py test-regenconf: extends: .test-stage script: - python3 -m pytest src/tests/test_regenconf.py - only: - changes: - - src/regenconf.py test-service: extends: .test-stage script: - python3 -m pytest src/tests/test_service.py - only: - changes: - - src/service.py test-ldapauth: extends: .test-stage script: - python3 -m pytest src/tests/test_ldapauth.py - only: - changes: - - src/authenticators/*.py + +test-sso-and-portalapi: + extends: .test-stage + script: + - python3 -m pytest src/tests/test_sso_and_portalapi.py + +######################################## +# COVERAGE REPORT +######################################## + +coverage: + stage: test + image: "core-tests" + needs: + # Yeah ... gotta list all of those individually ... https://gitlab.com/gitlab-org/gitlab/-/issues/332326 + - job: test-domains + artifacts: true + - job: test-dns + artifacts: true + - job: test-apps + artifacts: true + - job: test-appscatalog + artifacts: true + - job: test-appurl + artifacts: true + - job: test-questions + artifacts: true + - job: test-app-config + artifacts: true + - job: test-app-resources + artifacts: true + - job: test-changeurl + artifacts: true + - job: test-backuprestore + artifacts: true + - job: test-permission + artifacts: true + - job: test-settings + artifacts: true + - job: test-user-group + artifacts: true + - job: test-regenconf + artifacts: true + - job: test-service + artifacts: true + - job: test-ldapauth + artifacts: true + - job: test-sso-and-portalapi + artifacts: true + script: + - coverage combine ./.coverage_* + - coverage report diff --git a/.gitlab/ci/translation.gitlab-ci.yml b/.gitlab/ci/translation.gitlab-ci.yml deleted file mode 100644 index bc9e1308d0..0000000000 --- a/.gitlab/ci/translation.gitlab-ci.yml +++ /dev/null @@ -1,36 +0,0 @@ -######################################## -# TRANSLATION -######################################## -test-i18n-keys: - stage: translation - script: - - python3 maintenance/missing_i18n_keys.py --check - only: - changes: - - locales/en.json - - src/*.py - - src/diagnosers/*.py - -autofix-translated-strings: - stage: translation - image: "build-and-lint" - needs: [] - before_script: - - git config --global user.email "yunohost@yunohost.org" - - git config --global user.name "$GITHUB_USER" - - hub clone --branch ${CI_COMMIT_REF_NAME} "https://$GITHUB_TOKEN:x-oauth-basic@github.com/YunoHost/yunohost.git" github_repo - - cd github_repo - script: - # create a local branch that will overwrite distant one - - git checkout -b "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}" --no-track - - python3 maintenance/missing_i18n_keys.py --fix - - python3 maintenance/autofix_locale_format.py - - '[ $(git diff --ignore-blank-lines --ignore-all-space --ignore-space-at-eol --ignore-cr-at-eol | wc -l) != 0 ] || exit 0' # stop if there is nothing to commit - - git commit -am "[CI] Reformat / remove stale translated strings" || true - - git push -f origin "ci-autofix-translated-strings-${CI_COMMIT_REF_NAME}":"ci-remove-stale-translated-strings-${CI_COMMIT_REF_NAME}" - - hub pull-request -m "[CI] Reformat / remove stale translated strings" -b Yunohost:$CI_COMMIT_REF_NAME -p || true # GITHUB_USER and GITHUB_TOKEN registered here https://gitlab.com/yunohost/yunohost/-/settings/ci_cd - only: - variables: - - $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH - changes: - - locales/* diff --git a/bin/yunohost-portal-api b/bin/yunohost-portal-api new file mode 100755 index 0000000000..66751e66fe --- /dev/null +++ b/bin/yunohost-portal-api @@ -0,0 +1,53 @@ +#! /usr/bin/python3 +# -*- coding: utf-8 -*- + +import argparse +import yunohost + +# Default server configuration +DEFAULT_HOST = "localhost" +DEFAULT_PORT = 6788 + + +def _parse_api_args(): + """Parse main arguments for the api""" + parser = argparse.ArgumentParser( + add_help=False, + description="Run the YunoHost API to manage your server.", + ) + srv_group = parser.add_argument_group("server configuration") + srv_group.add_argument( + "-h", + "--host", + action="store", + default=DEFAULT_HOST, + help="Host to listen on (default: %s)" % DEFAULT_HOST, + ) + srv_group.add_argument( + "-p", + "--port", + action="store", + default=DEFAULT_PORT, + type=int, + help="Port to listen on (default: %d)" % DEFAULT_PORT, + ) + glob_group = parser.add_argument_group("global arguments") + glob_group.add_argument( + "--debug", + action="store_true", + default=False, + help="Set log level to DEBUG", + ) + glob_group.add_argument( + "--help", + action="help", + help="Show this help message and exit", + ) + + return parser.parse_args() + + +if __name__ == "__main__": + opts = _parse_api_args() + # Run the server + yunohost.portalapi(debug=opts.debug, host=opts.host, port=opts.port) diff --git a/bin/yunomdns b/bin/yunomdns index 1bdcf88caf..da9233045f 100755 --- a/bin/yunomdns +++ b/bin/yunomdns @@ -132,12 +132,8 @@ def main() -> bool: ) continue - # Only broadcast IPv4 because IPv6 is buggy ... because we ain't using python3-ifaddr >= 0.1.7 - # Buster only ships 0.1.6 - # Bullseye ships 0.1.7 - # To be re-enabled once we're on bullseye... - # ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"] - ips: List[str] = interfaces[interface]["ipv4"] + # Broadcast IPv4 and IPv6 + ips: List[str] = interfaces[interface]["ipv4"] + interfaces[interface]["ipv6"] # If at least one IP is listed if not ips: diff --git a/conf/dnsmasq/domain.tpl b/conf/dnsmasq/domain.tpl index 50b946176c..e72511f484 100644 --- a/conf/dnsmasq/domain.tpl +++ b/conf/dnsmasq/domain.tpl @@ -1,13 +1,9 @@ {% set interfaces_list = interfaces.split(' ') %} {% for interface in interfaces_list %} interface-name={{ domain }},{{ interface }} -interface-name=xmpp-upload.{{ domain }},{{ interface }} {% endfor %} {% if ipv6 %} host-record={{ domain }},{{ ipv6 }} -host-record=xmpp-upload.{{ domain }},{{ ipv6 }} {% endif %} txt-record={{ domain }},"v=spf1 mx a -all" mx-host={{ domain }},{{ domain }},5 -srv-host=_xmpp-client._tcp.{{ domain }},{{ domain }},5222,0,5 -srv-host=_xmpp-server._tcp.{{ domain }},{{ domain }},5269,0,5 diff --git a/conf/dovecot/dovecot.conf b/conf/dovecot/dovecot.conf index c014a4d237..810fcd4d28 100644 --- a/conf/dovecot/dovecot.conf +++ b/conf/dovecot/dovecot.conf @@ -13,9 +13,8 @@ protocols = imap sieve {% if pop3_enabled == "True" %}pop3{% endif %} mail_plugins = $mail_plugins quota notify push_notification ############################################################################### - -# generated 2020-08-18, Mozilla Guideline v5.6, Dovecot 2.3.4, OpenSSL 1.1.1d, intermediate configuration -# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.4&config=intermediate&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, Dovecot 2.3.19, OpenSSL 3.0.9, intermediate configuration +# https://ssl-config.mozilla.org/#server=dovecot&version=2.3.19&config=intermediate&openssl=3.0.9&guideline=5.7 ssl = required @@ -32,7 +31,7 @@ ssl_dh = , # and return true if the IP is to be ignored. False otherwise. @@ -56,15 +98,18 @@ ignoreip = 127.0.0.1/8 ignorecommand = # "bantime" is the number of seconds that a host is banned. -bantime = 600 +bantime = 10m # A host is banned if it has generated "maxretry" during the last "findtime" # seconds. -findtime = 600 +findtime = 10m # "maxretry" is the number of failures before a host get banned. maxretry = 10 +# "maxmatches" is the number of matches stored in ticket (resolvable via tag in actions). +maxmatches = %(maxretry)s + # "backend" specifies the backend used to get files modification. # Available options are "pyinotify", "gamin", "polling", "systemd" and "auto". # This option can be overridden in each jail as well. @@ -113,10 +158,13 @@ logencoding = auto enabled = false +# "mode" defines the mode of the filter (see corresponding filter implementation for more info). +mode = normal + # "filter" defines the filter to use by the jail. # By default jails have names matching their filter name # -filter = %(__name__)s +filter = %(__name__)s[mode=%(mode)s] # @@ -140,7 +188,7 @@ mta = sendmail # Default protocol protocol = tcp -# Specify chain where jumps would need to be added in iptables-* actions +# Specify chain where jumps would need to be added in ban-actions expecting parameter chain chain = INPUT # Ports to be banned @@ -161,51 +209,53 @@ banaction = iptables-multiport banaction_allports = iptables-allports # The simplest action to take: ban only -action_ = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] +action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report to the destemail. -action_mw = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] +action_mw = %(action_)s + %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"] # ban & send an e-mail with whois report and relevant log lines # to the destemail. -action_mwl = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] +action_mwl = %(action_)s + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # See the IMPORTANT note in action.d/xarf-login-attack for when to use this action # # ban & send a xarf e-mail to abuse contact of IP address and include relevant log lines # to the destemail. -action_xarf = %(banaction)s[name=%(__name__)s, bantime="%(bantime)s", port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"] - xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath=%(logpath)s, port="%(port)s"] +action_xarf = %(action_)s + xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"] + +# ban & send a notification to one or more of the 50+ services supported by Apprise. +# See https://github.com/caronc/apprise/wiki for details on what is supported. +# +# You may optionally over-ride the default configuration line (containing the Apprise URLs) +# by using 'apprise[config="/alternate/path/to/apprise.cfg"]' otherwise +# /etc/fail2ban/apprise.conf is sourced for your supported notification configuration. +# action = %(action_)s +# apprise # ban IP on CloudFlare & send an e-mail with whois report and relevant log lines # to the destemail. action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"] - %(mta)s-whois-lines[name=%(__name__)s, sender="%(sender)s", dest="%(destemail)s", logpath=%(logpath)s, chain="%(chain)s"] + %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"] # Report block via blocklist.de fail2ban reporting service API # -# See the IMPORTANT note in action.d/blocklist_de.conf for when to -# use this action. Create a file jail.d/blocklist_de.local containing -# [Init] -# blocklist_de_apikey = {api key from registration] +# See the IMPORTANT note in action.d/blocklist_de.conf for when to use this action. +# Specify expected parameters in file action.d/blocklist_de.local or if the interpolation +# `action_blocklist_de` used for the action, set value of `blocklist_de_apikey` +# in your `jail.local` globally (section [DEFAULT]) or per specific jail section (resp. in +# corresponding jail.d/my-jail.local file). # -action_blocklist_de = blocklist_de[email="%(sender)s", service=%(filter)s, apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] +action_blocklist_de = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"] -# Report ban via badips.com, and use as blacklist -# -# See BadIPsAction docstring in config/action.d/badips.py for -# documentation for this action. +# Report ban via abuseipdb.com. # -# NOTE: This action relies on banaction being present on start and therefore -# should be last action defined for a jail. +# See action.d/abuseipdb.conf for usage example and details. # -action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"] -# -# Report ban via badips.com (uses action.d/badips.conf for reporting only) -# -action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"] +action_abuseipdb = abuseipdb # Choose default action. To change, just override value of 'action' with the # interpolation to the chosen action shortcut (e.g. action_mw, action_mwl, etc) in jail.local @@ -223,15 +273,10 @@ action = %(action_)s [sshd] -port = ssh -logpath = %(sshd_log)s -backend = %(sshd_backend)s - - -[sshd-ddos] -# This jail corresponds to the standard configuration in Fail2ban. -# The mail-whois action send a notification e-mail with a whois request -# in the body. +# To use more aggressive sshd modes set filter parameter "mode" in jail.local: +# normal (default), ddos, extra or aggressive (combines all). +# See "tests/files/logs/sshd" or "filter.d/sshd.conf" for usage example and details. +#mode = normal port = ssh logpath = %(sshd_log)s backend = %(sshd_backend)s @@ -265,7 +310,7 @@ logpath = %(apache_error_log)s # for email addresses. The mail outputs are buffered. port = http,https logpath = %(apache_access_log)s -bantime = 172800 +bantime = 48h maxretry = 1 @@ -301,7 +346,7 @@ maxretry = 2 port = http,https logpath = %(apache_access_log)s maxretry = 1 -ignorecommand = %(ignorecommands_dir)s/apache-fakegooglebot +ignorecommand = %(fail2ban_confpath)s/filter.d/ignorecommands/apache-fakegooglebot [apache-modsecurity] @@ -321,12 +366,15 @@ maxretry = 1 [openhab-auth] filter = openhab -action = iptables-allports[name=NoAuthFailures] +banaction = %(banaction_allports)s logpath = /opt/openhab/logs/request.log +# To use more aggressive http-auth modes set filter parameter "mode" in jail.local: +# normal (default), aggressive (combines all), auth or fallback +# See "tests/files/logs/nginx-http-auth" or "filter.d/nginx-http-auth.conf" for usage example and details. [nginx-http-auth] - +# mode = normal port = http,https logpath = %(nginx_error_log)s @@ -342,8 +390,10 @@ logpath = %(nginx_error_log)s port = http,https logpath = %(nginx_error_log)s -maxretry = 2 +[nginx-bad-request] +port = http,https +logpath = %(nginx_access_log)s # Ban attackers that try to use PHP's URL-fopen() functionality # through GET/POST variables. - Experimental, with more than a year @@ -377,6 +427,8 @@ logpath = %(lighttpd_error_log)s port = http,https logpath = %(roundcube_errors_log)s +# Use following line in your jail.local if roundcube logs to journal. +#backend = %(syslog_backend)s [openwebmail] @@ -426,11 +478,13 @@ backend = %(syslog_backend)s port = http,https logpath = /var/log/tomcat*/catalina.out +#logpath = /var/log/guacamole.log [monit] #Ban clients brute-forcing the monit gui login port = 2812 logpath = /var/log/monit + /var/log/monit.log [webmin-auth] @@ -513,27 +567,29 @@ logpath = %(vsftpd_log)s # ASSP SMTP Proxy Jail [assp] -port = smtp,submission +port = smtp,465,submission logpath = /root/path/to/assp/logs/maillog.txt [courier-smtp] -port = smtp,submission +port = smtp,465,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix] - -port = smtp,submission -logpath = %(postfix_log)s -backend = %(postfix_backend)s +# To use another modes set filter parameter "mode" in jail.local: +mode = more +port = smtp,465,submission +logpath = %(postfix_log)s +backend = %(postfix_backend)s [postfix-rbl] -port = smtp,submission +filter = postfix[mode=rbl] +port = smtp,465,submission logpath = %(postfix_log)s backend = %(postfix_backend)s maxretry = 1 @@ -541,14 +597,17 @@ maxretry = 1 [sendmail-auth] -port = submission,smtp +port = submission,465,smtp logpath = %(syslog_mail)s backend = %(syslog_backend)s [sendmail-reject] - -port = smtp,submission +# To use more aggressive modes set filter parameter "mode" in jail.local: +# normal (default), extra or aggressive +# See "tests/files/logs/sendmail-reject" or "filter.d/sendmail-reject.conf" for usage example and details. +#mode = normal +port = smtp,465,submission logpath = %(syslog_mail)s backend = %(syslog_backend)s @@ -556,7 +615,7 @@ backend = %(syslog_backend)s [qmail-rbl] filter = qmail -port = smtp,submission +port = smtp,465,submission logpath = /service/qmail/log/main/current @@ -564,14 +623,14 @@ logpath = /service/qmail/log/main/current # but can be set by syslog_facility in the dovecot configuration. [dovecot] -port = pop3,pop3s,imap,imaps,submission,sieve +port = pop3,pop3s,imap,imaps,submission,465,sieve logpath = %(dovecot_log)s backend = %(dovecot_backend)s [sieve] -port = smtp,submission +port = smtp,465,submission logpath = %(dovecot_log)s backend = %(dovecot_backend)s @@ -583,20 +642,21 @@ logpath = %(solidpop3d_log)s [exim] - -port = smtp,submission +# see filter.d/exim.conf for further modes supported from filter: +#mode = normal +port = smtp,465,submission logpath = %(exim_main_log)s [exim-spam] -port = smtp,submission +port = smtp,465,submission logpath = %(exim_main_log)s [kerio] -port = imap,smtp,imaps +port = imap,smtp,imaps,465 logpath = /opt/kerio/mailserver/store/logs/security.log @@ -607,14 +667,15 @@ logpath = /opt/kerio/mailserver/store/logs/security.log [courier-auth] -port = smtp,submission,imaps,pop3,pop3s +port = smtp,465,submission,imap,imaps,pop3,pop3s logpath = %(syslog_mail)s backend = %(syslog_backend)s [postfix-sasl] -port = smtp,submission,imap,imaps,pop3,pop3s +filter = postfix[mode=auth] +port = smtp,465,submission,imap,imaps,pop3,pop3s # You might consider monitoring /var/log/mail.warn instead if you are # running postfix since it would provide the same log lines at the # "warn" level but overall at the smaller filesize. @@ -631,7 +692,7 @@ backend = %(syslog_backend)s [squirrelmail] -port = smtp,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks +port = smtp,465,submission,imap,imap2,imaps,pop3,pop3s,http,https,socks logpath = /var/lib/squirrelmail/prefs/squirrelmail_access_log @@ -684,8 +745,8 @@ logpath = /var/log/named/security.log [nsd] port = 53 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/nsd.log @@ -696,9 +757,8 @@ logpath = /var/log/nsd.log [asterisk] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/asterisk/messages maxretry = 10 @@ -706,16 +766,22 @@ maxretry = 10 [freeswitch] port = 5060,5061 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] - %(mta)s-whois[name=%(__name__)s, dest="%(destemail)s"] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/freeswitch.log maxretry = 10 +# enable adminlog; it will log to a file inside znc's directory by default. +[znc-adminlog] + +port = 6667 +logpath = /var/lib/znc/moddata/adminlog/znc.log + + # To log wrong MySQL access attempts add to /etc/my.cnf in [mysqld] or # equivalent section: -# log-warning = 2 +# log-warnings = 2 # # for syslog (daemon facility) # [mysqld_safe] @@ -731,6 +797,14 @@ logpath = %(mysql_log)s backend = %(mysql_backend)s +[mssql-auth] +# Default configuration for Microsoft SQL Server for Linux +# See the 'mssql-conf' manpage how to change logpath or port +logpath = /var/opt/mssql/log/errorlog +port = 1433 +filter = mssql-auth + + # Log wrong MongoDB auth (for details see filter 'filter.d/mongodb-auth.conf') [mongodb-auth] # change port when running with "--shardsvr" or "--configsvr" runtime operation @@ -749,8 +823,8 @@ logpath = /var/log/mongodb/mongodb.log logpath = /var/log/fail2ban.log banaction = %(banaction_allports)s -bantime = 604800 ; 1 week -findtime = 86400 ; 1 day +bantime = 1w +findtime = 1d # Generic filter for PAM. Has to be used with action which bans all @@ -786,11 +860,31 @@ logpath = /var/log/ejabberd/ejabberd.log [counter-strike] logpath = /opt/cstrike/logs/L[0-9]*.log -# Firewall: http://www.cstrike-planet.com/faq/6 tcpport = 27030,27031,27032,27033,27034,27035,27036,27037,27038,27039 udpport = 1200,27000,27001,27002,27003,27004,27005,27006,27007,27008,27009,27010,27011,27012,27013,27014,27015 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp", chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp", chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, port="%(tcpport)s", protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, port="%(udpport)s", protocol="udp"] + +[softethervpn] +port = 500,4500 +protocol = udp +logpath = /usr/local/vpnserver/security_log/*/sec.log + +[gitlab] +port = http,https +logpath = /var/log/gitlab/gitlab-rails/application.log + +[grafana] +port = http,https +logpath = /var/log/grafana/grafana.log + +[bitwarden] +port = http,https +logpath = /home/*/bwdata/logs/identity/Identity/log.txt + +[centreon] +port = http,https +logpath = /var/log/centreon/login.log # consider low maxretry and a long bantime # nobody except your own Nagios server should ever probe nrpe @@ -824,7 +918,9 @@ filter = apache-pass[knocking_url="%(knocking_url)s"] logpath = %(apache_access_log)s blocktype = RETURN returntype = DROP -bantime = 3600 +action = %(action_)s[blocktype=%(blocktype)s, returntype=%(returntype)s, + actionstart_on_demand=false, actionrepair_on_unban=true] +bantime = 1h maxretry = 1 findtime = 1 @@ -832,8 +928,8 @@ findtime = 1 [murmur] # AKA mumble-server port = 64738 -action = %(banaction)s[name=%(__name__)s-tcp, port="%(port)s", protocol=tcp, chain="%(chain)s", actname=%(banaction)s-tcp] - %(banaction)s[name=%(__name__)s-udp, port="%(port)s", protocol=udp, chain="%(chain)s", actname=%(banaction)s-udp] +action_ = %(default/action_)s[name=%(__name__)s-tcp, protocol="tcp"] + %(default/action_)s[name=%(__name__)s-udp, protocol="udp"] logpath = /var/log/mumble-server/mumble-server.log @@ -851,5 +947,34 @@ logpath = /var/log/haproxy.log [slapd] port = ldap,ldaps -filter = slapd logpath = /var/log/slapd.log + +[domino-smtp] +port = smtp,ssmtp +logpath = /home/domino01/data/IBM_TECHNICAL_SUPPORT/console.log + +[phpmyadmin-syslog] +port = http,https +logpath = %(syslog_authpriv)s +backend = %(syslog_backend)s + + +[zoneminder] +# Zoneminder HTTP/HTTPS web interface auth +# Logs auth failures to apache2 error log +port = http,https +logpath = %(apache_error_log)s + +[traefik-auth] +# to use 'traefik-auth' filter you have to configure your Traefik instance, +# see `filter.d/traefik-auth.conf` for details and service example. +port = http,https +logpath = /var/log/traefik/access.log + +[scanlogd] +logpath = %(syslog_local0)s +banaction = %(banaction_allports)s + +[monitorix] +port = 8080 +logpath = /var/log/monitorix-httpd diff --git a/conf/fail2ban/yunohost-jails.conf b/conf/fail2ban/yunohost-jails.conf index 911f9cd856..d04ea41fd1 100644 --- a/conf/fail2ban/yunohost-jails.conf +++ b/conf/fail2ban/yunohost-jails.conf @@ -31,3 +31,12 @@ protocol = tcp filter = yunohost logpath = /var/log/nginx/*error.log /var/log/nginx/*access.log + +[yunohost-portal] +enabled = true +port = http,https +protocol = tcp +filter = yunohost-portal +logpath = /var/log/nginx/*error.log + /var/log/nginx/*access.log +maxretry = 20 diff --git a/conf/fail2ban/yunohost-portal.conf b/conf/fail2ban/yunohost-portal.conf new file mode 100644 index 0000000000..c4a16570f7 --- /dev/null +++ b/conf/fail2ban/yunohost-portal.conf @@ -0,0 +1,3 @@ +[Definition] +failregex = ^ -.*\"POST /yunohost/portalapi/login HTTP/\d.\d\" 401 +ignoreregex = diff --git a/conf/fail2ban/yunohost.conf b/conf/fail2ban/yunohost.conf index 26d7327400..be20e231b2 100644 --- a/conf/fail2ban/yunohost.conf +++ b/conf/fail2ban/yunohost.conf @@ -1,24 +1,3 @@ -# Fail2Ban configuration file -# -# Author: Adrien Beudin -# -# $Revision: 2 $ -# - [Definition] - -# Option: failregex -# Notes.: regex to match the password failure messages in the logfile. The -# host must be matched by a group named "host". The tag "" can -# be used for standard IP/hostname matching and is only an alias for -# (?:::f{4,6}:)?(?P[\w\-.^_]+) -# Values: TEXT -# -failregex = helpers.lua:[0-9]+: authenticate\(\): Connection failed for: .*, client: - ^ -.*\"POST /yunohost/api/login HTTP/\d.\d\" 401 - -# Option: ignoreregex -# Notes.: regex to ignore. If this regex matches, the line is ignored. -# Values: TEXT -# +failregex = ^ -.*\"POST /yunohost/api/login HTTP/\d.\d\" 401 ignoreregex = diff --git a/conf/metronome/domain.tpl.cfg.lua b/conf/metronome/domain.tpl.cfg.lua deleted file mode 100644 index 7391479dcc..0000000000 --- a/conf/metronome/domain.tpl.cfg.lua +++ /dev/null @@ -1,75 +0,0 @@ -VirtualHost "{{ domain }}" - enable = true - ssl = { - key = "/etc/yunohost/certs/{{ domain }}/key.pem"; - certificate = "/etc/yunohost/certs/{{ domain }}/crt.pem"; - } - authentication = "ldap2" - ldap = { - hostname = "localhost", - user = { - basedn = "ou=users,dc=yunohost,dc=org", - filter = "(&(objectClass=posixAccount)(mail=*@{{ domain }})(permission=cn=xmpp.main,ou=permission,dc=yunohost,dc=org))", - usernamefield = "mail", - namefield = "cn", - }, - } - - -- Discovery items - disco_items = { - { "muc.{{ domain }}" }, - { "pubsub.{{ domain }}" }, - { "jabber.{{ domain }}" }, - { "vjud.{{ domain }}" }, - { "xmpp-upload.{{ domain }}" }, - }; - --- contact_info = { --- abuse = { "mailto:abuse@{{ domain }}", "xmpp:admin@{{ domain }}" }; --- admin = { "mailto:root@{{ domain }}", "xmpp:admin@{{ domain }}" }; --- }; - ------- Components ------ --- You can specify components to add hosts that provide special services, --- like multi-user conferences, and transports. - ----Set up a MUC (multi-user chat) room server -Component "muc.{{ domain }}" "muc" - name = "{{ domain }} Chatrooms" - - modules_enabled = { - "muc_limits"; - "muc_log"; - "muc_log_mam"; - "muc_log_http"; - "muc_vcard"; - } - - muc_event_rate = 0.5 - muc_burst_factor = 10 - room_default_config = { - logging = true, - persistent = true - }; - ----Set up a PubSub server -Component "pubsub.{{ domain }}" "pubsub" - name = "{{ domain }} Publish/Subscribe" - - unrestricted_node_creation = true -- Anyone can create a PubSub node (from any server) - ----Set up a HTTP Upload service -Component "xmpp-upload.{{ domain }}" "http_upload" - name = "{{ domain }} Sharing Service" - - http_file_path = "/var/xmpp-upload/{{ domain }}/upload" - http_external_url = "https://xmpp-upload.{{ domain }}:443" - http_file_base_path = "/upload" - http_file_size_limit = 6*1024*1024 - http_file_quota = 60*1024*1024 - http_upload_file_size_limit = 100 * 1024 * 1024 -- bytes - http_upload_quota = 10 * 1024 * 1024 * 1024 -- bytes - ----Set up a VJUD service -Component "vjud.{{ domain }}" "vjud" - vjud_disco_name = "{{ domain }} User Directory" diff --git a/conf/metronome/metronome.cfg.lua b/conf/metronome/metronome.cfg.lua deleted file mode 100644 index 0e2a62f01f..0000000000 --- a/conf/metronome/metronome.cfg.lua +++ /dev/null @@ -1,123 +0,0 @@ --- ** Metronome's config file example ** --- --- The format is exactly equal to Prosody's: --- --- Lists are written { "like", "this", "one" } --- Lists can also be of { 1, 2, 3 } numbers, etc. --- Either commas, or semi-colons; may be used as seperators. --- --- A table is a list of values, except each value has a name. An --- example would be: --- --- ssl = { key = "keyfile.key", certificate = "certificate.cert" } --- --- Tip: You can check that the syntax of this file is correct when you have finished --- by running: luac -p metronome.cfg.lua --- If there are any errors, it will let you know what and where they are, otherwise it --- will keep quiet. - --- Global settings go in this section - --- This is the list of modules Metronome will load on startup. --- It looks for mod_modulename.lua in the plugins folder, so make sure that exists too. - -modules_enabled = { - -- Generally required - "roster"; -- Allow users to have a roster. Recommended. - "saslauth"; -- Authentication for clients. Recommended if you want to log in. - "tls"; -- Add support for secure TLS on c2s/s2s connections - "disco"; -- Service discovery - - -- Not essential, but recommended - "private"; -- Private XML storage (for room bookmarks, etc.) - "vcard"; -- Allow users to set vCards - "pep"; -- Allows setting of mood, tune, etc. - "pubsub"; -- Publish-subscribe XEP-0060 - "posix"; -- POSIX functionality, sends server to background, enables syslog, etc. - "bidi"; -- Enables Bidirectional Server-to-Server Streams. - - -- Nice to have - "version"; -- Replies to server version requests - "uptime"; -- Report how long server has been running - "time"; -- Let others know the time here on this server - "ping"; -- Replies to XMPP pings with pongs - "register"; -- Allow users to register on this server using a client and change passwords - "stream_management"; -- Allows clients and servers to use Stream Management - "stanza_optimizations"; -- Allows clients to use Client State Indication and SIFT - "message_carbons"; -- Allows clients to enable carbon copies of messages - "mam"; -- Enable server-side message archives using Message Archive Management - "push"; -- Enable Push Notifications via PubSub using XEP-0357 - "lastactivity"; -- Enables clients to know the last presence status of an user - "adhoc_cm"; -- Allow to set client certificates to login through SASL External via adhoc - "admin_adhoc"; -- administration adhoc commands - "bookmarks"; -- XEP-0048 Bookmarks synchronization between PEP and Private Storage - "sec_labels"; -- Allows to use a simplified version XEP-0258 Security Labels and related ACDFs. - "privacy"; -- Add privacy lists and simple blocking command support - - -- Other specific functionality - --"admin_telnet"; -- administration console, telnet to port 5582 - --"admin_web"; -- administration web interface - "bosh"; -- Enable support for BOSH clients, aka "XMPP over Bidirectional Streams over Synchronous HTTP" - --"compression"; -- Allow clients to enable Stream Compression - --"spim_block"; -- Require authorization via OOB form for messages from non-contacts and block unsollicited messages - --"gate_guard"; -- Enable config-based blacklisting and hit-based auto-banning features - --"incidents_handling"; -- Enable Incidents Handling support (can be administered via adhoc commands) - --"server_presence"; -- Enables Server Buddies extension support - --"service_directory"; -- Enables Service Directories extension support - --"public_service"; -- Enables Server vCard support for public services in directories and advertises in features - --"register_api"; -- Provides secure API for both Out-Of-Band and In-Band registration for E-Mail verification - "websocket"; -- Enable support for WebSocket clients, aka "XMPP over WebSockets" -}; - --- Server PID -pidfile = "/var/run/metronome/metronome.pid" - --- HTTP server -http_ports = { 5290 } -http_interfaces = { "127.0.0.1", "::1" } - ---https_ports = { 5291 } ---https_interfaces = { "127.0.0.1", "::1" } - --- Enable IPv6 -use_ipv6 = true - --- BOSH configuration (mod_bosh) -consider_bosh_secure = true -cross_domain_bosh = true - --- WebSocket configuration (mod_websocket) -consider_websocket_secure = true -cross_domain_websocket = true - --- Disable account creation by default, for security -allow_registration = false - --- Use LDAP storage backend for all stores -storage = "ldap" - --- stanza optimization -csi_config_queue_all_muc_messages_but_mentions = false; - - --- Logging configuration -log = { - info = "/var/log/metronome/metronome.log"; -- Change 'info' to 'debug' for verbose logging - error = "/var/log/metronome/metronome.err"; - -- "*syslog"; -- Uncomment this for logging to syslog - -- "*console"; -- Log to the console, useful for debugging with daemonize=false -} - ------- Components ------ --- You can specify components to add hosts that provide special services, --- like multi-user conferences, and transports. - ----Set up a local BOSH service -Component "localhost" "http" - modules_enabled = { "bosh" } - ------------ Virtual hosts ----------- --- You need to add a VirtualHost entry for each domain you wish Metronome to serve. --- Settings under each VirtualHost entry apply *only* to that host. - -Include "conf.d/*.cfg.lua" diff --git a/conf/metronome/modules/ldap.lib.lua b/conf/metronome/modules/ldap.lib.lua deleted file mode 100644 index 6774e735fe..0000000000 --- a/conf/metronome/modules/ldap.lib.lua +++ /dev/null @@ -1,270 +0,0 @@ --- vim:sts=4 sw=4 - --- Prosody IM --- Copyright (C) 2008-2010 Matthew Wild --- Copyright (C) 2008-2010 Waqas Hussain --- Copyright (C) 2012 Rob Hoelz --- --- This project is MIT/X11 licensed. Please see the --- COPYING file in the source package for more information. --- - -local ldap; -local connection; -local params = module:get_option("ldap"); -local format = string.format; -local tconcat = table.concat; - -local _M = {}; - -local config_params = { - hostname = 'string', - user = { - basedn = 'string', - namefield = 'string', - filter = 'string', - usernamefield = 'string', - }, - groups = { - basedn = 'string', - namefield = 'string', - memberfield = 'string', - - _member = { - name = 'string', - admin = 'boolean?', - }, - }, - admin = { - _optional = true, - basedn = 'string', - namefield = 'string', - filter = 'string', - } -} - -local function run_validation(params, config, prefix) - prefix = prefix or ''; - - -- verify that every required member of config is present in params - for k, v in pairs(config) do - if type(k) == 'string' and k:sub(1, 1) ~= '_' then - local is_optional; - if type(v) == 'table' then - is_optional = v._optional; - else - is_optional = v:sub(-1) == '?'; - end - - if not is_optional and params[k] == nil then - return nil, prefix .. k .. ' is required'; - end - end - end - - for k, v in pairs(params) do - local expected_type = config[k]; - - local ok, err = true; - - if type(k) == 'string' then - -- verify that this key is present in config - if k:sub(1, 1) == '_' or expected_type == nil then - return nil, 'invalid parameter ' .. prefix .. k; - end - - -- type validation - if type(expected_type) == 'string' then - if expected_type:sub(-1) == '?' then - expected_type = expected_type:sub(1, -2); - end - - if type(v) ~= expected_type then - return nil, 'invalid type for parameter ' .. prefix .. k; - end - else -- it's a table (or had better be) - if type(v) ~= 'table' then - return nil, 'invalid type for parameter ' .. prefix .. k; - end - - -- recurse into child - ok, err = run_validation(v, expected_type, prefix .. k .. '.'); - end - else -- it's an integer (or had better be) - if not config._member then - return nil, 'invalid parameter ' .. prefix .. tostring(k); - end - ok, err = run_validation(v, config._member, prefix .. tostring(k) .. '.'); - end - - if not ok then - return ok, err; - end - end - - return true; -end - -local function validate_config() - if true then - return true; -- XXX for now - end - - -- this is almost too clever (I mean that in a bad - -- maintainability sort of way) - -- - -- basically this allows a free pass for a key in group members - -- equal to params.groups.namefield - setmetatable(config_params.groups._member, { - __index = function(_, k) - if k == params.groups.namefield then - return 'string'; - end - end - }); - - local ok, err = run_validation(params, config_params); - - setmetatable(config_params.groups._member, nil); - - if ok then - -- a little extra validation that doesn't fit into - -- my recursive checker - local group_namefield = params.groups.namefield; - for i, group in ipairs(params.groups) do - if not group[group_namefield] then - return nil, format('groups.%d.%s is required', i, group_namefield); - end - end - - -- fill in params.admin if you can - if not params.admin and params.groups then - local admingroup; - - for _, groupconfig in ipairs(params.groups) do - if groupconfig.admin then - admingroup = groupconfig; - break; - end - end - - if admingroup then - params.admin = { - basedn = params.groups.basedn, - namefield = params.groups.memberfield, - filter = group_namefield .. '=' .. admingroup[group_namefield], - }; - end - end - end - - return ok, err; -end - --- what to do if connection isn't available? -local function connect() - return ldap.open_simple(params.hostname, params.bind_dn, params.bind_password, params.use_tls); -end - --- this is abstracted so we can maintain persistent connections at a later time -function _M.getconnection() - return connect(); -end - -function _M.getparams() - return params; -end - --- XXX consider renaming this...it doesn't bind the current connection -function _M.bind(username, password) - local conn = _M.getconnection(); - local filter = format('%s=%s', params.user.usernamefield, username); - if params.user.usernamefield == 'mail' then - filter = format('mail=%s@*', username); - end - - if filter then - filter = _M.filter.combine_and(filter, params.user.filter); - end - - local who = _M.singlematch { - attrs = params.user.usernamefield, - base = params.user.basedn, - filter = filter, - }; - - if who then - who = who.dn; - module:log('debug', '_M.bind - who: %s', who); - else - module:log('debug', '_M.bind - no DN found for username = %s', username); - return nil, format('no DN found for username = %s', username); - end - - local conn, err = ldap.open_simple(params.hostname, who, password, params.use_tls); - - if conn then - conn:close(); - return true; - end - - return conn, err; -end - -function _M.singlematch(query) - local ld = _M.getconnection(); - - query.sizelimit = 1; - query.scope = 'subtree'; - - for dn, attribs in ld:search(query) do - attribs.dn = dn; - return attribs; - end -end - -_M.filter = {}; - -function _M.filter.combine_and(...) - local parts = { '(&' }; - - local arg = { ... }; - - for _, filter in ipairs(arg) do - if filter:sub(1, 1) ~= '(' and filter:sub(-1) ~= ')' then - filter = '(' .. filter .. ')' - end - parts[#parts + 1] = filter; - end - - parts[#parts + 1] = ')'; - - return tconcat(parts, ''); -end - -do - local ok, err; - - metronome.unlock_globals(); - ok, ldap = pcall(require, 'lualdap'); - metronome.lock_globals(); - if not ok then - module:log("error", "Failed to load the LuaLDAP library for accessing LDAP: %s", ldap); - module:log("error", "More information on install LuaLDAP can be found at http://www.keplerproject.org/lualdap"); - return; - end - - if not params then - module:log("error", "LDAP configuration required to use the LDAP storage module"); - return; - end - - ok, err = validate_config(); - - if not ok then - module:log("error", "LDAP configuration is invalid: %s", tostring(err)); - return; - end -end - -return _M; diff --git a/conf/metronome/modules/mod_auth_ldap2.lua b/conf/metronome/modules/mod_auth_ldap2.lua deleted file mode 100644 index f127e43572..0000000000 --- a/conf/metronome/modules/mod_auth_ldap2.lua +++ /dev/null @@ -1,90 +0,0 @@ --- vim:sts=4 sw=4 - --- Metronome IM --- Copyright (C) 2008-2010 Matthew Wild --- Copyright (C) 2008-2010 Waqas Hussain --- Copyright (C) 2012 Rob Hoelz --- Copyright (C) 2015 YUNOHOST.ORG --- --- This project is MIT/X11 licensed. Please see the --- COPYING file in the source package for more information. --- --- https://github.com/YunoHost/yunohost-config-metronome/blob/unstable/lib/modules/mod_auth_ldap2.lua --- adapted to use common LDAP store on Metronome - -local ldap = module:require 'ldap'; -local new_sasl = require 'util.sasl'.new; -local jsplit = require 'util.jid'.split; - -local log = module._log - -if not ldap then - return; -end - -function new_default_provider(host) - local provider = { name = "ldap2" }; - log("debug", "initializing ldap2 authentication provider for host '%s'", host); - - function provider.test_password(username, password) - return ldap.bind(username, password); - end - - function provider.user_exists(username) - local params = ldap.getparams() - - local filter = ldap.filter.combine_and(params.user.filter, params.user.usernamefield .. '=' .. username); - if params.user.usernamefield == 'mail' then - filter = ldap.filter.combine_and(params.user.filter, 'mail=' .. username .. '@*'); - end - - return ldap.singlematch { - base = params.user.basedn, - filter = filter, - }; - end - - function provider.get_password(username) - return nil, "Passwords unavailable for LDAP."; - end - - function provider.set_password(username, password) - return nil, "Passwords unavailable for LDAP."; - end - - function provider.create_user(username, password) - return nil, "Account creation/modification not available with LDAP."; - end - - function provider.get_sasl_handler(session) - local testpass_authentication_profile = { - session = session, - plain_test = function(sasl, username, password, realm) - return provider.test_password(username, password), true; - end, - order = { "plain_test" }, - }; - return new_sasl(module.host, testpass_authentication_profile); - end - - function provider.is_admin(jid) - local admin_config = ldap.getparams().admin; - - if not admin_config then - return; - end - - local ld = ldap:getconnection(); - local username = jsplit(jid); - local filter = ldap.filter.combine_and(admin_config.filter, admin_config.namefield .. '=' .. username); - - return ldap.singlematch { - base = admin_config.basedn, - filter = filter, - }; - end - - return provider; -end - -module:add_item("auth-provider", new_default_provider(module.host)); diff --git a/conf/metronome/modules/mod_legacyauth.lua b/conf/metronome/modules/mod_legacyauth.lua deleted file mode 100644 index ae90f72d66..0000000000 --- a/conf/metronome/modules/mod_legacyauth.lua +++ /dev/null @@ -1,86 +0,0 @@ --- Prosody IM --- Copyright (C) 2008-2010 Matthew Wild --- Copyright (C) 2008-2010 Waqas Hussain --- --- This project is MIT/X11 licensed. Please see the --- COPYING file in the source package for more information. --- - - - -local st = require "util.stanza"; -local t_concat = table.concat; - -local secure_auth_only = module:get_option("c2s_require_encryption") - or module:get_option("require_encryption") - or not(module:get_option("allow_unencrypted_plain_auth")); - -local sessionmanager = require "core.sessionmanager"; -local usermanager = require "core.usermanager"; -local nodeprep = require "util.encodings".stringprep.nodeprep; -local resourceprep = require "util.encodings".stringprep.resourceprep; - -module:add_feature("jabber:iq:auth"); -module:hook("stream-features", function(event) - local origin, features = event.origin, event.features; - if secure_auth_only and not origin.secure then - -- Sorry, not offering to insecure streams! - return; - elseif not origin.username then - features:tag("auth", {xmlns='http://jabber.org/features/iq-auth'}):up(); - end -end); - -module:hook("stanza/iq/jabber:iq:auth:query", function(event) - local session, stanza = event.origin, event.stanza; - - if session.type ~= "c2s_unauthed" then - (session.sends2s or session.send)(st.error_reply(stanza, "cancel", "service-unavailable", "Legacy authentication is only allowed for unauthenticated client connections.")); - return true; - end - - if secure_auth_only and not session.secure then - session.send(st.error_reply(stanza, "modify", "not-acceptable", "Encryption (SSL or TLS) is required to connect to this server")); - return true; - end - - local username = stanza.tags[1]:child_with_name("username"); - local password = stanza.tags[1]:child_with_name("password"); - local resource = stanza.tags[1]:child_with_name("resource"); - if not (username and password and resource) then - local reply = st.reply(stanza); - session.send(reply:query("jabber:iq:auth") - :tag("username"):up() - :tag("password"):up() - :tag("resource"):up()); - else - username, password, resource = t_concat(username), t_concat(password), t_concat(resource); - username = nodeprep(username); - resource = resourceprep(resource) - if not (username and resource) then - session.send(st.error_reply(stanza, "modify", "bad-request")); - return true; - end - if usermanager.test_password(username, session.host, password) then - -- Authentication successful! - local success, err = sessionmanager.make_authenticated(session, username); - if success then - local err_type, err_msg; - success, err_type, err, err_msg = sessionmanager.bind_resource(session, resource); - if not success then - session.send(st.error_reply(stanza, err_type, err, err_msg)); - session.username, session.type = nil, "c2s_unauthed"; -- FIXME should this be placed in sessionmanager? - return true; - elseif resource ~= session.resource then -- server changed resource, not supported by legacy auth - session.send(st.error_reply(stanza, "cancel", "conflict", "The requested resource could not be assigned to this session.")); - session:close(); -- FIXME undo resource bind and auth instead of closing the session? - return true; - end - end - session.send(st.reply(stanza)); - else - session.send(st.error_reply(stanza, "auth", "not-authorized")); - end - end - return true; -end); diff --git a/conf/metronome/modules/mod_storage_ldap.lua b/conf/metronome/modules/mod_storage_ldap.lua deleted file mode 100644 index 6f1cac4cdb..0000000000 --- a/conf/metronome/modules/mod_storage_ldap.lua +++ /dev/null @@ -1,243 +0,0 @@ --- vim:sts=4 sw=4 - --- Metronome IM --- Copyright (C) 2008-2010 Matthew Wild --- Copyright (C) 2008-2010 Waqas Hussain --- Copyright (C) 2012 Rob Hoelz --- Copyright (C) 2015 YUNOHOST.ORG --- --- This project is MIT/X11 licensed. Please see the --- COPYING file in the source package for more information. - ----------------------------------------- --- Constants and such -- ----------------------------------------- - -local setmetatable = setmetatable; - -local get_config = require "core.configmanager".get; -local ldap = module:require 'ldap'; -local vcardlib = module:require 'vcard'; -local st = require 'util.stanza'; -local gettime = require 'socket'.gettime; - -local log = module._log - -if not ldap then - return; -end - -local CACHE_EXPIRY = 300; - ----------------------------------------- --- Utility Functions -- ----------------------------------------- - -local function ldap_record_to_vcard(record, format) - return vcardlib.create { - record = record, - format = format, - } -end - -local get_alias_for_user; - -do - local user_cache; - local last_fetch_time; - - local function populate_user_cache() - local user_c = get_config(module.host, 'ldap').user; - if not user_c then return; end - - local ld = ldap.getconnection(); - - local usernamefield = user_c.usernamefield; - local namefield = user_c.namefield; - - user_cache = {}; - - for _, attrs in ld:search { base = user_c.basedn, scope = 'onelevel', filter = user_c.filter } do - user_cache[attrs[usernamefield]] = attrs[namefield]; - end - last_fetch_time = gettime(); - end - - function get_alias_for_user(user) - if last_fetch_time and last_fetch_time + CACHE_EXPIRY < gettime() then - user_cache = nil; - end - if not user_cache then - populate_user_cache(); - end - return user_cache[user]; - end -end - ----------------------------------------- --- Base LDAP store class -- ----------------------------------------- - -local function ldap_store(config) - local self = {}; - local config = config; - - function self:get(username) - return nil, "Data getting is not available for this storage backend"; - end - - function self:set(username, data) - return nil, "Data setting is not available for this storage backend"; - end - - return self; -end - -local adapters = {}; - ----------------------------------------- --- Roster Storage Implementation -- ----------------------------------------- - -adapters.roster = function (config) - -- Validate configuration requirements - if not config.groups then return nil; end - - local self = ldap_store(config) - - function self:get(username) - local ld = ldap.getconnection(); - local contacts = {}; - - local memberfield = config.groups.memberfield; - local namefield = config.groups.namefield; - local filter = memberfield .. '=' .. tostring(username); - - local groups = {}; - for _, config in ipairs(config.groups) do - groups[ config[namefield] ] = config.name; - end - - log("debug", "Found %d group(s) for user %s", select('#', groups), username) - - -- XXX this kind of relies on the way we do groups at INOC - for _, attrs in ld:search { base = config.groups.basedn, scope = 'onelevel', filter = filter } do - if groups[ attrs[namefield] ] then - local members = attrs[memberfield]; - - for _, user in ipairs(members) do - if user ~= username then - local jid = user .. '@' .. module.host; - local record = contacts[jid]; - - if not record then - record = { - subscription = 'both', - groups = {}, - name = get_alias_for_user(user), - }; - contacts[jid] = record; - end - - record.groups[ groups[ attrs[namefield] ] ] = true; - end - end - end - end - - return contacts; - end - - function self:set(username, data) - log("warn", "Setting data in Roster LDAP storage is not supported yet") - return nil, "not supported"; - end - - return self; -end - ----------------------------------------- --- vCard Storage Implementation -- ----------------------------------------- - -adapters.vcard = function (config) - -- Validate configuration requirements - if not config.vcard_format or not config.user then return nil; end - - local self = ldap_store(config) - - function self:get(username) - local ld = ldap.getconnection(); - local filter = config.user.usernamefield .. '=' .. tostring(username); - - log("debug", "Retrieving vCard for user '%s'", username); - - local match = ldap.singlematch { - base = config.user.basedn, - filter = filter, - }; - if match then - match.jid = username .. '@' .. module.host - return st.preserialize(ldap_record_to_vcard(match, config.vcard_format)); - else - return nil, "username not found"; - end - end - - function self:set(username, data) - log("warn", "Setting data in vCard LDAP storage is not supported yet") - return nil, "not supported"; - end - - return self; -end - ----------------------------------------- --- Driver Definition -- ----------------------------------------- - -cache = {}; - -local driver = { name = "ldap" }; - -function driver:open(store) - log("debug", "Opening ldap storage backend for host '%s' and store '%s'", module.host, store); - - if not cache[module.host] then - log("debug", "Caching adapters for the host '%s'", module.host); - - local ad_config = get_config(module.host, "ldap"); - local ad_cache = {}; - for k, v in pairs(adapters) do - ad_cache[k] = v(ad_config); - end - - cache[module.host] = ad_cache; - end - - local adapter = cache[module.host][store]; - - if not adapter then - log("info", "Unavailable adapter for store '%s'", store); - return nil, "unsupported-store"; - end - return adapter; -end - -function driver:stores(username, type, pattern) - return nil, "not implemented"; -end - -function driver:store_exists(username, type) - return nil, "not implemented"; -end - -function driver:purge(username) - return nil, "not implemented"; -end - -function driver:nodes(type) - return nil, "not implemented"; -end - -module:add_item("data-driver", driver); diff --git a/conf/metronome/modules/vcard.lib.lua b/conf/metronome/modules/vcard.lib.lua deleted file mode 100644 index de2f789237..0000000000 --- a/conf/metronome/modules/vcard.lib.lua +++ /dev/null @@ -1,162 +0,0 @@ --- vim:sts=4 sw=4 - --- Prosody IM --- Copyright (C) 2008-2010 Matthew Wild --- Copyright (C) 2008-2010 Waqas Hussain --- Copyright (C) 2012 Rob Hoelz --- --- This project is MIT/X11 licensed. Please see the --- COPYING file in the source package for more information. --- - -local st = require 'util.stanza'; - -local VCARD_NS = 'vcard-temp'; - -local builder_methods = {}; - -local base64_encode = require('util.encodings').base64.encode; - -function builder_methods:addvalue(key, value) - self.vcard:tag(key):text(value):up(); -end - -function builder_methods:addphotofield(tagname, format_section) - local record = self.record; - local format = self.format; - local vcard = self.vcard; - local config = format[format_section]; - - if not config then - return; - end - - if config.extval then - if record[config.extval] then - local tag = vcard:tag(tagname); - tag:tag('EXTVAL'):text(record[config.extval]):up(); - end - elseif config.type and config.binval then - if record[config.binval] then - local tag = vcard:tag(tagname); - tag:tag('TYPE'):text(config.type):up(); - tag:tag('BINVAL'):text(base64_encode(record[config.binval])):up(); - end - else - module:log('error', 'You have an invalid %s config section', tagname); - return; - end - - vcard:up(); -end - -function builder_methods:addregularfield(tagname, format_section) - local record = self.record; - local format = self.format; - local vcard = self.vcard; - - if not format[format_section] then - return; - end - - local tag = vcard:tag(tagname); - - for k, v in pairs(format[format_section]) do - tag:tag(string.upper(k)):text(record[v]):up(); - end - - vcard:up(); -end - -function builder_methods:addmultisectionedfield(tagname, format_section) - local record = self.record; - local format = self.format; - local vcard = self.vcard; - - if not format[format_section] then - return; - end - - for k, v in pairs(format[format_section]) do - local tag = vcard:tag(tagname); - - if type(k) == 'string' then - tag:tag(string.upper(k)):up(); - end - - for k2, v2 in pairs(v) do - if type(v2) == 'boolean' then - tag:tag(string.upper(k2)):up(); - else - tag:tag(string.upper(k2)):text(record[v2]):up(); - end - end - - vcard:up(); - end -end - -function builder_methods:build() - local record = self.record; - local format = self.format; - - self:addvalue( 'VERSION', '2.0'); - self:addvalue( 'FN', record[format.displayname]); - self:addregularfield( 'N', 'name'); - self:addvalue( 'NICKNAME', record[format.nickname]); - self:addphotofield( 'PHOTO', 'photo'); - self:addvalue( 'BDAY', record[format.birthday]); - self:addmultisectionedfield('ADR', 'address'); - self:addvalue( 'LABEL', nil); -- we don't support LABEL...yet. - self:addmultisectionedfield('TEL', 'telephone'); - self:addmultisectionedfield('EMAIL', 'email'); - self:addvalue( 'JABBERID', record.jid); - self:addvalue( 'MAILER', record[format.mailer]); - self:addvalue( 'TZ', record[format.timezone]); - self:addregularfield( 'GEO', 'geo'); - self:addvalue( 'TITLE', record[format.title]); - self:addvalue( 'ROLE', record[format.role]); - self:addphotofield( 'LOGO', 'logo'); - self:addvalue( 'AGENT', nil); -- we don't support AGENT...yet. - self:addregularfield( 'ORG', 'org'); - self:addvalue( 'CATEGORIES', nil); -- we don't support CATEGORIES...yet. - self:addvalue( 'NOTE', record[format.note]); - self:addvalue( 'PRODID', nil); -- we don't support PRODID...yet. - self:addvalue( 'REV', record[format.rev]); - self:addvalue( 'SORT-STRING', record[format.sortstring]); - self:addregularfield( 'SOUND', 'sound'); - self:addvalue( 'UID', record[format.uid]); - self:addvalue( 'URL', record[format.url]); - self:addvalue( 'CLASS', nil); -- we don't support CLASS...yet. - self:addregularfield( 'KEY', 'key'); - self:addvalue( 'DESC', record[format.description]); - - return self.vcard; -end - -local function new_builder(params) - local vcard_tag = st.stanza('vCard', { xmlns = VCARD_NS }); - - local object = { - vcard = vcard_tag, - __index = builder_methods, - }; - - for k, v in pairs(params) do - object[k] = v; - end - - setmetatable(object, object); - - return object; -end - -local _M = {}; - -function _M.create(params) - local builder = new_builder(params); - - return builder:build(); -end - -return _M; diff --git a/conf/nginx/plain/acme-challenge.conf.inc b/conf/nginx/acme-challenge.conf.inc similarity index 100% rename from conf/nginx/plain/acme-challenge.conf.inc rename to conf/nginx/acme-challenge.conf.inc diff --git a/conf/nginx/plain/global.conf b/conf/nginx/global.conf similarity index 100% rename from conf/nginx/plain/global.conf rename to conf/nginx/global.conf diff --git a/conf/nginx/plain/yunohost_panel.conf.inc b/conf/nginx/plain/yunohost_panel.conf.inc deleted file mode 100644 index 16a6e6b29e..0000000000 --- a/conf/nginx/plain/yunohost_panel.conf.inc +++ /dev/null @@ -1,8 +0,0 @@ -# Insert YunoHost button + portal overlay -sub_filter ''; -sub_filter_once on; -# Apply to other mime types than text/html -sub_filter_types application/xhtml+xml; -# Prevent YunoHost panel files from being blocked by specific app rules -location ~ (ynh_portal.js|ynh_overlay.css|ynh_userinfo.json|ynhtheme/custom_portal.js|ynhtheme/custom_overlay.css) { -} diff --git a/conf/nginx/plain/yunohost_sso.conf.inc b/conf/nginx/plain/yunohost_sso.conf.inc deleted file mode 100644 index 308e5a9a4b..0000000000 --- a/conf/nginx/plain/yunohost_sso.conf.inc +++ /dev/null @@ -1,7 +0,0 @@ -# Avoid the nginx path/alias traversal weakness ( #1037 ) -rewrite ^/yunohost/sso$ /yunohost/sso/ permanent; - -location /yunohost/sso/ { - # This is an empty location, only meant to avoid other locations - # from matching /yunohost/sso, such that it's correctly handled by ssowat -} diff --git a/conf/nginx/security.conf.inc b/conf/nginx/security.conf.inc index 44d7f86b45..c4b4d723f0 100644 --- a/conf/nginx/security.conf.inc +++ b/conf/nginx/security.conf.inc @@ -3,16 +3,16 @@ ssl_session_cache shared:SSL:50m; # about 200000 sessions ssl_session_tickets off; {% if compatibility == "modern" %} -# generated 2020-08-14, Mozilla Guideline v5.6, nginx 1.14.2, OpenSSL 1.1.1d, modern configuration -# https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=modern&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, nginx 1.22.1, OpenSSL 3.0.9, modern configuration +# https://ssl-config.mozilla.org/#server=nginx&version=1.22.1&config=modern&openssl=3.0.9&guideline=5.7 ssl_protocols TLSv1.3; ssl_prefer_server_ciphers off; {% else %} # Ciphers with intermediate compatibility -# generated 2020-08-14, Mozilla Guideline v5.6, nginx 1.14.2, OpenSSL 1.1.1d, intermediate configuration -# https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, nginx 1.22.1, OpenSSL 3.0.9, intermediate configuration +# https://ssl-config.mozilla.org/#server=nginx&version=1.22.1&config=intermediate&openssl=3.0.9&guideline=5.7 ssl_protocols TLSv1.2 TLSv1.3; -ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; +ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305; ssl_prefer_server_ciphers off; # Pre-defined FFDHE group (RFC 7919) diff --git a/conf/nginx/server.tpl.conf b/conf/nginx/server.tpl.conf index ac05b38c80..a5216cf318 100644 --- a/conf/nginx/server.tpl.conf +++ b/conf/nginx/server.tpl.conf @@ -6,7 +6,7 @@ map $http_upgrade $connection_upgrade { server { listen 80; listen [::]:80; - server_name {{ domain }}{% if xmpp_enabled == "True" %} xmpp-upload.{{ domain }} muc.{{ domain }}{% endif %}; + server_name {{ domain }}; access_by_lua_file /usr/share/ssowat/access.lua; @@ -85,55 +85,3 @@ server { access_log /var/log/nginx/{{ domain }}-access.log; error_log /var/log/nginx/{{ domain }}-error.log; } - -{% if xmpp_enabled == "True" %} -# vhost dedicated to XMPP http_upload -server { - - {% if tls_passthrough_enabled != "True" %} - listen 443 ssl http2; - listen [::]:443 ssl http2; - {% else %} - listen 127.0.0.1:444 ssl http2; - # Prevent 301/302 rewrite/redirect from including the 444 port - port_in_redirect off; - {% endif %} - server_name xmpp-upload.{{ domain }}; - root /dev/null; - - location /upload/ { - alias /var/xmpp-upload/{{ domain }}/upload/; - # Pass all requests to metronome, except for GET and HEAD requests. - limit_except GET HEAD { - proxy_pass http://localhost:5290; - } - - include proxy_params; - add_header 'Access-Control-Allow-Origin' '*'; - add_header 'Access-Control-Allow-Methods' 'HEAD, GET, PUT, OPTIONS'; - add_header 'Access-Control-Allow-Headers' 'Authorization'; - add_header 'Access-Control-Allow-Credentials' 'true'; - client_max_body_size 105M; # Choose a value a bit higher than the max upload configured in XMPP server - } - - include /etc/nginx/conf.d/security.conf.inc; - - ssl_certificate /etc/yunohost/certs/{{ domain }}/crt.pem; - ssl_certificate_key /etc/yunohost/certs/{{ domain }}/key.pem; - - {% if domain_cert_ca != "selfsigned" %} - more_set_headers "Strict-Transport-Security : max-age=63072000; includeSubDomains; preload"; - {% endif %} - {% if domain_cert_ca == "letsencrypt" %} - # OCSP settings - ssl_stapling on; - ssl_stapling_verify on; - ssl_trusted_certificate /etc/yunohost/certs/{{ domain }}/crt.pem; - resolver 1.1.1.1 9.9.9.9 valid=300s; - resolver_timeout 5s; - {% endif %} - - access_log /var/log/nginx/xmpp-upload.{{ domain }}-access.log; - error_log /var/log/nginx/xmpp-upload.{{ domain }}-error.log; -} -{% endif %} diff --git a/conf/nginx/plain/ssowat.conf b/conf/nginx/ssowat.conf similarity index 100% rename from conf/nginx/plain/ssowat.conf rename to conf/nginx/ssowat.conf diff --git a/conf/nginx/yunohost_api.conf.inc b/conf/nginx/yunohost_api.conf.inc index f434dbe964..9cb4ff00da 100644 --- a/conf/nginx/yunohost_api.conf.inc +++ b/conf/nginx/yunohost_api.conf.inc @@ -23,3 +23,24 @@ location = /yunohost/api/error/502 { add_header Content-Type text/plain; internal; } + +location /yunohost/portalapi/ { + + proxy_read_timeout 5s; + proxy_pass http://127.0.0.1:6788/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + + # Custom 502 error page + error_page 502 /yunohost/portalapi/error/502; +} + + +# Yunohost admin output complete 502 error page, so use only plain text. +location = /yunohost/portalapi/error/502 { + return 502 '502 - Bad Gateway'; + add_header Content-Type text/plain; + internal; +} diff --git a/conf/nginx/plain/yunohost_http_errors.conf.inc b/conf/nginx/yunohost_http_errors.conf.inc similarity index 100% rename from conf/nginx/plain/yunohost_http_errors.conf.inc rename to conf/nginx/yunohost_http_errors.conf.inc diff --git a/conf/nginx/yunohost_sso.conf.inc b/conf/nginx/yunohost_sso.conf.inc new file mode 100644 index 0000000000..839b50c405 --- /dev/null +++ b/conf/nginx/yunohost_sso.conf.inc @@ -0,0 +1,28 @@ +# Avoid the nginx path/alias traversal weakness ( #1037 ) +rewrite ^/yunohost/sso$ /yunohost/sso/ permanent; + +location /yunohost/sso/ { + alias /usr/share/yunohost/portal/; + default_type text/html; + index index.html; + try_files $uri $uri/ /index.html; + + location = /yunohost/sso/index.html { + etag off; + expires off; + more_set_headers "Cache-Control: no-store, no-cache, must-revalidate"; + } + + location /yunohost/sso/applogos/ { + alias /usr/share/yunohost/applogos/; + } + + location = /yunohost/sso/customassets/custom.css { + alias /usr/share/yunohost/portal/customassets/$host.custom.css; + etag off; + expires off; + more_set_headers "Cache-Control: no-store, no-cache, must-revalidate"; + } + + more_set_headers "Content-Security-Policy: upgrade-insecure-requests; default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; object-src 'none'; img-src 'self' data:;"; +} diff --git a/conf/opendkim/opendkim.conf b/conf/opendkim/opendkim.conf new file mode 100644 index 0000000000..303e504b74 --- /dev/null +++ b/conf/opendkim/opendkim.conf @@ -0,0 +1,31 @@ +# General daemon config +Socket inet:8891@localhost +PidFile /run/opendkim/opendkim.pid +UserID opendkim +UMask 007 + +AutoRestart yes +AutoRestartCount 10 +AutoRestartRate 10/1h + +# Logging +Syslog yes +SyslogSuccess yes +LogWhy yes + +# Common signing and verification parameters. In Debian, the "From" header is +# oversigned, because it is often the identity key used by reputation systems +# and thus somewhat security sensitive. +Canonicalization relaxed/simple +Mode sv +OversignHeaders From +#On-BadSignature reject + +# Key / signing table +KeyTable file:/etc/dkim/keytable +SigningTable refile:/etc/dkim/signingtable + +# The trust anchor enables DNSSEC. In Debian, the trust anchor file is provided +# by the package dns-root-data. +TrustAnchorFile /usr/share/dns/root.key +#Nameservers 127.0.0.1 diff --git a/conf/postfix/main.cf b/conf/postfix/main.cf index bf26f89c61..b8280ddead 100644 --- a/conf/postfix/main.cf +++ b/conf/postfix/main.cf @@ -30,8 +30,8 @@ smtpd_tls_chain_files = tls_server_sni_maps = hash:/etc/postfix/sni {% if compatibility == "intermediate" %} -# generated 2020-08-18, Mozilla Guideline v5.6, Postfix 3.4.14, OpenSSL 1.1.1d, intermediate configuration -# https://ssl-config.mozilla.org/#server=postfix&version=3.4.14&config=intermediate&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, Postfix 3.7.5, OpenSSL 3.0.9, intermediate configuration +# https://ssl-config.mozilla.org/#server=postfix&version=3.7.5&config=intermediate&openssl=3.0.9&guideline=5.7 smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1 smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1 @@ -41,10 +41,10 @@ smtpd_tls_mandatory_ciphers = medium # not actually 1024 bits, this applies to all DHE >= 1024 bits smtpd_tls_dh1024_param_file = /usr/share/yunohost/ffdhe2048.pem -tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 +tls_medium_cipherlist = ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305 {% else %} -# generated 2020-08-18, Mozilla Guideline v5.6, Postfix 3.4.14, OpenSSL 1.1.1d, modern configuration -# https://ssl-config.mozilla.org/#server=postfix&version=3.4.14&config=modern&openssl=1.1.1d&guideline=5.6 +# generated 2023-06-13, Mozilla Guideline v5.7, Postfix 3.7.5, OpenSSL 3.0.9, modern configuration +# https://ssl-config.mozilla.org/#server=postfix&version=3.7.5&config=modern&openssl=3.0.9&guideline=5.7 smtpd_tls_mandatory_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 smtpd_tls_protocols = !SSLv2, !SSLv3, !TLSv1, !TLSv1.1, !TLSv1.2 @@ -182,9 +182,10 @@ smtp_header_checks = regexp:/etc/postfix/header_checks smtp_reply_filter = pcre:/etc/postfix/smtp_reply_filter # Rmilter -milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen} +milter_mail_macros = i {mail_addr} {client_addr} {client_name} {auth_authen} {auth_type} milter_protocol = 6 -smtpd_milters = inet:localhost:11332 +smtpd_milters = inet:localhost:8891 +non_smtpd_milters = inet:localhost:8891 # Skip email without checking if milter has died milter_default_action = accept diff --git a/conf/rspamd/dkim_signing.conf b/conf/rspamd/dkim_signing.conf deleted file mode 100644 index 26718e021a..0000000000 --- a/conf/rspamd/dkim_signing.conf +++ /dev/null @@ -1,16 +0,0 @@ -allow_envfrom_empty = true; -allow_hdrfrom_mismatch = false; -allow_hdrfrom_multiple = false; -allow_username_mismatch = true; - -auth_only = true; -path = "/etc/dkim/$domain.$selector.key"; -selector = "mail"; -sign_local = true; -symbol = "DKIM_SIGNED"; -try_fallback = true; -use_domain = "header"; -use_esld = false; -use_redis = false; -key_prefix = "DKIM_KEYS"; - diff --git a/conf/rspamd/metrics.local.conf b/conf/rspamd/metrics.local.conf deleted file mode 100644 index 583280e702..0000000000 --- a/conf/rspamd/metrics.local.conf +++ /dev/null @@ -1,8 +0,0 @@ -# Metrics settings -# This define overridden options. - -actions { - reject = 21; - add_header = 8; - greylist = 4; -} diff --git a/conf/rspamd/milter_headers.conf b/conf/rspamd/milter_headers.conf deleted file mode 100644 index d57aa69587..0000000000 --- a/conf/rspamd/milter_headers.conf +++ /dev/null @@ -1,9 +0,0 @@ -use = ["spam-header"]; - -routines { - spam-header { - header = "X-Spam"; - value = "Yes"; - remove = 1; - } -} diff --git a/conf/rspamd/redis.conf b/conf/rspamd/redis.conf deleted file mode 100644 index 03152581ea..0000000000 --- a/conf/rspamd/redis.conf +++ /dev/null @@ -1,2 +0,0 @@ -# set redis server -servers = "127.0.0.1"; diff --git a/conf/rspamd/rspamd.sieve b/conf/rspamd/rspamd.sieve deleted file mode 100644 index 56a30c3c10..0000000000 --- a/conf/rspamd/rspamd.sieve +++ /dev/null @@ -1,4 +0,0 @@ -require ["fileinto"]; -if header :is "X-Spam" "Yes" { - fileinto "Junk"; -} diff --git a/conf/slapd/db_init.ldif b/conf/slapd/db_init.ldif index 8703afb85f..55fad59c6d 100644 --- a/conf/slapd/db_init.ldif +++ b/conf/slapd/db_init.ldif @@ -56,7 +56,6 @@ objectClass: groupOfNamesYnh gidNumber: 4002 cn: all_users permission: cn=mail.main,ou=permission,dc=yunohost,dc=org -permission: cn=xmpp.main,ou=permission,dc=yunohost,dc=org dn: cn=visitors,ou=groups,dc=yunohost,dc=org objectClass: posixGroup @@ -75,17 +74,6 @@ gidNumber: 5001 showTile: FALSE authHeader: FALSE -dn: cn=xmpp.main,ou=permission,dc=yunohost,dc=org -groupPermission: cn=all_users,ou=groups,dc=yunohost,dc=org -cn: xmpp.main -objectClass: posixGroup -objectClass: permissionYnh -isProtected: TRUE -label: XMPP -gidNumber: 5002 -showTile: FALSE -authHeader: FALSE - dn: cn=ssh.main,ou=permission,dc=yunohost,dc=org cn: ssh.main objectClass: posixGroup diff --git a/conf/ssl/openssl.cnf b/conf/ssl/openssl.cnf index a19a9c3df1..84da1b9a00 100644 --- a/conf/ssl/openssl.cnf +++ b/conf/ssl/openssl.cnf @@ -192,7 +192,7 @@ authorityKeyIdentifier=keyid,issuer basicConstraints = CA:FALSE keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName=DNS:yunohost.org,DNS:www.yunohost.org,DNS:ns.yunohost.org,DNS:xmpp-upload.yunohost.org +subjectAltName=DNS:yunohost.org,DNS:www.yunohost.org,DNS:ns.yunohost.org [ v3_ca ] diff --git a/conf/yunohost/services.yml b/conf/yunohost/services.yml index 45621876ee..6d5f6746ea 100644 --- a/conf/yunohost/services.yml +++ b/conf/yunohost/services.yml @@ -8,11 +8,6 @@ fail2ban: log: /var/log/fail2ban.log category: security test_conf: fail2ban-server --test -metronome: - log: [/var/log/metronome/metronome.log,/var/log/metronome/metronome.err] - needs_exposed_ports: [5222, 5269] - category: xmpp - ignore_if_package_is_not_installed: metronome mysql: log: [/var/log/mysql.log,/var/log/mysql.err,/var/log/mysql/error.log] actual_systemd_service: mariadb @@ -28,21 +23,22 @@ nginx: # log: /var/log/php7.4-fpm.log # test_conf: php-fpm7.4 --test # category: web +opendkim: + category: email + test_conf: opendkim -n postfix: log: [/var/log/mail.log,/var/log/mail.err] actual_systemd_service: postfix@- needs_exposed_ports: [25, 587] category: email postgresql: - actual_systemd_service: 'postgresql@13-main' + actual_systemd_service: 'postgresql@15-main' category: database - ignore_if_package_is_not_installed: postgresql-13 + ignore_if_package_is_not_installed: postgresql-15 redis-server: log: /var/log/redis/redis-server.log category: database -rspamd: - log: /var/log/rspamd/rspamd.log - category: email + ignore_if_package_is_not_installed: redis-server slapd: category: database test_conf: slapd -Tt @@ -51,6 +47,9 @@ ssh: test_conf: sshd -t needs_exposed_ports: [22] category: admin +yunohost-portal-api: + log: /var/log/yunohost-portal-api.log + category: userportal yunohost-api: log: /var/log/yunohost/yunohost-api.log category: admin @@ -60,21 +59,6 @@ yunohost-firewall: category: security yunomdns: category: mdns -glances: null -nsswitch: null -ssl: null -yunohost: null -bind9: null -tahoe-lafs: null -memcached: null -udisks2: null -udisk-glue: null -amavis: null -postgrey: null -spamassassin: null -rmilter: null php5-fpm: null php7.0-fpm: null php7.3-fpm: null -nslcd: null -avahi-daemon: null diff --git a/conf/yunohost/yunohost-portal-api.service b/conf/yunohost/yunohost-portal-api.service new file mode 100644 index 0000000000..006af00809 --- /dev/null +++ b/conf/yunohost/yunohost-portal-api.service @@ -0,0 +1,48 @@ +[Unit] +Description=YunoHost Portal API +After=network.target + +[Service] +User=ynh-portal +Group=ynh-portal +Type=simple +ExecStart=/usr/bin/yunohost-portal-api +Restart=always +RestartSec=5 +TimeoutStopSec=30 + +# Sandboxing options to harden security +# Details for these options: https://www.freedesktop.org/software/systemd/man/systemd.exec.html +NoNewPrivileges=yes +PrivateTmp=yes +PrivateDevices=yes +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK +RestrictNamespaces=yes +RestrictRealtime=yes +DevicePolicy=closed +ProtectClock=yes +ProtectHostname=yes +ProtectProc=invisible +ProtectSystem=full +ProtectControlGroups=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +LockPersonality=yes +SystemCallArchitectures=native +SystemCallFilter=~@clock @debug @module @mount @obsolete @reboot @setuid @swap @cpu-emulation @privileged + +# Denying access to capabilities that should not be relevant +# Doc: https://man7.org/linux/man-pages/man7/capabilities.7.html +CapabilityBoundingSet=~CAP_RAWIO CAP_MKNOD +CapabilityBoundingSet=~CAP_AUDIT_CONTROL CAP_AUDIT_READ CAP_AUDIT_WRITE +CapabilityBoundingSet=~CAP_SYS_BOOT CAP_SYS_TIME CAP_SYS_MODULE CAP_SYS_PACCT +CapabilityBoundingSet=~CAP_LEASE CAP_LINUX_IMMUTABLE CAP_IPC_LOCK +CapabilityBoundingSet=~CAP_BLOCK_SUSPEND CAP_WAKE_ALARM +CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG +CapabilityBoundingSet=~CAP_MAC_ADMIN CAP_MAC_OVERRIDE +CapabilityBoundingSet=~CAP_NET_ADMIN CAP_NET_BROADCAST CAP_NET_RAW +CapabilityBoundingSet=~CAP_SYS_ADMIN CAP_SYS_PTRACE CAP_SYSLOG + + +[Install] +WantedBy=multi-user.target diff --git a/debian/changelog b/debian/changelog index f7483d0e77..e3012f5cbf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,132 @@ +yunohost (12.0.6) stable; urgency=low + + - Sync with bullseye branch + + -- Alexandre Aubin Thu, 31 Oct 2024 18:04:21 +0100 + +yunohost (12.0.5.3) stable; urgency=low + + - typo in generate-helpers-doc (ab7bbefe5) + - Merge branch 'dev' into bookworm (9d24a071d) + + Thanks to all contributors <3 ! (Alexandre Aubin, Kayou) + + -- tituspijean Wed, 30 Oct 2024 21:46:48 +0100 + +yunohost (12.0.5.2) testing; urgency=low + + - typo² x_x + + -- Alexandre Aubin Wed, 30 Oct 2024 01:49:37 +0100 + +yunohost (12.0.5.1) testing; urgency=low + + - typo x_x + + -- Alexandre Aubin Wed, 30 Oct 2024 01:21:37 +0100 + +yunohost (12.0.5) testing; urgency=low + + - sso: ihatemoney and owncloud needs the basic auth workaround ([#1984](http://github.com/YunoHost/yunohost/pull/1984)) + - sso: pass email, username and fullname to ssowat via the JWT such that SSOwat can define custom HTTP headers ([#1981](http://github.com/YunoHost/yunohost/pull/1981)) + - portal: fix backup and restore portal settings (d0005e66a) + - portal: fix cookie validity issues by defining 'maxage' + re-setting the cookie every time the validity is extended, such that the client always know wether the cookie is expired or not (deeabfabb) + - portal: support different style/theme for the app tiles via a portal setting ([#1986](http://github.com/YunoHost/yunohost/pull/1986)) + - portal: custom css setting couldnt be emptied (f64cc3f46) + - postinstall/tos: Allow bypassing the terms of services at postinstall time ([#1966](http://github.com/YunoHost/yunohost/pull/1966)) + - helpersv2.1/fail2ban: Fix reload-or-restart match line (4105df9d8) + - helpers/nginx: yoloremove the REMOTE_USER lines from app's conf because it's already provided by /etc/nginx/fastcgi_params + make sure it is mapped to the YNH_USER from SSOwat rather than $remote_user (cd367575f) + - configpanel: better error message when uploading a file from unsupported mimetype (d9f59f3c1) + - i18n: Translations updated for Basque, Chinese (Simplified), French, Kabyle, Turkish + + Thanks to all contributors <3 ! (Aksel Azwaw, axolotle, Éric Gaspar, Félix Piédallu, Furkan Samed Acet, Jinx, Josué Tille, Kay0u, ljf, Mateusz, oleole39, OniriCorpe, Tagada, Thatoo, tituspijean, xabirequejo) + + -- Alexandre Aubin Wed, 30 Oct 2024 00:12:27 +0100 + +yunohost (12.0.4.1) testing; urgency=low + + - ssowat: allow secondaries instances of apps_that_need_external_auth_maybe to be recognized ([#1954](http://github.com/YunoHost/yunohost/pull/1954)) + - domains: fix missing translation param in domain add (cebd8864c) + + Thanks to all contributors <3 ! (axolotle, Nathanaël H.) + + -- Alexandre Aubin Wed, 25 Sep 2024 21:04:38 +0200 + +yunohost (12.0.4) testing; urgency=low + + - general: add TOS acknowledgement during postinstall, dyndns domain creation, and migration for existing installs ([#1949](http://github.com/YunoHost/yunohost/pull/1949)) + - domains: fix `custom_css` may not be in the the form when changing other domain panel values (d5d6fb87a) + - domains: add option to install a lets encrypt certificate when adding a subdomain of an already added domain ([#1936](http://github.com/YunoHost/yunohost/pull/1936)) + - configpanels: fix boolean option in context evaluation for custom yes/no values (f19b6f84a) + - dns: make auto dns feature optional ([#1951](http://github.com/YunoHost/yunohost/pull/1951)) + - cli: Add yunohost-portal to the --version output ([#1946](http://github.com/YunoHost/yunohost/pull/1946)) + + Thanks to all contributors <3 ! (axolotle, Kay0u, Mer, selfhoster1312) + + -- Alexandre Aubin Mon, 16 Sep 2024 18:18:14 +0200 + +yunohost (12.0.3) testing; urgency=low + + - (sync bullseye changes since 12.0.2) + - apps: magically handle yarn as a regular package instead of an 'extra' repo now that yarn's repo is in the core ([#1888](http://github.com/YunoHost/yunohost/pull/1888)) + - portalapi: we don't need absolute URLs for app logos ? (This ain't working when enabling the 'show other domains apps' because of CSP) (bc93a2e07) + - portalapi: fix portal_user_intro not being sent when authenticated, hence not displayed at all (cdf443c86) + - portal/domain settings: Improve explanation about search engine (24fb87725) + - portal/domain settings: Reduce theme list because there were too many (cf change in yunohost-portal) (9973cc703) + - portal/domain settings: add proper i18n string + help for new settings (831131476, ff0388556) + - domain settings: add a title to the Email section to have a separation w.r.t. the portal settings (279f33288) + - portal: fix extra app tiles not being displayed, gotta use the perm id as key, not just the app id (credit rodinux) (603c64e34) + - portal/sso: with the public app page, fix the root of the domain not redirecting to /yunohost/sso (a6b7ba843) + - portal: allow to configure custom CSS from the domain config panel (8f636561d) + - portal: change the way the new 'public apps' page in the portal is configured: add a simple bool toggle instead of having the 'public apps page' as a default app option, which allows to still configure a default app while the portal has the public apps page (748a20d86) + - ci: fix test_permission_propagation_on_ssowat, auth header tests (656e5c75d, 9e9313067, 44920d891) + - ci: optimizations, cleanups, partial refactor because of new CI image build process (fe9a4fba5, 059818254, a9e71e88d, 7f2da0af7, 94594e5a3, d639e1c42, 4f3b9df3f, 5a6a915af, 55e7e798f, 2fe24424f, 4fc929005, fd040b864, 0bbc14f54, 2976e7bf6) + - quality: add type hints to user.py (1ba75df0e, d4f39da20, 611846aa1, efce7f9f0, fe1c04fb2) + - i18n: Translations updated for Basque, French, Galician, Greek, Indonesian, Russian, Turkish + + Thanks to all contributors <3 ! (Ali Çırçır, cjdw, craftrac, Emmanuel Averty, Félix Piédallu, Ivan Davydov, José M, Josué Tille, ljf, OniriCorpe, ppr, selfhoster1312, Tagada, tituspijean, xabirequejo) + + -- Alexandre Aubin Sat, 31 Aug 2024 19:45:00 +0200 + +yunohost (12.0.2) testing; urgency=low + + - Cleanup redis regen conf since redis ain't installed by default anymore (7b50c4eb6) + - bullseye->bookworm: add a trick to flag the migration as done if it's still marked as pending (0503a38a7) + - Sync with main branch + + Thanks to all contributors <3 ! (Kayou) + + -- Alexandre Aubin Thu, 01 Aug 2024 18:08:33 +0200 + +yunohost (12.0.1) testing; urgency=low + + - The user portal and SSO system have been reworked and split into three distinct pieces + - SSOwat only handling only the SSO/ACL logic (nginx lua middleware) + - A new “portal API” (yunohost-portal-api) service delivering authentication cookies and allowing users to retrieve/update infos + - A new portal front end (yunohost-portal) + - More information on the release note on the forum + - The base system does not install Mysql/Mariadb and PHP anymore + - Rspamd (antispam system) and Metronome (XMPP server) are not part of the core anymore. Instead, they are now separate applications : rspamd_ynh and metronome_ynh + - webadmin: rework cookie/session expiration mechanism. Cookies are now still valid after restarting the API (preventing clumsy disconnect during self-upgrades) and the cookie validity is automatically extended every time an API request is performed. + - mail: DKIM email signing is now done using opendkim instead of rspamd + - various compatibility tweakings for Bookworm + - regenconf: update nginx and dovecot ciphers according to Mozilla recommendation + - regenconf: update fail2ban config + - configpanels: refactor to use pydantic for more typing and consistency, add proper autogenerated doc + - apps: Yarn third-party repo is now available by default in apt config just like Sury, no need for an extra apt resource thingy + - various legacy cleanups (more info on the release note on the forum) + - perf: minimize regen-conf calls to yunohost settings get, and other misc lazy-loading optimizations + - quality: simplify the logging mess + - quality: rework ci tests workflow + + -- Alexandre Aubin Fri, 26 Jul 2024 22:40:16 +0200 + +yunohost (12.0.0) unstable; urgency=low + + - Tmp changelog to prepare Bookworm + + -- Alexandre Aubin Thu, 04 May 2023 20:30:19 +0200 + yunohost (11.3.0.2) stable; urgency=low - Fix migration by early-importing _ldap ([#1987](http://github.com/YunoHost/yunohost/pull/1987)) diff --git a/debian/control b/debian/control index 06a665dec6..fe5cb5a20c 100644 --- a/debian/control +++ b/debian/control @@ -2,21 +2,22 @@ Source: yunohost Section: utils Priority: extra Maintainer: YunoHost Contributors -Build-Depends: debhelper (>=9), debhelper-compat (= 13), dh-python, python3-all (>= 3.7), python3-yaml, python3-jinja2 +Build-Depends: debhelper (>=9), debhelper-compat (= 13), dh-python, python3-all (>= 3.11), python3-yaml, python3-jinja2 (>= 3.0) Standards-Version: 3.9.6 Homepage: https://yunohost.org/ Package: yunohost Essential: yes Architecture: all -Depends: ${python3:Depends}, ${misc:Depends} - , moulinette (>= 11.1), moulinette (<< 12.0), ssowat (>= 11.1), ssowat (<< 12.0) +Depends: python3-all (>= 3.11), + , moulinette (>= 12.0), ssowat (>= 12.0), , python3-psutil, python3-requests, python3-dnspython, python3-openssl - , python3-miniupnpc, python3-dbus, python3-jinja2 + , python3-miniupnpc, python3-dbus, python3-jinja2 (>= 3.0) , python3-toml, python3-packaging, python3-publicsuffix2 - , python3-ldap, python3-zeroconf (>= 0.36), python3-lexicon, - , python-is-python3 - , nginx, nginx-extras (>=1.18) + , python3-ldap, python3-zeroconf (>= 0.47), python3-lexicon, + , python3-cryptography, python3-jwt, python3-passlib, python3-magic + , python-is-python3, python3-pydantic, python3-email-validator + , nginx, nginx-extras (>=1.22) , apt, apt-transport-https, apt-utils, aptitude, dirmngr , openssh-server, iptables, fail2ban, bind9-dnsutils , openssl, ca-certificates, netcat-openbsd, iproute2 @@ -24,31 +25,26 @@ Depends: ${python3:Depends}, ${misc:Depends} , dnsmasq, resolvconf, libnss-myhostname , postfix, postfix-ldap, postfix-policyd-spf-perl, postfix-pcre , dovecot-core, dovecot-ldap, dovecot-lmtpd, dovecot-managesieved, dovecot-antispam - , rspamd, opendkim-tools, postsrsd, procmail, mailutils - , redis-server + , opendkim-tools, opendkim, postsrsd, procmail, mailutils , acl , git, curl, wget, cron, unzip, jq, bc, at, procps, j2cli , lsb-release, haveged, fake-hwclock, lsof, whois -Recommends: yunohost-admin +Recommends: yunohost-admin, yunohost-portal (>= 12.0) , ntp, inetutils-ping | iputils-ping , bash-completion, rsyslog - , php7.4-common, php7.4-fpm, php7.4-ldap, php7.4-intl - , mariadb-server, php7.4-mysql - , php7.4-gd, php7.4-curl, php-php-gettext - , python3-pip , unattended-upgrades , libdbd-ldap-perl, libnet-dns-perl - , metronome (>=3.14.0) Conflicts: iptables-persistent , apache2 , bind9 - , nginx-extras (>= 1.19) - , openssl (>= 3.0) - , slapd (>= 2.4.58) - , dovecot-core (>= 1:2.3.14) - , redis-server (>= 5:6.1) - , fail2ban (>= 0.11.3) - , iptables (>= 1.8.8) + , openresolv + , systemd-resolved + , nginx-extras (>= 1.23) + , openssl (>= 3.1) + , slapd (>= 2.6) + , dovecot-core (>= 1:2.4) + , fail2ban (>= 1.1) + , iptables (>= 1.8.10) Description: manageable and configured self-hosting server YunoHost aims to make self-hosting accessible to everyone. It configures an email, Web and IM server alongside a LDAP base. It also provides diff --git a/debian/install b/debian/install index 86636fa93c..1a0cf583cd 100644 --- a/debian/install +++ b/debian/install @@ -6,5 +6,4 @@ conf/* /usr/share/yunohost/conf/ locales/* /usr/share/yunohost/locales/ doc/yunohost.8.gz /usr/share/man/man8/ doc/bash_completion.d/* /etc/bash_completion.d/ -conf/metronome/modules/* /usr/lib/metronome/modules/ src/* /usr/lib/python3/dist-packages/yunohost/ diff --git a/debian/postinst b/debian/postinst index b8a07442fe..4f784a516a 100644 --- a/debian/postinst +++ b/debian/postinst @@ -4,6 +4,10 @@ set -e do_configure() { + mkdir -p /etc/yunohost + mkdir -p /etc/yunohost/apps + mkdir -p /etc/yunohost/portal + if [ ! -f /etc/yunohost/installed ]; then # If apps/ is not empty, we're probably already installed in the past and # something funky happened ... @@ -33,6 +37,8 @@ do_configure() { yunohost tools update apps --output-as none || true fi + systemctl restart yunohost-portal-api + # Trick to let yunohost handle the restart of the API, # to prevent the webadmin from cutting the branch it's sitting on if systemctl is-enabled yunohost-api --quiet diff --git a/doc/generate_configpanel_and_formoptions_doc.py b/doc/generate_configpanel_and_formoptions_doc.py new file mode 100644 index 0000000000..156a769fc3 --- /dev/null +++ b/doc/generate_configpanel_and_formoptions_doc.py @@ -0,0 +1,181 @@ +import ast +import datetime +import subprocess + +version = open("../debian/changelog").readlines()[0].split()[1].strip("()") +today = datetime.datetime.now().strftime("%d/%m/%Y") + + +def get_current_commit(): + p = subprocess.Popen( + "git rev-parse --verify HEAD", + shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + stdout, stderr = p.communicate() + + current_commit = stdout.strip().decode("utf-8") + return current_commit + + +current_commit = get_current_commit() + + +def print_config_panel_docs(): + fname = "../src/utils/configpanel.py" + content = open(fname).read() + + # NB: This magic is because we want to be able to run this script outside of a YunoHost context, + # in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports... + tree = ast.parse(content) + + ConfigPanelClasses = reversed( + [ + c + for c in tree.body + if isinstance(c, ast.ClassDef) + and c.name in {"SectionModel", "PanelModel", "ConfigPanelModel"} + ] + ) + + print("## Configuration panel structure") + + for c in ConfigPanelClasses: + doc = ast.get_docstring(c) + print("") + print(f"### {c.name.replace('Model', '')}") + print("") + print(doc) + print("") + print("---") + + +def print_form_doc(): + fname = "../src/utils/form.py" + content = open(fname).read() + + # NB: This magic is because we want to be able to run this script outside of a YunoHost context, + # in which we cant really 'import' the file because it will trigger a bunch of moulinette/yunohost imports... + tree = ast.parse(content) + + OptionClasses = [ + c + for c in tree.body + if isinstance(c, ast.ClassDef) and c.name.endswith("Option") + ] + + OptionDocString = {} + + print("## List of all option types") + + for c in OptionClasses: + if not isinstance(c.body[0], ast.Expr): + continue + option_type = None + + if c.name in {"BaseOption", "BaseInputOption"}: + option_type = c.name + elif c.body[1].target.id == "type": + option_type = c.body[1].value.attr + + generaltype = ( + c.bases[0].id.replace("Option", "").replace("Base", "").lower() + if c.bases + else None + ) + + docstring = ast.get_docstring(c) + if docstring: + if "#### Properties" not in docstring: + docstring += """ +#### Properties + +- [common properties](#common-properties)""" + OptionDocString[option_type] = { + "doc": docstring, + "generaltype": generaltype, + } + + # Dirty hack to have "BaseOption" as first and "BaseInputOption" as 2nd in list + + base = OptionDocString.pop("BaseOption") + baseinput = OptionDocString.pop("BaseInputOption") + OptionDocString2 = { + "BaseOption": base, + "BaseInputOption": baseinput, + } + OptionDocString2.update(OptionDocString) + + for option_type, infos in OptionDocString2.items(): + if option_type == "display_text": + # display_text is kind of legacy x_x + continue + print("") + if option_type == "BaseOption": + print("### Common properties") + elif option_type == "BaseInputOption": + print("### Common inputs properties") + else: + print( + f"### `{option_type}`" + + (f" ({infos['generaltype']})" if infos["generaltype"] else "") + ) + print("") + print(infos["doc"]) + print("") + print("---") + + +print( + rf"""--- +title: Technical details for config panel structure and form option types +template: docs +taxonomy: + category: docs +routes: + default: '/dev/forms' +--- + +Doc auto-generated by [this script](https://github.com/YunoHost/yunohost/blob/{current_commit}/doc/generate_options_doc.py) on {today} (YunoHost version {version}) + +## Glossary + +You may encounter some named types which are used for simplicity. + +- `Translation`: a translated property + - used for properties: `ask`, `help` and `Pattern.error` + - a `dict` with locales as keys and translations as values: + ```toml + ask.en = "The text in english" + ask.fr = "Le texte en français" + ``` + It is not currently possible for translators to translate those string in weblate. + - a single `str` for a single english default string + ```toml + help = "The text in english" + ``` +- `JSExpression`: a `str` JS expression to be evaluated to `true` or `false`: + - used for properties: `visible` and `enabled` + - operators availables: `==`, `!=`, `>`, `>=`, `<`, `<=`, `!`, `&&`, `||`, `+`, `-`, `*`, `/`, `%` and `match()` +- `Binding`: bind a value to a file/property/variable/getter/setter/validator + - save the value in `settings.yaml` when not defined + - nothing at all with `"null"` + - a custom getter/setter/validator with `"null"` + a function starting with `get__`, `set__`, `validate__` in `scripts/config` + - a variable/property in a file with `:__FINALPATH__/my_file.php` + - a whole file with `__FINALPATH__/my_file.php` +- `Pattern`: a `dict` with a regex to match the value against and an error message + ```toml + pattern.regexp = '^[A-F]\d\d$' + pattern.error = "Provide a room number such as F12: one uppercase and 2 numbers" + # or with translated error + pattern.error.en = "Provide a room number such as F12: one uppercase and 2 numbers" + pattern.error.fr = "Entrez un numéro de salle comme F12: une lettre majuscule et deux chiffres." + ``` + - IMPORTANT: your `pattern.regexp` should be between simple quote, not double. + +""" +) + +print_config_panel_docs() +print_form_doc() diff --git a/doc/generate_helper_doc.py b/doc/generate_helper_doc.py index d572b03da0..c4978863a7 100644 --- a/doc/generate_helper_doc.py +++ b/doc/generate_helper_doc.py @@ -2,7 +2,6 @@ import sys import os -import glob import datetime import subprocess @@ -269,7 +268,7 @@ def main(): for subsection in section["subsections"]: print(f"Parsing {subsection} ...") helper_file = f"../helpers/helpers.v{version}.d/{subsection}" - assert os.path.isfile(helper_file), f"Uhoh, {file} doesn't exists?" + assert os.path.isfile(helper_file), f"Uhoh, {helper_file} doesn't exists?" p = Parser(helper_file) p.parse_blocks() for b in p.blocks: diff --git a/doc/generate_json_schema.py b/doc/generate_json_schema.py new file mode 100644 index 0000000000..1abf88915f --- /dev/null +++ b/doc/generate_json_schema.py @@ -0,0 +1,4 @@ +from yunohost.utils.configpanel import ConfigPanelModel + + +print(ConfigPanelModel.schema_json(indent=2)) diff --git a/helpers/helpers.v1.d/fail2ban b/helpers/helpers.v1.d/fail2ban index ae4ce86502..548989e5f2 100644 --- a/helpers/helpers.v1.d/fail2ban +++ b/helpers/helpers.v1.d/fail2ban @@ -113,7 +113,7 @@ ignoreregex = chown -R "$app:$app" "/var/log/$app" chmod -R u=rwX,g=rX,o= "/var/log/$app" - ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + ynh_systemd_action --service_name=fail2ban --action=reload --line_match="(Started|Reloaded) fail2ban.service" --log_path=systemd local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then diff --git a/helpers/helpers.v1.d/mysql b/helpers/helpers.v1.d/mysql index f4b1d6a06c..cf56173f76 100644 --- a/helpers/helpers.v1.d/mysql +++ b/helpers/helpers.v1.d/mysql @@ -227,6 +227,9 @@ ynh_mysql_setup_db() { # If $db_pwd is not provided, use new_db_pwd instead for db_pwd db_pwd="${db_pwd:-$new_db_pwd}" + # Dirty patch for super-legacy apps + dpkg --list | grep -q "^ii mariadb-server" || { ynh_print_warn "Packager: you called ynh_mysql_setup_db without declaring a dependency to mariadb-server. Please add it to your apt dependencies !"; ynh_apt install mariadb-server; } + ynh_mysql_create_db "$db_name" "$db_user" "$db_pwd" ynh_app_setting_set --app=$app --key=mysqlpwd --value=$db_pwd } diff --git a/helpers/helpers.v1.d/nginx b/helpers/helpers.v1.d/nginx index 78a2002aab..d4ba7bac82 100644 --- a/helpers/helpers.v1.d/nginx +++ b/helpers/helpers.v1.d/nginx @@ -28,6 +28,11 @@ ynh_add_nginx_config() { ynh_replace_string --match_string="^#root_path_only" --replace_string="" --target_file="$finalnginxconf" fi + # Delete REMOTE_USER mapping, it's already provided by + # /etc/nginx/fastcgi_params which all PHP apps include, and maps to the + # appropriate YNH_USER HTTP header instead of $remote_user + sed -i '/fastcgi_param\s*REMOTE_USER/d' "$finalnginxconf" + ynh_store_file_checksum --file="$finalnginxconf" ynh_systemd_action --service_name=nginx --action=reload diff --git a/helpers/helpers.v1.d/php b/helpers/helpers.v1.d/php index 1052de330d..22628da855 100644 --- a/helpers/helpers.v1.d/php +++ b/helpers/helpers.v1.d/php @@ -1,6 +1,6 @@ #!/bin/bash -readonly YNH_DEFAULT_PHP_VERSION=7.4 +readonly YNH_DEFAULT_PHP_VERSION=8.2 # Declare the actual PHP version to use. # A packager willing to use another version of PHP can override the variable into its _common.sh. YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} @@ -70,17 +70,14 @@ YNH_PHP_VERSION=${YNH_PHP_VERSION:-$YNH_DEFAULT_PHP_VERSION} ynh_add_fpm_config() { local _globalphpversion=${phpversion-:} # Declare an array to define the options of this helper. - local legacy_args=vufpdg - local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [p]=package= [d]=dedicated_service [g]=group=) + local legacy_args=vufg + local -A args_array=([v]=phpversion= [u]=usage= [f]=footprint= [g]=group=) local group local phpversion local usage local footprint - local package - local dedicated_service # Manage arguments with getopts ynh_handle_getopts_args "$@" - package=${package:-} group=${group:-} # The default behaviour is to use the template. @@ -105,8 +102,6 @@ ynh_add_fpm_config() { fi fi - # Do not use a dedicated service by default - dedicated_service=${dedicated_service:-0} # Set the default PHP-FPM version by default if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then @@ -129,45 +124,16 @@ ynh_add_fpm_config() { fi fi - # Legacy args (packager should just list their php dependency as regular apt dependencies... - if [ -n "$package" ]; then - # Install the additionnal packages from the default repository - ynh_print_warn --message "Argument --package of ynh_add_fpm_config is deprecated and to be removed in the future" - ynh_install_app_dependencies "$package" - fi - - if [ $dedicated_service -eq 1 ]; then - ynh_print_warn --message "Argument --dedicated_service of ynh_add_fpm_config is deprecated and to be removed in the future" - local fpm_service="${app}-phpfpm" - local fpm_config_dir="/etc/php/$phpversion/dedicated-fpm" - else - local fpm_service="php${phpversion}-fpm" - local fpm_config_dir="/etc/php/$phpversion/fpm" - fi + local fpm_service="php${phpversion}-fpm" + local fpm_config_dir="/etc/php/$phpversion/fpm" # Create the directory for FPM pools mkdir --parents "$fpm_config_dir/pool.d" ynh_app_setting_set --app=$app --key=fpm_config_dir --value="$fpm_config_dir" ynh_app_setting_set --app=$app --key=fpm_service --value="$fpm_service" - ynh_app_setting_set --app=$app --key=fpm_dedicated_service --value="$dedicated_service" ynh_app_setting_set --app=$app --key=phpversion --value=$phpversion - # Migrate from mutual PHP service to dedicated one. - if [ $dedicated_service -eq 1 ]; then - local old_fpm_config_dir="/etc/php/$phpversion/fpm" - # If a config file exist in the common pool, move it. - if [ -e "$old_fpm_config_dir/pool.d/$app.conf" ]; then - ynh_print_info --message="Migrate to a dedicated php-fpm service for $app." - # Create a backup of the old file before migration - ynh_backup_if_checksum_is_different --file="$old_fpm_config_dir/pool.d/$app.conf" - # Remove the old PHP config file - ynh_secure_remove --file="$old_fpm_config_dir/pool.d/$app.conf" - # Reload PHP to release the socket and allow the dedicated service to use it - ynh_systemd_action --service_name=php${phpversion}-fpm --action=reload - fi - fi - if [ $autogenconf == "false" ]; then # Usage 1, use the template in conf/php-fpm.conf local phpfpm_path="$YNH_APP_BASEDIR/conf/php-fpm.conf" @@ -221,56 +187,14 @@ pm.process_idle_timeout = 10s local finalphpconf="$fpm_config_dir/pool.d/$app.conf" ynh_add_config --template="$phpfpm_path" --destination="$finalphpconf" - if [ -e "$YNH_APP_BASEDIR/conf/php-fpm.ini" ]; then - ynh_print_warn --message="Packagers ! Please do not use a separate php ini file, merge your directives in the pool file instead." - ynh_add_config --template="php-fpm.ini" --destination="$fpm_config_dir/conf.d/20-$app.ini" + # Validate that the new php conf doesn't break php-fpm entirely + if ! php-fpm${phpversion} --test 2>/dev/null; then + php-fpm${phpversion} --test || true + ynh_secure_remove --file="$finalphpconf" + ynh_die --message="The new configuration broke php-fpm?" fi - if [ $dedicated_service -eq 1 ]; then - # Create a dedicated php-fpm.conf for the service - local globalphpconf=$fpm_config_dir/php-fpm-$app.conf - - echo "[global] -pid = /run/php/php__PHPVERSION__-fpm-__APP__.pid -error_log = /var/log/php/fpm-php.__APP__.log -syslog.ident = php-fpm-__APP__ -include = __FINALPHPCONF__ -" > $YNH_APP_BASEDIR/conf/php-fpm-$app.conf - - ynh_add_config --template="php-fpm-$app.conf" --destination="$globalphpconf" - - # Create a config for a dedicated PHP-FPM service for the app - echo "[Unit] -Description=PHP __PHPVERSION__ FastCGI Process Manager for __APP__ -After=network.target - -[Service] -Type=notify -PIDFile=/run/php/php__PHPVERSION__-fpm-__APP__.pid -ExecStart=/usr/sbin/php-fpm__PHPVERSION__ --nodaemonize --fpm-config __GLOBALPHPCONF__ -ExecReload=/bin/kill -USR2 \$MAINPID - -[Install] -WantedBy=multi-user.target -" > $YNH_APP_BASEDIR/conf/$fpm_service - - # Create this dedicated PHP-FPM service - ynh_add_systemd_config --service=$fpm_service --template=$fpm_service - # Integrate the service in YunoHost admin panel - yunohost service add $fpm_service --log /var/log/php/fpm-php.$app.log --description "Php-fpm dedicated to $app" - # Configure log rotate - ynh_use_logrotate --logfile=/var/log/php - # Restart the service, as this service is either stopped or only for this app - ynh_systemd_action --service_name=$fpm_service --action=restart - else - # Validate that the new php conf doesn't break php-fpm entirely - if ! php-fpm${phpversion} --test 2> /dev/null; then - php-fpm${phpversion} --test || true - ynh_secure_remove --file="$finalphpconf" - ynh_die --message="The new configuration broke php-fpm?" - fi - ynh_systemd_action --service_name=$fpm_service --action=reload - fi + ynh_systemd_action --service_name=$fpm_service --action=reload } # Remove the dedicated PHP-FPM config @@ -281,8 +205,6 @@ WantedBy=multi-user.target ynh_remove_fpm_config() { local fpm_config_dir=$(ynh_app_setting_get --app=$app --key=fpm_config_dir) local fpm_service=$(ynh_app_setting_get --app=$app --key=fpm_service) - local dedicated_service=$(ynh_app_setting_get --app=$app --key=fpm_dedicated_service) - dedicated_service=${dedicated_service:-0} # Get the version of PHP used by this app local phpversion=$(ynh_app_setting_get --app=$app --key=phpversion) @@ -296,69 +218,7 @@ ynh_remove_fpm_config() { fi ynh_secure_remove --file="$fpm_config_dir/pool.d/$app.conf" - if [ -e $fpm_config_dir/conf.d/20-$app.ini ]; then - ynh_secure_remove --file="$fpm_config_dir/conf.d/20-$app.ini" - fi - - if [ $dedicated_service -eq 1 ]; then - # Remove the dedicated service PHP-FPM service for the app - ynh_remove_systemd_config --service=$fpm_service - # Remove the global PHP-FPM conf - ynh_secure_remove --file="$fpm_config_dir/php-fpm-$app.conf" - # Remove the service from the list of services known by YunoHost - yunohost service remove $fpm_service - elif ynh_package_is_installed --package="php${phpversion}-fpm"; then - ynh_systemd_action --service_name=$fpm_service --action=reload - fi - - # If the PHP version used is not the default version for YunoHost - # The second part with YNH_APP_PURGE is an ugly hack to guess that we're inside the remove script - # (we don't actually care about its value, we just check its not empty hence it exists) - if [ "$phpversion" != "$YNH_DEFAULT_PHP_VERSION" ] && [ -n "${YNH_APP_PURGE:-}" ] && dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} lt 2; then - # Remove app dependencies ... but ideally should happen via an explicit call from packager - ynh_remove_app_dependencies - fi -} - -# Install another version of PHP. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_install_php --phpversion=phpversion [--package=packages] -# | arg: -v, --phpversion= - Version of PHP to install. -# | arg: -p, --package= - Additionnal PHP packages to install -# -# Requires YunoHost version 3.8.1 or higher. -ynh_install_php() { - # Declare an array to define the options of this helper. - local legacy_args=vp - local -A args_array=([v]=phpversion= [p]=package=) - local phpversion - local package - # Manage arguments with getopts - ynh_handle_getopts_args "$@" - package=${package:-} - - if [ "$phpversion" == "$YNH_DEFAULT_PHP_VERSION" ]; then - ynh_die --message="Do not use ynh_install_php to install php$YNH_DEFAULT_PHP_VERSION" - fi - - ynh_install_app_dependencies "$package" -} - -# Remove the specific version of PHP used by the app. -# -# [internal] -# -# Legacy, to be remove on bullseye -# -# usage: ynh_remove_php -# -# Requires YunoHost version 3.8.1 or higher. -ynh_remove_php() { - ynh_remove_app_dependencies + ynh_systemd_action --service_name=$fpm_service --action=reload } # Define the values to configure PHP-FPM diff --git a/helpers/helpers.v1.d/postgresql b/helpers/helpers.v1.d/postgresql index 82a5e4cdf0..4daf8005e6 100644 --- a/helpers/helpers.v1.d/postgresql +++ b/helpers/helpers.v1.d/postgresql @@ -1,7 +1,7 @@ #!/bin/bash PSQL_ROOT_PWD_FILE=/etc/yunohost/psql -PSQL_VERSION=13 +PSQL_VERSION=15 # Open a connection as a user # diff --git a/helpers/helpers.v1.d/setting b/helpers/helpers.v1.d/setting index 5f4104a8dd..d604076028 100644 --- a/helpers/helpers.v1.d/setting +++ b/helpers/helpers.v1.d/setting @@ -18,11 +18,7 @@ ynh_app_setting_get() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key - else - ynh_app_setting "get" "$app" "$key" - fi + ynh_app_setting "get" "$app" "$key" } # Set an application setting @@ -45,11 +41,7 @@ ynh_app_setting_set() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ $key =~ (unprotected|protected|skipped)_ ]]; then - yunohost app setting $app $key -v $value - else - ynh_app_setting "set" "$app" "$key" "$value" - fi + ynh_app_setting "set" "$app" "$key" "$value" } # Set an application setting but only if the "$key" variable ain't set yet @@ -106,11 +98,7 @@ ynh_app_setting_delete() { ynh_handle_getopts_args "$@" app="${app:-$_globalapp}" - if [[ "$key" =~ (unprotected|skipped|protected)_ ]]; then - yunohost app setting $app $key -d - else - ynh_app_setting "delete" "$app" "$key" - fi + ynh_app_setting "delete" "$app" "$key" } # Small "hard-coded" interface to avoid calling "yunohost app" directly each diff --git a/helpers/helpers.v1.d/utils b/helpers/helpers.v1.d/utils index 780b62c611..0a390e8087 100644 --- a/helpers/helpers.v1.d/utils +++ b/helpers/helpers.v1.d/utils @@ -367,15 +367,11 @@ ynh_compare_current_package_version() { _ynh_apply_default_permissions() { local target=$1 - local ynh_requirement=$(ynh_read_manifest --manifest_key="requirements.yunohost" | tr -d '<>= ') - - if dpkg --compare-versions ${YNH_APP_PACKAGING_FORMAT:-0} ge 2 || [ -z "$ynh_requirement" ] || [ "$ynh_requirement" == "null" ] || dpkg --compare-versions $ynh_requirement ge 4.2; then - chmod o-rwx $target - chmod g-w $target - chown -R root:root $target - if ynh_system_user_exists $app; then - chown $app:$app $target - fi + chmod o-rwx $target + chmod g-w $target + chown -R root:root $target + if ynh_system_user_exists $app; then + chown $app:$app $target fi # Crons should be owned by root @@ -388,7 +384,7 @@ _ynh_apply_default_permissions() { } int_to_bool() { - sed -e 's/^1$/True/g' -e 's/^0$/False/g' + sed -e 's/^1$/True/g' -e 's/^0$/False/g' -e 's/^true$/True/g' -e 's/^false$/False/g' } toml_to_json() { diff --git a/helpers/helpers.v2.1.d/fail2ban b/helpers/helpers.v2.1.d/fail2ban index 8967bf5c82..9a6305376c 100644 --- a/helpers/helpers.v2.1.d/fail2ban +++ b/helpers/helpers.v2.1.d/fail2ban @@ -99,7 +99,7 @@ ignoreregex = chown -R "$app:$app" "/var/log/$app" chmod -R u=rwX,g=rX,o= "/var/log/$app" - ynh_systemctl --service=fail2ban --action=reload --wait_until="(Started|Reloaded) Fail2Ban Service" --log_path=systemd + ynh_systemctl --service=fail2ban --action=reload --wait_until="(Started|Reloaded) fail2ban.service" --log_path=systemd local fail2ban_error="$(journalctl --no-hostname --unit=fail2ban | tail --lines=50 | grep "WARNING.*$app.*")" if [[ -n "$fail2ban_error" ]]; then diff --git a/helpers/helpers.v2.1.d/nginx b/helpers/helpers.v2.1.d/nginx index 7797621d42..e824ded000 100644 --- a/helpers/helpers.v2.1.d/nginx +++ b/helpers/helpers.v2.1.d/nginx @@ -26,6 +26,11 @@ ynh_config_add_nginx() { ynh_replace --match="^#root_path_only" --replace="" --file="$finalnginxconf" fi + # Delete REMOTE_USER mapping, it's already provided by + # /etc/nginx/fastcgi_params which all PHP apps include, and maps to the + # appropriate YNH_USER HTTP header instead of $remote_user + sed -i '/fastcgi_param\s*REMOTE_USER/d' "$finalnginxconf" + ynh_store_file_checksum "$finalnginxconf" ynh_systemctl --service=nginx --action=reload diff --git a/hooks/backup/20-conf_ynh_settings b/hooks/backup/20-conf_ynh_settings index 0820978e77..f960bfeb4b 100644 --- a/hooks/backup/20-conf_ynh_settings +++ b/hooks/backup/20-conf_ynh_settings @@ -12,6 +12,7 @@ backup_dir="${1}/conf/ynh" # Backup the configuration ynh_backup "/etc/yunohost/firewall.yml" "${backup_dir}/firewall.yml" ynh_backup "/etc/yunohost/current_host" "${backup_dir}/current_host" +[ ! -d "/etc/yunohost/portal" ] || ynh_backup "/etc/yunohost/portal" "${backup_dir}/portal" [ ! -d "/etc/yunohost/domains" ] || ynh_backup "/etc/yunohost/domains" "${backup_dir}/domains" [ ! -e "/etc/yunohost/settings.yml" ] || ynh_backup "/etc/yunohost/settings.yml" "${backup_dir}/settings.yml" [ ! -d "/etc/yunohost/dyndns" ] || ynh_backup "/etc/yunohost/dyndns" "${backup_dir}/dyndns" diff --git a/hooks/backup/27-data_xmpp b/hooks/backup/27-data_xmpp deleted file mode 100644 index 2cd93e02b1..0000000000 --- a/hooks/backup/27-data_xmpp +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -# Exit hook on subcommand error or unset variable -set -eu - -# Source YNH helpers -source /usr/share/yunohost/helpers - -# Backup destination -backup_dir="${1}/data/xmpp" - -ynh_backup /var/lib/metronome "${backup_dir}/var_lib_metronome" -ynh_backup /var/xmpp-upload/ "${backup_dir}/var_xmpp-upload" diff --git a/hooks/conf_regen/01-yunohost b/hooks/conf_regen/01-yunohost index 5701a8997c..7fb2812cdc 100755 --- a/hooks/conf_regen/01-yunohost +++ b/hooks/conf_regen/01-yunohost @@ -2,61 +2,183 @@ set -e -do_init_regen() { - if [[ $EUID -ne 0 ]]; then - echo "You must be root to run this script" 1>&2 - exit 1 - fi +base_folder_and_perm_init() { - cd /usr/share/yunohost/conf/yunohost + ############################# + # Base yunohost conf folder # + ############################# - [[ -d /etc/yunohost ]] || mkdir -p /etc/yunohost + mkdir -p /etc/yunohost + # NB: x permission for 'others' is important for ssl-cert (and maybe mdns), otherwise slapd will fail to start because can't access the certs + chmod 755 /etc/yunohost - # set default current_host - [[ -f /etc/yunohost/current_host ]] \ - || echo "yunohost.org" > /etc/yunohost/current_host + ################ + # Logs folders # + ################ + + mkdir -p /var/log/yunohost + chown root:root /var/log/yunohost + chmod 750 /var/log/yunohost + + ################## + # Portal folders # + ################## + + getent passwd ynh-portal &>/dev/null || useradd --no-create-home --shell /usr/sbin/nologin --system --user-group ynh-portal + + mkdir -p /etc/yunohost/portal + chmod 500 /etc/yunohost/portal + chown ynh-portal:ynh-portal /etc/yunohost/portal + + mkdir -p /usr/share/yunohost/portal/customassets + chmod 775 /usr/share/yunohost/portal/customassets + chown root:root /usr/share/yunohost/portal/customassets + + touch /var/log/yunohost-portalapi.log + chown ynh-portal:root /var/log/yunohost-portalapi.log + chmod 600 /var/log/yunohost-portalapi.log + + ############################### + # Sessions folder and secrets # + ############################### + + # Portal + mkdir -p /var/cache/yunohost-portal/sessions + chown ynh-portal:www-data /var/cache/yunohost-portal + chmod 510 /var/cache/yunohost-portal + chown ynh-portal:www-data /var/cache/yunohost-portal/sessions + chmod 710 /var/cache/yunohost-portal/sessions + + # Webadmin + mkdir -p /var/cache/yunohost/sessions + chown root:root /var/cache/yunohost/sessions + chmod 700 /var/cache/yunohost/sessions + + if test -e /etc/yunohost/installed + then + # Initialize session secrets + # Obviously we only do this in the post_regen, ie during the postinstall, because we don't want every pre-installed instance to have the same secret + if [ ! -e /etc/yunohost/.admin_cookie_secret ]; then + dd if=/dev/urandom bs=1 count=1000 2>/dev/null | tr --complement --delete 'A-Za-z0-9' | head -c 64 > /etc/yunohost/.admin_cookie_secret + fi + chown root:root /etc/yunohost/.admin_cookie_secret + chmod 400 /etc/yunohost/.admin_cookie_secret + + if [ ! -e /etc/yunohost/.ssowat_cookie_secret ]; then + # NB: we need this to be exactly 32 char long, because it is later used as a key for AES256 + dd if=/dev/urandom bs=1 count=1000 2>/dev/null | tr --complement --delete 'A-Za-z0-9' | head -c 32 > /etc/yunohost/.ssowat_cookie_secret + fi + chown ynh-portal:root /etc/yunohost/.ssowat_cookie_secret + chmod 400 /etc/yunohost/.ssowat_cookie_secret + fi - # copy default services and firewall - [[ -f /etc/yunohost/firewall.yml ]] \ - || cp firewall.yml /etc/yunohost/firewall.yml + ################## + # Domain folders # + ################## - # allow users to access /media directory - [[ -d /etc/skel/media ]] \ - || (mkdir -p /media && ln -s /media /etc/skel/media) + mkdir -p /etc/yunohost/domains + chown root /etc/yunohost/domains + chmod 700 /etc/yunohost/domains - # Cert folders - mkdir -p /etc/yunohost/certs - chown -R root:ssl-cert /etc/yunohost/certs - chmod 750 /etc/yunohost/certs + ############### + # App folders # + ############### - # App folders mkdir -p /etc/yunohost/apps + chown root /etc/yunohost/apps chmod 700 /etc/yunohost/apps + + ##################### + # Apps data folders # + ##################### + mkdir -p /home/yunohost.app chmod 755 /home/yunohost.app - # Domain settings - mkdir -p /etc/yunohost/domains - chmod 700 /etc/yunohost/domains + ################ + # Certs folder # + ################ + + mkdir -p /etc/yunohost/certs + chown -R root:ssl-cert /etc/yunohost/certs + chmod 750 /etc/yunohost/certs + # We do this with find because there could be a lot of them... + find /etc/yunohost/certs/ -type f -exec chmod 640 {} \; + find /etc/yunohost/certs/ -type d -exec chmod 750 {} \; + + ################## + # Backup folders # + ################## - # Backup folders mkdir -p /home/yunohost.backup/archives - chmod 750 /home/yunohost.backup/archives - chown root:root /home/yunohost.backup/archives # This is later changed to root:admins once the admins group exists + chmod 770 /home/yunohost.backup + chmod 770 /home/yunohost.backup/archives + + if test -e /etc/yunohost/installed + then + # The admins group only exist after the postinstall + chown root:admins /home/yunohost.backup + chown root:admins /home/yunohost.backup/archives + else + chown root:root /home/yunohost.backup + chown root:root /home/yunohost.backup/archives + fi + + ######## + # Misc # + ######## + + mkdir -p /etc/yunohost/hooks.d + chown root /etc/yunohost/hooks.d + chmod 700 /etc/yunohost/hooks.d + + mkdir -p /var/cache/yunohost/repo + chown root:root /var/cache/yunohost + chmod 700 /var/cache/yunohost + + [ ! -e /var/www/.well-known/ynh-diagnosis/ ] || chmod 775 /var/www/.well-known/ynh-diagnosis/ + + if test -e /etc/yunohost/installed + then + setfacl -m g:all_users:--- /var/www + setfacl -m g:all_users:--- /var/log/nginx + setfacl -m g:all_users:--- /etc/yunohost + setfacl -m g:all_users:--- /etc/ssowat + fi +} + +do_init_regen() { + + cd /usr/share/yunohost/conf/yunohost + + base_folder_and_perm_init # Empty ssowat json persistent conf - echo "{}" > '/etc/ssowat/conf.json.persistent' + echo "{}" >'/etc/ssowat/conf.json.persistent' chmod 644 /etc/ssowat/conf.json.persistent chown root:root /etc/ssowat/conf.json.persistent + echo "{}" >'/etc/ssowat/conf.json' + chmod 644 /etc/ssowat/conf.json + chown root:root /etc/ssowat/conf.json # Empty service conf touch /etc/yunohost/services.yml - mkdir -p /var/cache/yunohost/repo - chown root:root /var/cache/yunohost - chmod 700 /var/cache/yunohost + # set default current_host + [[ -f /etc/yunohost/current_host ]] \ + || echo "yunohost.org" > /etc/yunohost/current_host + + # copy default services and firewall + [[ -f /etc/yunohost/firewall.yml ]] \ + || cp firewall.yml /etc/yunohost/firewall.yml + # allow users to access /media directory + [[ -d /etc/skel/media ]] \ + || (mkdir -p /media && ln -s /media /etc/skel/media) + + # YunoHost services cp yunohost-api.service /etc/systemd/system/yunohost-api.service + cp yunohost-portal-api.service /etc/systemd/system/yunohost-portal-api.service cp yunohost-firewall.service /etc/systemd/system/yunohost-firewall.service cp yunoprompt.service /etc/systemd/system/yunoprompt.service @@ -65,6 +187,9 @@ do_init_regen() { systemctl enable yunohost-api.service --quiet systemctl start yunohost-api.service + systemctl enable yunohost-portal-api.service --quiet + systemctl start yunohost-portal-api.service + # Enable yunoprompt (in particular for installs from ISO where we want this to show on first boot instead of asking for a login/password) systemctl enable yunoprompt --quiet @@ -157,6 +282,7 @@ HandleLidSwitchExternalPower=ignore EOF cp yunohost-api.service ${pending_dir}/etc/systemd/system/yunohost-api.service + cp yunohost-portal-api.service ${pending_dir}/etc/systemd/system/yunohost-portal-api.service cp yunohost-firewall.service ${pending_dir}/etc/systemd/system/yunohost-firewall.service cp yunoprompt.service ${pending_dir}/etc/systemd/system/yunoprompt.service cp proc-hidepid.service ${pending_dir}/etc/systemd/system/proc-hidepid.service @@ -171,62 +297,45 @@ EOF do_post_regen() { regen_conf_files=$1 - ###################### - # Enfore permissions # - ###################### + # Re-mkdir / apply permission to all basic folders etc + base_folder_and_perm_init - chmod 770 /home/yunohost.backup - chmod 770 /home/yunohost.backup/archives - chmod 700 /var/cache/yunohost - chown root:admins /home/yunohost.backup - chown root:admins /home/yunohost.backup/archives - chown root:root /var/cache/yunohost - - [ ! -e /var/www/.well-known/ynh-diagnosis/ ] || chmod 775 /var/www/.well-known/ynh-diagnosis/ - - # NB: x permission for 'others' is important for ssl-cert (and maybe mdns), otherwise slapd will fail to start because can't access the certs - chmod 755 /etc/yunohost + # Legacy log tree structure + if [ ! -e /var/log/yunohost/operations ] + then + mkdir -p /var/log/yunohost/operations + fi + if [ -d /var/log/yunohost/categories/operation ] && [ ! -L /var/log/yunohost/categories/operation ] + then + # (we use find -type f instead of mv /folder/* to make sure to also move hidden files which are not included in globs by default) + find /var/log/yunohost/categories/operation/ -type f -print0 | xargs -0 -I {} mv {} /var/log/yunohost/operations/ + # Attempt to delete the old dir (because we want it to be a symlink) or just rename it if it can't be removed (not empty) for some reason + rmdir /var/log/yunohost/categories/operation || mv /var/log/yunohost/categories/operation /var/log/yunohost/categories/operation.old + ln -s /var/log/yunohost/operations /var/log/yunohost/categories/operation + fi + # Make sure conf files why may be created by apps are owned and writable only by root find /etc/systemd/system/*.service -type f | xargs -r chown root:root find /etc/systemd/system/*.service -type f | xargs -r chmod 0644 - if ls -l /etc/php/*/fpm/pool.d/*.conf; then + if ls -l /etc/php/*/fpm/pool.d/*.conf 2>/dev/null; then chown root:root /etc/php/*/fpm/pool.d/*.conf chmod 644 /etc/php/*/fpm/pool.d/*.conf fi - # Certs - # We do this with find because there could be a lot of them... - chown -R root:ssl-cert /etc/yunohost/certs - chmod 750 /etc/yunohost/certs - find /etc/yunohost/certs/ -type f -exec chmod 640 {} \; - find /etc/yunohost/certs/ -type d -exec chmod 750 {} \; - find /etc/cron.*/yunohost-* -type f -exec chmod 755 {} \; find /etc/cron.d/yunohost-* -type f -exec chmod 644 {} \; find /etc/cron.*/yunohost-* -type f -exec chown root:root {} \; - setfacl -m g:all_users:--- /var/www - setfacl -m g:all_users:--- /var/log/nginx - setfacl -m g:all_users:--- /etc/yunohost - setfacl -m g:all_users:--- /etc/ssowat for USER in $(yunohost user list --quiet --output-as json | jq -r '.users | .[] | .username'); do [ ! -e "/home/$USER" ] || setfacl -m g:all_users:--- /home/$USER done - # Domain settings - mkdir -p /etc/yunohost/domains - # Misc configuration / state files chown root:root $(ls /etc/yunohost/{*.yml,*.yaml,*.json,mysql,psql} 2> /dev/null | grep -vw mdns.yml) chmod 600 $(ls /etc/yunohost/{*.yml,*.yaml,*.json,mysql,psql} 2> /dev/null) - # Apps folder, custom hooks folder - [[ ! -e /etc/yunohost/hooks.d ]] || (chown root /etc/yunohost/hooks.d && chmod 700 /etc/yunohost/hooks.d) - [[ ! -e /etc/yunohost/apps ]] || (chown root /etc/yunohost/apps && chmod 700 /etc/yunohost/apps) - [[ ! -e /etc/yunohost/domains ]] || (chown root /etc/yunohost/domains && chmod 700 /etc/yunohost/domains) - # Create ssh.app and sftp.app groups if they don't exist yet grep -q '^ssh.app:' /etc/group || groupadd ssh.app grep -q '^sftp.app:' /etc/group || groupadd sftp.app @@ -238,6 +347,7 @@ do_post_regen() { systemctl restart ntp } fi + [[ ! "$regen_conf_files" =~ "nftables.service.d/ynh-override.conf" ]] || systemctl daemon-reload [[ ! "$regen_conf_files" =~ "login.conf.d/ynh-override.conf" ]] || { systemctl daemon-reload @@ -245,6 +355,7 @@ do_post_regen() { } [[ ! "$regen_conf_files" =~ "yunohost-firewall.service" ]] || systemctl daemon-reload [[ ! "$regen_conf_files" =~ "yunohost-api.service" ]] || systemctl daemon-reload + [[ ! "$regen_conf_files" =~ "yunohost-portal-api.service" ]] || systemctl daemon-reload if [[ "$regen_conf_files" =~ "yunoprompt.service" ]]; then systemctl daemon-reload @@ -257,6 +368,9 @@ do_post_regen() { systemctl $action proc-hidepid --quiet --now fi + systemctl enable yunohost-portal-api.service --quiet + systemctl is-active yunohost-portal-api --quiet || systemctl start yunohost-portal-api.service + # Change dpkg vendor # see https://wiki.debian.org/Derivatives/Guidelines#Vendor if readlink -f /etc/dpkg/origins/default | grep -q debian; then diff --git a/hooks/conf_regen/03-ssh b/hooks/conf_regen/03-ssh index 0206bc1570..7473897279 100755 --- a/hooks/conf_regen/03-ssh +++ b/hooks/conf_regen/03-ssh @@ -9,17 +9,16 @@ do_pre_regen() { cd /usr/share/yunohost/conf/ssh + # Support different strategy for security configurations + export compatibility="$(jq -r '.ssh_compatibility' <<< "$YNH_SETTINGS")" + export port="$(jq -r '.ssh_port' <<< "$YNH_SETTINGS")" + export password_authentication="$(jq -r '.ssh_password_authentication' <<< "$YNH_SETTINGS" | int_to_bool)" + export ssh_keys=$(ls /etc/ssh/ssh_host_{ed25519,rsa,ecdsa}_key 2>/dev/null || true) + # do not listen to IPv6 if unavailable [[ -f /proc/net/if_inet6 ]] && ipv6_enabled=true || ipv6_enabled=false - - ssh_keys=$(ls /etc/ssh/ssh_host_{ed25519,rsa,ecdsa}_key 2> /dev/null || true) - - # Support different strategy for security configurations - export compatibility="$(yunohost settings get 'security.ssh.ssh_compatibility')" - export port="$(yunohost settings get 'security.ssh.ssh_port')" - export password_authentication="$(yunohost settings get 'security.ssh.ssh_password_authentication' | int_to_bool)" - export ssh_keys export ipv6_enabled + ynh_render_template "sshd_config" "${pending_dir}/etc/ssh/sshd_config" } diff --git a/hooks/conf_regen/06-slapd b/hooks/conf_regen/06-slapd index 82c6f515c4..e7be7ba10a 100755 --- a/hooks/conf_regen/06-slapd +++ b/hooks/conf_regen/06-slapd @@ -8,10 +8,6 @@ config="/usr/share/yunohost/conf/slapd/config.ldif" db_init="/usr/share/yunohost/conf/slapd/db_init.ldif" do_init_regen() { - if [[ $EUID -ne 0 ]]; then - echo "You must be root to run this script" 1>&2 - exit 1 - fi do_pre_regen "" diff --git a/hooks/conf_regen/10-apt b/hooks/conf_regen/10-apt index 8fb85afd51..caa76537a9 100755 --- a/hooks/conf_regen/10-apt +++ b/hooks/conf_regen/10-apt @@ -2,7 +2,7 @@ set -e -readonly YNH_DEFAULT_PHP_VERSION=7.4 +readonly YNH_DEFAULT_PHP_VERSION=8.2 do_pre_regen() { pending_dir=$1 @@ -11,7 +11,7 @@ do_pre_regen() { # Add sury mkdir -p ${pending_dir}/etc/apt/sources.list.d/ - echo "deb https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list" + echo "deb [signed-by=/etc/apt/trusted.gpg.d/extra_php_version.gpg] https://packages.sury.org/php/ $(lsb_release --codename --short) main" > "${pending_dir}/etc/apt/sources.list.d/extra_php_version.list" # Ban some packages from sury echo " @@ -23,19 +23,33 @@ Pin-Priority: 500" >> "${pending_dir}/etc/apt/preferences.d/extra_php_version" for package in $packages_to_refuse_from_sury; do echo " Package: $package -Pin: origin \"packages.sury.org\" +Pin: origin \"packages.sury.org\" Pin-Priority: -1" >> "${pending_dir}/etc/apt/preferences.d/extra_php_version" done + # Add yarn + echo "deb [signed-by=/etc/apt/trusted.gpg.d/yarn.gpg] https://dl.yarnpkg.com/debian/ stable main" > "${pending_dir}/etc/apt/sources.list.d/yarn.list" + + # Ban everything from Yarn except Yarn + echo " +Package: * +Pin: origin \"dl.yarnpkg.com\" +Pin-Priority: -1 + +Package: yarn +Pin: origin \"dl.yarnpkg.com\" +Pin-Priority: 500" >>"${pending_dir}/etc/apt/preferences.d/yarn" + + # Ban apache2, bind9 echo " # PLEASE READ THIS WARNING AND DON'T EDIT THIS FILE -# You are probably reading this file because you tried to install apache2 or +# You are probably reading this file because you tried to install apache2 or # bind9. These 2 packages conflict with YunoHost. # Installing apache2 will break nginx and break the entire YunoHost ecosystem -# on your server, therefore don't remove those lines! +# on your server, therefore don't remove those lines! # You have been warned. @@ -71,6 +85,12 @@ do_post_regen() { wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor > "/etc/apt/trusted.gpg.d/extra_php_version.gpg" fi + # Similar to Sury + if [[ ! -s /etc/apt/trusted.gpg.d/yarn.gpg ]] + then + wget --timeout 900 --quiet "https://dl.yarnpkg.com/debian/pubkey.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/yarn.gpg" + fi + # Make sure php7.4 is the default version when using php in cli if test -e /usr/bin/php$YNH_DEFAULT_PHP_VERSION; then update-alternatives --set php /usr/bin/php$YNH_DEFAULT_PHP_VERSION diff --git a/hooks/conf_regen/12-metronome b/hooks/conf_regen/12-metronome deleted file mode 100755 index 7ccae110cf..0000000000 --- a/hooks/conf_regen/12-metronome +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -set -e - -if ! dpkg --list | grep -q 'ii *metronome '; then - echo 'metronome is not installed, skipping' - exit 0 -fi - -do_pre_regen() { - pending_dir=$1 - - cd /usr/share/yunohost/conf/metronome - - # create directories for pending conf - metronome_dir="${pending_dir}/etc/metronome" - metronome_conf_dir="${metronome_dir}/conf.d" - mkdir -p "$metronome_conf_dir" - - # retrieve variables - main_domain=$(cat /etc/yunohost/current_host) - - # install main conf file - cat metronome.cfg.lua \ - | sed "s/{{ main_domain }}/${main_domain}/g" \ - > "${metronome_dir}/metronome.cfg.lua" - - # Trick such that old conf files are flagged as to remove - for domain in $YNH_DOMAINS; do - touch "${metronome_conf_dir}/${domain}.cfg.lua" - done - - # add domain conf files - domain_list="$(yunohost domain list --features xmpp --output-as json | jq -r ".domains[]")" - for domain in $domain_list; do - cat domain.tpl.cfg.lua \ - | sed "s/{{ domain }}/${domain}/g" \ - > "${metronome_conf_dir}/${domain}.cfg.lua" - done - - # remove old domain conf files - conf_files=$(ls -1 /etc/metronome/conf.d \ - | awk '/^[^\.]+\.[^\.]+.*\.cfg\.lua$/ { print $1 }') - for file in $conf_files; do - domain=${file%.cfg.lua} - [[ $YNH_DOMAINS =~ $domain ]] \ - || touch "${metronome_conf_dir}/${file}" - done -} - -do_post_regen() { - regen_conf_files=$1 - - # retrieve variables - main_domain=$(cat /etc/yunohost/current_host) - - # create metronome directories for domains - for domain in $YNH_MAIN_DOMAINS; do - mkdir -p "/var/lib/metronome/${domain//./%2e}/pep" - # http_upload directory must be writable by metronome and readable by nginx - mkdir -p "/var/xmpp-upload/${domain}/upload" - # sgid bit allows that file created in that dir will be owned by www-data - # despite the fact that metronome ain't in the www-data group - chmod g+s "/var/xmpp-upload/${domain}/upload" - done - - # fix some permissions - [ ! -e '/var/xmpp-upload' ] || chown -R metronome:www-data "/var/xmpp-upload/" - [ ! -e '/var/xmpp-upload' ] || chmod 750 "/var/xmpp-upload/" - - # metronome should be in ssl-cert group to let it access SSL certificates - usermod -aG ssl-cert metronome - chown -R metronome: /var/lib/metronome/ - chown -R metronome: /etc/metronome/conf.d/ - - if [[ -z "$(ls /etc/metronome/conf.d/*.cfg.lua 2> /dev/null)" ]]; then - if systemctl is-enabled metronome &> /dev/null; then - systemctl disable metronome --now 2> /dev/null - fi - else - if ! systemctl is-enabled metronome &> /dev/null; then - systemctl enable metronome --now 2> /dev/null - sleep 3 - fi - - [[ -z "$regen_conf_files" ]] \ - || systemctl restart metronome - fi -} - -do_$1_regen ${@:2} diff --git a/hooks/conf_regen/15-nginx b/hooks/conf_regen/15-nginx index 44aba55bec..4846d423f1 100755 --- a/hooks/conf_regen/15-nginx +++ b/hooks/conf_regen/15-nginx @@ -4,25 +4,20 @@ set -e . /usr/share/yunohost/helpers -do_init_regen() { - if [[ $EUID -ne 0 ]]; then - echo "You must be root to run this script" 1>&2 - exit 1 - fi - - cd /usr/share/yunohost/conf/nginx +do_base_regen() { - nginx_dir="/etc/nginx" + pending_dir=$1 + nginx_dir="${pending_dir}/etc/nginx" nginx_conf_dir="${nginx_dir}/conf.d" mkdir -p "$nginx_conf_dir" # install plain conf files - cp plain/* "$nginx_conf_dir" - - # probably run with init: just disable default site, restart NGINX and exit - rm -f "${nginx_dir}/sites-enabled/default" + cp acme-challenge.conf.inc "$nginx_conf_dir" + cp global.conf "$nginx_conf_dir" + cp ssowat.conf "$nginx_conf_dir" + cp yunohost_http_errors.conf.inc "$nginx_conf_dir" + cp yunohost_sso.conf.inc "$nginx_conf_dir" - export compatibility="intermediate" ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc" ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf" ynh_render_template "yunohost_admin.conf.inc" "${nginx_conf_dir}/yunohost_admin.conf.inc" @@ -30,6 +25,17 @@ do_init_regen() { mkdir -p $nginx_conf_dir/default.d/ cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/ +} + +do_init_regen() { + + cd /usr/share/yunohost/conf/nginx + + export compatibility="intermediate" + do_base_regen "" + + # probably run with init: just disable default site, restart NGINX and exit + rm -f "${nginx_dir}/sites-enabled/default" # Restart nginx if conf looks good, otherwise display error and exit unhappy nginx -t 2> /dev/null || { @@ -53,23 +59,20 @@ do_pre_regen() { nginx_conf_dir="${nginx_dir}/conf.d" mkdir -p "$nginx_conf_dir" - # install / update plain conf files - cp plain/* "$nginx_conf_dir" - # remove the panel overlay if this is specified in settings - panel_overlay=$(yunohost settings get 'misc.portal.ssowat_panel_overlay_enabled' | int_to_bool) - if [ "$panel_overlay" == "False" ]; then - echo "#" > "${nginx_conf_dir}/yunohost_panel.conf.inc" + export webadmin_allowlist_enabled="$(jq -r '.webadmin_allowlist_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" + if [ "$webadmin_allowlist_enabled" == "True" ]; then + export webadmin_allowlist="$(jq -r '.webadmin_allowlist' <<< "$YNH_SETTINGS" | sed 's/^null$//g')" fi - # retrieve variables - main_domain=$(cat /etc/yunohost/current_host) - # Support different strategy for security configurations - export redirect_to_https="$(yunohost settings get 'security.nginx.nginx_redirect_to_https' | int_to_bool)" - export compatibility="$(yunohost settings get 'security.nginx.nginx_compatibility')" - export experimental="$(yunohost settings get 'security.experimental.security_experimental_enabled' | int_to_bool)" - export tls_passthrough_enabled="$(yunohost settings get 'misc.tls_passthrough.tls_passthrough_enabled' | int_to_bool)" - export tls_passthrough_list="$(yunohost settings get 'misc.tls_passthrough.tls_passthrough_list')" + export redirect_to_https="$(jq -r '.nginx_redirect_to_https' <<< "$YNH_SETTINGS" | int_to_bool)" + export compatibility="$(jq -r '.nginx_compatibility' <<< "$YNH_SETTINGS" | int_to_bool)" + export experimental="$(jq -r '.security_experimental_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" + export tls_passthrough_enabled="$(jq -r '.tls_passthrough_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" + export tls_passthrough_list="$(jq -r '.tls_passthrough_list' <<< "$YNH_SETTINGS" | int_to_bool)" + + do_base_regen "${pending_dir}" + local tls_passthrough_module="${pending_dir}/etc/nginx/modules-enabled/tls_passthrough.conf" mkdir -p "${pending_dir}/etc/nginx/modules-enabled/" @@ -86,8 +89,6 @@ do_pre_regen() { touch "${tls_passthrough_module}" fi - ynh_render_template "security.conf.inc" "${nginx_conf_dir}/security.conf.inc" - # "Touch" every known .conf file for every domain, # meaning it should be removed by the regen conf # - For real 'existing' domains, this file will be overwritten with an actual conf right after using ynh_render_template @@ -98,7 +99,6 @@ do_pre_regen() { # add domain conf files cert_status=$(yunohost domain cert status --json) - xmpp_domain_list="$(yunohost domain list --features xmpp --output-as json | jq -r ".domains[]")" mail_domain_list="$(yunohost domain list --features mail_in mail_out --output-as json | jq -r ".domains[]")" for domain in $YNH_DOMAINS; do domain_conf_dir="${nginx_conf_dir}/${domain}.d" @@ -111,11 +111,6 @@ do_pre_regen() { export domain_cert_ca=$(echo $cert_status \ | jq ".certificates.\"$domain\".CA_type" \ | tr -d '"') - if echo "$xmpp_domain_list" | grep -q "^$domain$"; then - export xmpp_enabled="True" - else - export xmpp_enabled="False" - fi if echo "$mail_domain_list" | grep -q "^$domain$"; then export mail_enabled="True" else @@ -131,15 +126,8 @@ do_pre_regen() { done - export webadmin_allowlist_enabled=$(yunohost settings get security.webadmin.webadmin_allowlist_enabled | int_to_bool) - if [ "$webadmin_allowlist_enabled" == "True" ]; then - export webadmin_allowlist=$(yunohost settings get security.webadmin.webadmin_allowlist) - fi - ynh_render_template "yunohost_admin.conf.inc" "${nginx_conf_dir}/yunohost_admin.conf.inc" - ynh_render_template "yunohost_api.conf.inc" "${nginx_conf_dir}/yunohost_api.conf.inc" - ynh_render_template "yunohost_admin.conf" "${nginx_conf_dir}/yunohost_admin.conf" - mkdir -p $nginx_conf_dir/default.d/ - cp "redirect_to_admin.conf" $nginx_conf_dir/default.d/ + # Legacy file to remove, but we can't really remove it because it may be included by app confs... + echo "# The old yunohost panel/tile/button doesn't exists anymore" > "$nginx_conf_dir"/yunohost_panel.conf.inc # remove old mail-autoconfig files autoconfig_files=$(ls -1 /var/www/.well-known/*/autoconfig/mail/config-v1.1.xml 2> /dev/null || true) @@ -157,6 +145,9 @@ do_pre_regen() { do_post_regen() { regen_conf_files=$1 + # Make sure fastcgi / PHP uses the YNH_USER auth header instead of $remote_user from the Authorization header + sed -i 's/$remote_user;/$http_ynh_user if_not_empty;/g' /etc/nginx/fastcgi_params + if ls -l /etc/nginx/conf.d/*.d/*.conf; then chown root:root /etc/nginx/conf.d/*.d/*.conf chmod 644 /etc/nginx/conf.d/*.d/*.conf diff --git a/hooks/conf_regen/19-postfix b/hooks/conf_regen/19-postfix index b849348cd8..d5e117d2f3 100755 --- a/hooks/conf_regen/19-postfix +++ b/hooks/conf_regen/19-postfix @@ -22,19 +22,19 @@ do_pre_regen() { main_domain=$(cat /etc/yunohost/current_host) # Support different strategy for security configurations - export compatibility="$(yunohost settings get 'security.postfix.postfix_compatibility')" + export compatibility="$(jq -r '.postfix_compatibility' <<< "$YNH_SETTINGS")" # Add possibility to specify a relay # Could be useful with some isp with no 25 port open or more complex setup export relay_port="" export relay_user="" export relay_host="" - export relay_enabled="$(yunohost settings get 'email.smtp.smtp_relay_enabled' | int_to_bool)" + export relay_enabled="$(jq -r '.smtp_relay_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" if [ "${relay_enabled}" == "True" ]; then - relay_host="$(yunohost settings get 'email.smtp.smtp_relay_host')" - relay_port="$(yunohost settings get 'email.smtp.smtp_relay_port')" - relay_user="$(yunohost settings get 'email.smtp.smtp_relay_user')" - relay_password="$(yunohost settings get 'email.smtp.smtp_relay_password')" + relay_host="$(jq -r '.smtp_relay_host' <<< "$YNH_SETTINGS")" + relay_port="$(jq -r '.smtp_relay_port' <<< "$YNH_SETTINGS")" + relay_user="$(jq -r '.smtp_relay_user' <<< "$YNH_SETTINGS")" + relay_password="$(jq -r '.smtp_relay_password' <<< "$YNH_SETTINGS")" # Avoid to display "Relay account paswword" to other users touch ${postfix_dir}/sasl_passwd @@ -69,7 +69,7 @@ do_pre_regen() { > "${default_dir}/postsrsd" # adapt it for IPv4-only hosts - ipv6="$(yunohost settings get 'email.smtp.smtp_allow_ipv6' | int_to_bool)" + ipv6="$(jq -r '.smtp_allow_ipv6' <<< "$YNH_SETTINGS" | int_to_bool)" if [ "$ipv6" == "False" ] || [ ! -f /proc/net/if_inet6 ]; then sed -i \ 's/ \[::ffff:127.0.0.0\]\/104 \[::1\]\/128//g' \ diff --git a/hooks/conf_regen/25-dovecot b/hooks/conf_regen/25-dovecot index 1c582a14ea..f45c6da9b7 100755 --- a/hooks/conf_regen/25-dovecot +++ b/hooks/conf_regen/25-dovecot @@ -16,7 +16,7 @@ do_pre_regen() { cp dovecot-ldap.conf "${dovecot_dir}/dovecot-ldap.conf" cp dovecot.sieve "${dovecot_dir}/global_script/dovecot.sieve" - export pop3_enabled="$(yunohost settings get 'email.pop3.pop3_enabled' | int_to_bool)" + export pop3_enabled="$(jq -r '.pop3_enabled' <<< "$YNH_SETTINGS" | int_to_bool)" export main_domain=$(cat /etc/yunohost/current_host) export domain_list="$(yunohost domain list --features mail_in mail_out --output-as json | jq -r ".domains[]" | tr '\n' ' ')" @@ -42,7 +42,7 @@ do_post_regen() { # create vmail user id vmail > /dev/null 2>&1 \ - || adduser --system --ingroup mail --uid 500 vmail --home /var/vmail --no-create-home + || { mkdir -p /var/vmail; adduser --system --ingroup mail --uid 500 vmail --home /var/vmail --no-create-home; } # Delete legacy home for vmail that existed in the past but was empty, poluting /home/ [ ! -e /home/vmail ] || rmdir --ignore-fail-on-non-empty /home/vmail diff --git a/hooks/conf_regen/30-opendkim b/hooks/conf_regen/30-opendkim new file mode 100755 index 0000000000..30a8927f41 --- /dev/null +++ b/hooks/conf_regen/30-opendkim @@ -0,0 +1,43 @@ +#!/bin/bash + +set -e + +do_pre_regen() { + pending_dir=$1 + + cd /usr/share/yunohost/conf/opendkim + + install -D -m 644 opendkim.conf "${pending_dir}/etc/opendkim.conf" +} + +do_post_regen() { + mkdir -p /etc/dkim + + # Create / empty those files because we're force-regenerating them + echo "" > /etc/dkim/keytable + echo "" > /etc/dkim/signingtable + + # create DKIM key for domains + domain_list="$(yunohost domain list --features mail_in mail_out --output-as json | jq -r ".domains[]" | tr '\n' ' ')" + for domain in $domain_list; do + domain_key="/etc/dkim/${domain}.mail.key" + [ ! -f "$domain_key" ] && { + # We use a 1024 bit size because nsupdate doesn't seem to be able to + # handle 2048... + opendkim-genkey --domain="$domain" \ + --selector=mail --directory=/etc/dkim -b 1024 + mv /etc/dkim/mail.private "$domain_key" + mv /etc/dkim/mail.txt "/etc/dkim/${domain}.mail.txt" + } + + echo "mail._domainkey.${domain} ${domain}:mail:${domain_key}" >> /etc/dkim/keytable + echo "*@$domain mail._domainkey.${domain}" >> /etc/dkim/signingtable + done + + chown -R opendkim /etc/dkim/ + chmod 700 /etc/dkim/ + + systemctl restart opendkim +} + +do_$1_regen ${@:2} diff --git a/hooks/conf_regen/31-rspamd b/hooks/conf_regen/31-rspamd deleted file mode 100755 index 25f55cf3d6..0000000000 --- a/hooks/conf_regen/31-rspamd +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -set -e - -do_pre_regen() { - pending_dir=$1 - - cd /usr/share/yunohost/conf/rspamd - - install -D -m 644 metrics.local.conf \ - "${pending_dir}/etc/rspamd/local.d/metrics.conf" - install -D -m 644 dkim_signing.conf \ - "${pending_dir}/etc/rspamd/local.d/dkim_signing.conf" - install -D -m 644 rspamd.sieve \ - "${pending_dir}/etc/dovecot/global_script/rspamd.sieve" - install -D -m 644 redis.conf \ - "${pending_dir}/etc/rspamd/local.d/redis.conf" -} - -do_post_regen() { - - ## - ## DKIM key generation - ## - - # create DKIM directory with proper permission - mkdir -p /etc/dkim - chown _rspamd /etc/dkim - - # create DKIM key for domains - domain_list="$(yunohost domain list --features mail_in mail_out --output-as json | jq -r ".domains[]" | tr '\n' ' ')" - for domain in $domain_list; do - domain_key="/etc/dkim/${domain}.mail.key" - [ ! -f "$domain_key" ] && { - # We use a 1024 bit size because nsupdate doesn't seem to be able to - # handle 2048... - opendkim-genkey --domain="$domain" \ - --selector=mail --directory=/etc/dkim -b 1024 - mv /etc/dkim/mail.private "$domain_key" - mv /etc/dkim/mail.txt "/etc/dkim/${domain}.mail.txt" - } - done - - # fix DKIM keys permissions - chown _rspamd /etc/dkim/*.mail.key - chmod 400 /etc/dkim/*.mail.key - - [ ! -e /var/log/rspamd ] || chown -R _rspamd:_rspamd /var/log/rspamd - - regen_conf_files=$1 - [ -z "$regen_conf_files" ] && exit 0 - - # compile sieve script - [[ "$regen_conf_files" =~ rspamd\.sieve ]] && { - sievec /etc/dovecot/global_script/rspamd.sieve - chown -R vmail:mail /etc/dovecot/global_script - systemctl restart dovecot - } - - # Restart rspamd due to the upgrade - # https://rspamd.com/announce/2016/08/01/rspamd-1.3.1.html - systemctl -q restart rspamd.service -} - -do_$1_regen ${@:2} diff --git a/hooks/conf_regen/36-redis b/hooks/conf_regen/36-redis deleted file mode 100755 index ac486f3731..0000000000 --- a/hooks/conf_regen/36-redis +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -do_pre_regen() { - : -} - -do_post_regen() { - # Enforce these damn permissions because for some reason in some weird cases - # they are spontaneously replaced by root:root -_- - chown -R redis:adm /var/log/redis -} - -do_$1_regen ${@:2} diff --git a/hooks/conf_regen/52-fail2ban b/hooks/conf_regen/52-fail2ban index baa0bc28f9..118a33eaf7 100755 --- a/hooks/conf_regen/52-fail2ban +++ b/hooks/conf_regen/52-fail2ban @@ -14,10 +14,11 @@ do_pre_regen() { mkdir -p "${fail2ban_dir}/jail.d" cp yunohost.conf "${fail2ban_dir}/filter.d/yunohost.conf" + cp yunohost-portal.conf "${fail2ban_dir}/filter.d/yunohost-portal.conf" cp postfix-sasl.conf "${fail2ban_dir}/filter.d/postfix-sasl.conf" cp jail.conf "${fail2ban_dir}/jail.conf" - export ssh_port="$(yunohost settings get 'security.ssh.ssh_port')" + export ssh_port="$(jq -r '.ssh_port' <<< "$YNH_SETTINGS")" ynh_render_template "yunohost-jails.conf" "${fail2ban_dir}/jail.d/yunohost-jails.conf" } diff --git a/hooks/restore/20-conf_ynh_settings b/hooks/restore/20-conf_ynh_settings index aba2b7a466..f962a47efb 100644 --- a/hooks/restore/20-conf_ynh_settings +++ b/hooks/restore/20-conf_ynh_settings @@ -2,6 +2,7 @@ backup_dir="$1/conf/ynh" cp -a "${backup_dir}/current_host" /etc/yunohost/current_host cp -a "${backup_dir}/firewall.yml" /etc/yunohost/firewall.yml +[ ! -d "${backup_dir}/portal"] || cp -a "${backup_dir]}/portal" /etc/yunohost/portal [ ! -d "${backup_dir}/domains" ] || cp -a "${backup_dir}/domains" /etc/yunohost/domains [ ! -e "${backup_dir}/settings.yml" ] || cp -a "${backup_dir}/settings.yml" "/etc/yunohost/settings.yml" [ ! -d "${backup_dir}/dyndns" ] || cp -raT "${backup_dir}/dyndns" "/etc/yunohost/dyndns" diff --git a/hooks/restore/27-data_xmpp b/hooks/restore/27-data_xmpp deleted file mode 100644 index 02a4c6703a..0000000000 --- a/hooks/restore/27-data_xmpp +++ /dev/null @@ -1,4 +0,0 @@ -backup_dir="$1/data/xmpp" - -cp -a $backup_dir/var_lib_metronome/. /var/lib/metronome -cp -a $backup_dir/var_xmpp-upload/. /var/xmpp-upload diff --git a/locales/en.json b/locales/en.json index 0ced1ab041..8a934df662 100644 --- a/locales/en.json +++ b/locales/en.json @@ -16,8 +16,6 @@ "app_arch_not_supported": "This app can only be installed on architectures {required} but your server architecture is {current}", "app_argument_choice_invalid": "Pick a valid value for argument '{name}': '{value}' is not among the available choices ({choices})", "app_argument_invalid": "Pick a valid value for the argument '{name}': {error}", - "app_argument_password_no_default": "Error while parsing password argument '{name}': password argument can't have a default value for security reasons", - "app_argument_required": "Argument '{name}' is required", "app_change_url_failed": "Could not change the url for {app}: {error}", "app_change_url_identical_domains": "The old and new domain/url_path are identical ('{domain}{path}'), nothing to do.", "app_change_url_no_script": "The app '{app_name}' doesn't support URL modification yet. Maybe you should upgrade it.", @@ -76,7 +74,6 @@ "app_yunohost_version_not_supported": "This app requires YunoHost >= {required} but current installed version is {current}", "apps_already_up_to_date": "All apps are already up-to-date", "apps_catalog_failed_to_download": "Unable to download the {apps_catalog} app catalog: {error}", - "apps_catalog_init_success": "App catalog system initialized!", "apps_catalog_obsolete_cache": "The app catalog cache is empty or obsolete.", "apps_catalog_update_success": "The application catalog has been updated!", "apps_catalog_updating": "Updating application catalog…", @@ -94,7 +91,7 @@ "ask_new_domain": "New domain", "ask_new_path": "New path", "ask_password": "Password", - "ask_user_domain": "Domain to use for the user's email address and XMPP account", + "ask_user_domain": "Domain to use for the user's email address", "backup_abstract_method": "This backup method has yet to be implemented", "backup_actually_backuping": "Creating a backup archive from the collected files…", "backup_app_failed": "Could not back up {app}", @@ -162,7 +159,6 @@ "certmanager_no_cert_file": "Could not read the certificate file for the domain {domain} (file: {file})", "certmanager_self_ca_conf_file_not_found": "Could not find configuration file for self-signing authority (file: {file})", "certmanager_unable_to_parse_self_CA_name": "Could not parse name of self-signing authority (file: {file})", - "certmanager_warning_subdomain_dns_record": "Subdomain '{subdomain}' does not resolve to the same IP address as '{domain}'. Some features will not be available until you fix this and regenerate the certificate.", "config_action_disabled": "Could not run action '{action}' since it is disabled, make sure to meet its constraints. help: {help}", "config_action_failed": "Failed to run action '{action}': {error}", "config_apply_failed": "Applying the new configuration failed: {error}", @@ -171,16 +167,12 @@ "config_forbidden_readonly_type": "The type '{type}' can't be set as readonly, use another type to render this value (relevant arg id: '{id}').", "config_no_panel": "No config panel found.", "config_unknown_filter_key": "The filter key '{filter_key}' is incorrect.", - "config_validate_color": "Should be a valid RGB hexadecimal color", - "config_validate_date": "Should be a valid date like in the format YYYY-MM-DD", - "config_validate_email": "Should be a valid email", - "config_validate_time": "Should be a valid time like HH:MM", - "config_validate_url": "Should be a valid web URL", "confirm_app_install_danger": "DANGER! This app is known to be still experimental (if not explicitly not working)! You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system… If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_thirdparty": "DANGER! This app is not part of YunoHost's app catalog. Installing third-party apps may compromise the integrity and security of your system. You should probably NOT install it unless you know what you are doing. NO SUPPORT will be provided if this app doesn't work or breaks your system… If you are willing to take that risk anyway, type '{answers}'", "confirm_app_install_warning": "Warning: This app may work, but is not well-integrated into YunoHost. Some features such as single sign-on and backup/restore might not be available. Install anyway? [{answers}] ", "confirm_app_insufficient_ram": "DANGER! This app requires {required} RAM to install/upgrade but only {current} is available right now. Even if this app could run, its installation/upgrade process requires a large amount of RAM so your server may freeze and fail miserably. If you are willing to take that risk anyway, type '{answers}'", "confirm_notifications_read": "WARNING: You should check the app notifications above before continuing, there might be important stuff to know. [{answers}]", + "confirm_tos_acknowledgement": "I have read and understand the Terms of Services [{answers}]", "custom_app_url_required": "You must provide a URL to upgrade your custom app {app}", "danger": "Danger:", "diagnosis_apps_allgood": "All installed apps respect basic packaging practices", @@ -334,14 +326,12 @@ "diagnosis_swap_ok": "The system has {total} of swap!", "diagnosis_swap_tip": "Please be careful and aware that if the server is hosting swap on an SD card or SSD storage, it may drastically reduce the life expectancy of the device.", "diagnosis_unknown_categories": "The following categories are unknown: {categories}", - "diagnosis_using_stable_codename": "apt (the system's package manager) is currently configured to install packages from codename 'stable', instead of the codename of the current Debian version (bullseye).", - "diagnosis_using_stable_codename_details": "This is usually caused by incorrect configuration from your hosting provider. This is dangerous, because as soon as the next Debian version becomes the new 'stable', apt will want to upgrade all system packages without going through a proper migration procedure. It is recommended to fix this by editing the apt source for base Debian repository, and replace the stable keyword by bullseye. The corresponding configuration file should be /etc/apt/sources.list, or a file in /etc/apt/sources.list.d/.", + "diagnosis_using_stable_codename": "apt (the system's package manager) is currently configured to install packages from codename 'stable', instead of the codename of the current Debian version (bookworm).", + "diagnosis_using_stable_codename_details": "This is usually caused by incorrect configuration from your hosting provider. This is dangerous, because as soon as the next Debian version becomes the new 'stable', apt will want to upgrade all system packages without going through a proper migration procedure. It is recommended to fix this by editing the apt source for base Debian repository, and replace the stable keyword by bookworm. The corresponding configuration file should be /etc/apt/sources.list, or a file in /etc/apt/sources.list.d/.", "diagnosis_using_yunohost_testing": "apt (the system's package manager) is currently configured to install any 'testing' upgrade for YunoHost core.", "diagnosis_using_yunohost_testing_details": "This is probably OK if you know what you are doing, but pay attention to the release notes before installing YunoHost upgrades! If you want to disable 'testing' upgrades, you should remove the testing keyword from /etc/apt/sources.list.d/yunohost.list.", "disk_space_not_sufficient_install": "There is not enough disk space left to install this application", "disk_space_not_sufficient_update": "There is not enough disk space left to update this application", - "domain_cannot_add_muc_upload": "You cannot add domains starting with 'muc.'. This kind of name is reserved for the XMPP multi-users chat feature integrated into YunoHost.", - "domain_cannot_add_xmpp_upload": "You cannot add domains starting with 'xmpp-upload.'. This kind of name is reserved for the XMPP upload feature integrated into YunoHost.", "domain_cannot_remove_main": "You cannot remove '{domain}' since it's the main domain, you first need to set another domain as the main domain using 'yunohost domain main-domain -n '; here is the list of candidate domains: {other_domains}", "domain_cannot_remove_main_add_new_one": "You cannot remove '{domain}' since it's the main domain and your only domain, you need to first add another domain using 'yunohost domain add ', then set is as the main domain using 'yunohost domain main-domain -n ' and then you can remove the domain '{domain}' using 'yunohost domain remove {domain}'.", "domain_cert_gen_failed": "Could not generate certificate", @@ -357,6 +347,7 @@ "domain_config_auth_token": "Authentication token", "domain_config_cert_install": "Install Let's Encrypt certificate", "domain_config_cert_issuer": "Certification authority", + "domain_config_cert_name": "Certificate", "domain_config_cert_no_checks": "Ignore diagnosis checks", "domain_config_cert_renew": "Renew Let's Encrypt certificate", "domain_config_cert_renew_help": "Certificate will be automatically renewed during the last 15 days of validity. You can manually renew it if you want to. (Not recommended).", @@ -367,12 +358,31 @@ "domain_config_cert_summary_ok": "Okay, current certificate looks good!", "domain_config_cert_summary_selfsigned": "WARNING: Current certificate is self-signed. Browsers will display a spooky warning to new visitors!", "domain_config_cert_validity": "Validity", + "domain_config_custom_css": "Custom CSS stylesheet", + "domain_config_custom_css_help": "This is for advanced admins willing to customize the appearance of the portal", "domain_config_default_app": "Default app", "domain_config_default_app_help": "People will automatically be redirected to this app when opening this domain. If no app is specified, people are redirected to the user portal login form.", + "domain_config_dns_name": "DNS", + "domain_config_enable_public_apps_page": "Show the list of public apps to visitors", + "domain_config_enable_public_apps_page_help": "Visitors will see a 'public apps' page when ending up on the portal instead of just the login form.", + "domain_config_feature_name": "Features", "domain_config_mail_in": "Incoming emails", "domain_config_mail_out": "Outgoing emails", - "domain_config_xmpp": "Instant messaging (XMPP)", - "domain_config_xmpp_help": "NB: some XMPP features will require that you update your DNS records and regenerate your Lets Encrypt certificate to be enabled", + "domain_config_portal_logo": "Custom logo", + "domain_config_portal_logo_help": "Accept .svg, .png and .jpeg. Prefer a monochrome .svg with fill: currentColor so that the logo adapts to the themes.", + "domain_config_portal_name": "Portal customization", + "domain_config_portal_public_intro": "Custom public intro", + "domain_config_portal_public_intro_help": "You can use HTML, basic styles will be applied to generic elements.", + "domain_config_portal_theme": "Default color theme", + "domain_config_portal_theme_help": "Users are allowed to choose another one in their settings.", + "domain_config_portal_tile_theme": "App tiles display theme", + "domain_config_portal_title": "Custom title", + "domain_config_portal_user_intro": "Custom user intro", + "domain_config_portal_user_intro_help": "You can use HTML, basic styles will be applied to generic elements.", + "domain_config_search_engine": "Search engine URL", + "domain_config_search_engine_help": "This is an optional feature, allowing to display a search bar in the portal (for example if you like to use your YunoHost portal as your browser's home page). This should be an URL with an empty query string such as `https://duckduckgo.com/?q=`, with `q=` as duckduckgo's empty query parameter", + "domain_config_search_engine_name": "Search engine name", + "domain_config_show_other_domains_apps": "Show other domain's apps", "domain_created": "Domain created", "domain_creation_failed": "Unable to create domain {domain}: {error}", "domain_deleted": "Domain deleted", @@ -393,6 +403,7 @@ "domain_dns_registrar_managed_in_parent_domain": "This domain is a subdomain of {parent_domain_link}. DNS registrar configuration should be managed in {parent_domain}'s configuration panel.", "domain_dns_registrar_not_supported": "YunoHost could not automatically detect the registrar handling this domain. You should manually configure your DNS records following the documentation at https://yunohost.org/dns.", "domain_dns_registrar_supported": "YunoHost automatically detected that this domain is handled by the registrar **{registrar}**. If you want, YunoHost will automatically configure this DNS zone, if you provide it with the appropriate API credentials. You can find documentation on how to obtain your API credentials on this page: https://yunohost.org/registar_api_{registrar}. (You can also manually configure your DNS records following the documentation at https://yunohost.org/dns )", + "domain_dns_registrar_use_auto": "Use automatic DNS feature", "domain_dns_registrar_yunohost": "This domain is a nohost.me / nohost.st / ynh.fr and its DNS configuration is therefore automatically handled by YunoHost without any further configuration. (see the 'yunohost dyndns update' command)", "domain_dyndns_already_subscribed": "You have already subscribed to a DynDNS domain", "domain_exists": "The domain already exists", @@ -438,30 +449,40 @@ "global_settings_setting_admin_strength_help": "These requirements are only enforced when initializing or changing the password", "global_settings_setting_backup_compress_tar_archives": "Compress backups", "global_settings_setting_backup_compress_tar_archives_help": "When creating new backups, compress the archives (.tar.gz) instead of uncompressed archives (.tar). N.B. : enabling this option means create lighter backup archives, but the initial backup procedure will be significantly longer and heavy on CPU.", + "global_settings_setting_backup_name": "Backup", "global_settings_setting_dns_exposure": "IP versions to consider for DNS configuration and diagnosis", "global_settings_setting_dns_exposure_help": "NB: This only affects the recommended DNS configuration and diagnosis checks. This does not affect system configurations.", + "global_settings_setting_email_name": "Email", + "global_settings_setting_experimental_name": "Experimental", + "global_settings_setting_misc_name": "Other", + "global_settings_setting_network_name": "Network", "global_settings_setting_nginx_compatibility": "NGINX Compatibility", "global_settings_setting_nginx_compatibility_help": "Compatibility vs. security tradeoff for the web server NGINX. Affects the ciphers (and other security-related aspects)", + "global_settings_setting_nginx_name": "NGINX (web server)", "global_settings_setting_nginx_redirect_to_https": "Force HTTPS", "global_settings_setting_nginx_redirect_to_https_help": "Redirect HTTP requests to HTTPs by default (DO NOT TURN OFF unless you really know what you're doing!)", + "global_settings_setting_password_name": "Passwords", "global_settings_setting_passwordless_sudo": "Allow admins to use 'sudo' without re-typing their passwords", "global_settings_setting_pop3_enabled": "Enable POP3", - "global_settings_setting_pop3_enabled_help": "Enable the POP3 protocol for the mail server", - "global_settings_setting_portal_theme": "Portal theme", - "global_settings_setting_portal_theme_help": "More info regarding creating custom portal themes at https://yunohost.org/theming", + "global_settings_setting_pop3_enabled_help": "Enable the POP3 protocol for the mail server. POP3 is an older protocol to access mailboxes from email clients and is more lightweight, but has less features than IMAP (enabled by default)", + "global_settings_setting_pop3_name": "POP3", "global_settings_setting_postfix_compatibility": "Postfix Compatibility", "global_settings_setting_postfix_compatibility_help": "Compatibility vs. security tradeoff for the Postfix server. Affects the ciphers (and other security-related aspects)", + "global_settings_setting_postfix_name": "Postfix (SMTP email server)", "global_settings_setting_root_access_explain": "On Linux systems, 'root' is the absolute admin. In YunoHost context, direct 'root' SSH login is by default disable - except from the local network of the server. Members of the 'admins' group can use the sudo command to act as root from the command line. However, it can be helpful to have a (robust) root password to debug the system if for some reason regular admins can not login anymore.", + "global_settings_setting_root_access_name": "Change root password", "global_settings_setting_root_password": "New root password", "global_settings_setting_root_password_confirm": "New root password (confirm)", "global_settings_setting_security_experimental_enabled": "Experimental security features", "global_settings_setting_security_experimental_enabled_help": "Enable experimental security features (don't enable this if you don't know what you're doing!)", + "global_settings_setting_security_name": "Security", "global_settings_setting_smtp_allow_ipv6": "Allow IPv6", "global_settings_setting_smtp_allow_ipv6_help": "Allow the use of IPv6 to receive and send mail", "global_settings_setting_smtp_backup_mx_domains": "Domains to act as secondary MX for", "global_settings_setting_smtp_backup_mx_domains_help": "Allow this server to act as a backup *secondary* MX domain for the listed domain. This means that if the main MX for the domain is not reachable (for example because of an outage), mails will still be sent to this server, which will keep them during a maximum of 20 days and try to relay them to the real destination once it goes back up. Several domains can be provided, separated by commas.", "global_settings_setting_smtp_backup_mx_emails_whitelisted": "SMTP backup MX emails whitelist", "global_settings_setting_smtp_backup_mx_emails_whitelisted_help": "When acting as a secondary MX, the exhaustive list of allowed recipient's email addresses must be provided (otherwise mails will be refused and discarded). Several entries can be provided, separated by commas.", + "global_settings_setting_smtp_name": "SMTP", "global_settings_setting_smtp_relay_enabled": "Enable SMTP relay", "global_settings_setting_smtp_relay_enabled_help": "Enable the SMTP relay to use in order to send mail instead of this yunohost instance. Useful if you are in one of this situation: your 25 port is blocked by your ISP or VPS provider, you have a residential IP listed on DUHL, you are not able to configure reverse DNS or this server is not directly exposed on the internet and you want use an other one to send mails.", "global_settings_setting_smtp_relay_host": "SMTP relay host", @@ -470,22 +491,24 @@ "global_settings_setting_smtp_relay_user": "SMTP relay user", "global_settings_setting_ssh_compatibility": "SSH Compatibility", "global_settings_setting_ssh_compatibility_help": "Compatibility vs. security tradeoff for the SSH server. Affects the ciphers (and other security-related aspects). See https://infosec.mozilla.org/guidelines/openssh for more info.", + "global_settings_setting_ssh_name": "SSH", "global_settings_setting_ssh_password_authentication": "Password authentication", "global_settings_setting_ssh_password_authentication_help": "Allow password authentication for SSH", "global_settings_setting_ssh_port": "SSH port", "global_settings_setting_ssh_port_help": "A port lower than 1024 is preferred to prevent usurpation attempts by non-administrator services on the remote machine. You should also avoid using a port already in use, such as 80 or 443.", - "global_settings_setting_ssowat_panel_overlay_enabled": "Enable the small 'YunoHost' portal shortcut square on apps", "global_settings_setting_tls_passthrough_enabled": "Enable TLS-passthrough / SNI-based forwarding", "global_settings_setting_tls_passthrough_enabled_help": "This is an advanced feature to reverse-proxy an entire domain to another machine *without* decrypting the traffic. Useful when you want to expose several machines behind the same IP but still allow each machine to handle the SSL termination.", "global_settings_setting_tls_passthrough_explain": "This feature is ADVANCED and EXPERIMENTAL and will trigger major changes in the nginx configuration of this server. Please DO NOT use it if you don't know what you are doing! In particular, you must be aware that fail2ban cannot be implemented on the proxied server (iptables cannot ban malicious traffic as all IP packets appear as coming from the front server). In addition, for now the proxied server's nginx configuration needs to manually tweaked to accept the `proxy_protocol`.", "global_settings_setting_tls_passthrough_list": "List of forwarding", "global_settings_setting_tls_passthrough_list_help": "Should be a list of DOMAIN;DESTINATION;PORT, such as domain.tld;192.168.1.42;443 or domain.tld;server.local;8123", + "global_settings_setting_tls_passthrough_name": "TLS-passthrough / SNI-based forwarding", "global_settings_setting_user_strength": "User password strength requirements", "global_settings_setting_user_strength_help": "These requirements are only enforced when initializing or changing the password", "global_settings_setting_webadmin_allowlist": "Webadmin IP allowlist", "global_settings_setting_webadmin_allowlist_enabled": "Enable Webadmin IP allowlist", "global_settings_setting_webadmin_allowlist_enabled_help": "Allow only some IPs to access the webadmin.", "global_settings_setting_webadmin_allowlist_help": "IP adresses allowed to access the webadmin. CIDR notation is allowed.", + "global_settings_setting_webadmin_name": "Webadmin", "good_practices_about_admin_password": "You are now about to define a new administration password. The password should be at least 8 characters long—though it is good practice to use a longer password (i.e. a passphrase) and/or to use a variation of characters (uppercase, lowercase, digits and special characters).", "good_practices_about_user_password": "You are now about to define a new user password. The password should be at least 8 characters long—though it is good practice to use a longer password (i.e. a passphrase) and/or to a variation of characters (uppercase, lowercase, digits and special characters).", "group_already_exist": "Group {group} already exists", @@ -518,8 +541,7 @@ "installation_complete": "Installation completed", "invalid_credentials": "Invalid password or username", "invalid_number": "Must be a number", - "invalid_number_max": "Must be lesser than {max}", - "invalid_number_min": "Must be greater than {min}", + "invalid_password": "Invalid password", "invalid_regex": "Invalid regex:'{regex}'", "invalid_shell": "Invalid shell: {shell}", "ip6tables_unavailable": "You cannot play with ip6tables here. You are either in a container or your kernel does not support it", @@ -580,6 +602,8 @@ "log_user_permission_update": "Update accesses for permission '{}'", "log_user_update": "Update info for user '{}'", "mail_alias_remove_failed": "Could not remove e-mail alias '{mail}'", + "mail_alias_unauthorized": "You are not authorized to add aliases related to domain '{domain}'", + "mail_already_exists": "Mail adress '{mail}' already exists", "mail_domain_unknown": "Invalid e-mail address for domain '{domain}'. Please, use a domain administrated by this server.", "mail_forward_remove_failed": "Could not remove e-mail forwarding '{mail}'", "mail_unavailable": "This e-mail address is reserved for the admins group", @@ -587,28 +611,6 @@ "mailbox_used_space_dovecot_down": "The Dovecot mailbox service needs to be up if you want to fetch used mailbox space", "main_domain_change_failed": "Unable to change the main domain", "main_domain_changed": "The main domain has been changed", - "migration_0021_cleaning_up": "Cleaning up cache and packages not useful anymore…", - "migration_0021_general_warning": "Please note that this migration is a delicate operation. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - Perform a backup of any critical data or app. More info on https://yunohost.org/backup;\n - Be patient after launching the migration: Depending on your Internet connection and hardware, it might take up to a few hours for everything to upgrade.", - "migration_0021_main_upgrade": "Starting main upgrade…", - "migration_0021_modified_files": "Please note that the following files were found to be manually modified and might be overwritten following the upgrade: {manually_modified_files}", - "migration_0021_not_buster2": "The current Debian distribution is not Buster! If you already ran the Buster -> Bullseye migration, then this error is symptomatic of the fact that the migration procedure was not 100% succesful (otherwise YunoHost would have flagged it as completed). It is recommended to investigate what happened with the support team, who will need the **full** log of the migration, which can be found in Tools > Logs in the webadmin.", - "migration_0021_not_enough_free_space": "Free space is pretty low in /var/! You should have at least 1GB free to run this migration.", - "migration_0021_patch_yunohost_conflicts": "Applying patch to workaround conflict issue…", - "migration_0021_patching_sources_list": "Patching the sources.lists…", - "migration_0021_problematic_apps_warning": "Please note that the following possibly problematic installed apps were detected. It looks like those were not installed from the YunoHost app catalog, or are not flagged as 'working'. Consequently, it cannot be guaranteed that they will still work after the upgrade: {problematic_apps}", - "migration_0021_start": "Starting migration to Bullseye", - "migration_0021_still_on_buster_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Buster", - "migration_0021_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Bullseye.", - "migration_0021_yunohost_upgrade": "Starting YunoHost core upgrade…", - "migration_0023_not_enough_space": "Make sufficient space available in {path} to run the migration.", - "migration_0023_postgresql_11_not_installed": "PostgreSQL was not installed on your system. Nothing to do.", - "migration_0023_postgresql_13_not_installed": "PostgreSQL 11 is installed, but not PostgreSQL 13!? Something weird might have happened on your system :(…", - "migration_0024_rebuild_python_venv_broken_app": "Skipping {app} because virtualenv can't easily be rebuilt for this app. Instead, you should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.", - "migration_0024_rebuild_python_venv_disclaimer_base": "Following the upgrade to Debian Bullseye, some Python applications needs to be partially rebuilt to get converted to the new Python version shipped in Debian (in technical terms: what's called the 'virtualenv' needs to be recreated). In the meantime, those Python applications may not work. YunoHost can attempt to rebuild the virtualenv for some of those, as detailed below. For other apps, or if the rebuild attempt fails, you will need to manually force an upgrade for those apps.", - "migration_0024_rebuild_python_venv_disclaimer_ignored": "Virtualenvs can't be rebuilt automatically for those apps. You need to force an upgrade for those, which can be done from the command line with: `yunohost app upgrade --force APP`: {ignored_apps}", - "migration_0024_rebuild_python_venv_disclaimer_rebuild": "Rebuilding the virtualenv will be attempted for the following apps (NB: the operation may take some time!): {rebuild_apps}", - "migration_0024_rebuild_python_venv_failed": "Failed to rebuild the Python virtualenv for {app}. The app may not work as long as this is not resolved. You should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.", - "migration_0024_rebuild_python_venv_in_progress": "Now attempting to rebuild the Python virtualenv for `{app}`", "migration_0027_cleaning_up": "Cleaning up cache and packages not useful anymore…", "migration_0027_delayed_api_restart": "The YunoHost API will automatically be restarted in 15 seconds. It may be unavailable for a few seconds, and then you will have to login again.", "migration_0027_general_warning": "Finally, please note that this migration is **a delicate operation**. The YunoHost team did its best to review and test it, but the migration might still break parts of the system or its apps.\n\nTherefore, it is recommended to:\n - **Perform backups** of any critical data or app. More info on https://yunohost.org/backup;\n - **Be patient** after launching the migration: depending on your Internet connection and hardware, it might take up to an hour for everything to upgrade properly;\n - **Reach the community** on the forum if you need help troubleshooting issues.", @@ -623,13 +625,21 @@ "migration_0027_still_on_bullseye_after_main_upgrade": "Something went wrong during the main upgrade, the system appears to still be on Debian Bullseye.", "migration_0027_system_not_fully_up_to_date": "Your system is not fully up-to-date. Please perform a regular upgrade before running the migration to Bookworm.", "migration_0027_yunohost_upgrade": "Starting YunoHost core upgrade…", - "migration_description_0021_migrate_to_bullseye": "Upgrade the system to Debian Bullseye and YunoHost 11.x", - "migration_description_0022_php73_to_php74_pools": "Migrate php7.3-fpm 'pool' conf files to php7.4", - "migration_description_0023_postgresql_11_to_13": "Migrate databases from PostgreSQL 11 to 13", - "migration_description_0024_rebuild_python_venv": "Repair Python app after bullseye migration", - "migration_description_0025_global_settings_to_configpanel": "Migrate legacy global settings nomenclature to the new, modern nomenclature", - "migration_description_0026_new_admins_group": "Migrate to the new 'multiple admins' system", + "migration_0029_not_enough_space": "Make sufficient space available in {path} to run the migration.", + "migration_0029_postgresql_13_not_installed": "PostgreSQL was not installed on your system. Nothing to do.", + "migration_0029_postgresql_15_not_installed": "PostgreSQL 13 is installed, but not PostgreSQL 15!? Something weird might have happened on your system :(…", + "migration_0030_rebuild_python_venv_in_bookworm_broken_app": "Skipping {app} because virtualenv can't easily be rebuilt for this app. Instead, you should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.", + "migration_0030_rebuild_python_venv_in_bookworm_disclaimer_base": "Following the upgrade to Debian Bookworm, some Python applications needs to be partially rebuilt to get converted to the new Python version shipped in Debian (in technical terms: what's called the 'virtualenv' needs to be recreated). In the meantime, those Python applications may not work. YunoHost can attempt to rebuild the virtualenv for some of those, as detailed below. For other apps, or if the rebuild attempt fails, you will need to manually force an upgrade for those apps.", + "migration_0030_rebuild_python_venv_in_bookworm_disclaimer_ignored": "Virtualenvs can't be rebuilt automatically for those apps. You need to force an upgrade for those, which can be done from the command line with: `yunohost app upgrade --force APP`: {ignored_apps}", + "migration_0030_rebuild_python_venv_in_bookworm_disclaimer_rebuild": "Rebuilding the virtualenv will be attempted for the following apps (NB: the operation may take some time!): {rebuild_apps}", + "migration_0030_rebuild_python_venv_in_bookworm_failed": "Failed to rebuild the Python virtualenv for {app}. The app may not work as long as this is not resolved. You should fix the situation by forcing the upgrade of this app using `yunohost app upgrade --force {app}`.", + "migration_0030_rebuild_python_venv_in_bookworm_in_progress": "Now attempting to rebuild the Python virtualenv for `{app}`", + "migration_0031_terms_of_services": "This migration is purely an informational message about the fact that the YunoHost project now publishes Terms of Services related to the technical and community services.", "migration_description_0027_migrate_to_bookworm": "Upgrade the system to Debian Bookworm and YunoHost 12", + "migration_description_0028_delete_legacy_xmpp_permission": "Delete the old XMPP permissions, Metronome is now an app", + "migration_description_0029_postgresql_13_to_15": "Migrate databases from PostgreSQL 13 to 15", + "migration_description_0030_rebuild_python_venv_in_bookworm": "Repair Python app after bookworm migration", + "migration_description_0031_terms_of_services": "Terms of services", "migration_ldap_backup_before_migration": "Creating a backup of LDAP database and apps settings prior to the actual migration.", "migration_ldap_can_not_backup_before_migration": "The backup of the system could not be completed before the migration failed. Error: {error}", "migration_ldap_migration_failed_trying_to_rollback": "Could not migrate… trying to roll back the system.", @@ -665,9 +675,7 @@ "pattern_domain": "Must be a valid domain name (e.g. my-domain.org)", "pattern_email": "Must be a valid e-mail address, without '+' symbol (e.g. someone@example.com)", "pattern_email_forward": "Must be a valid e-mail address, '+' symbol accepted (e.g. someone+tag@example.com)", - "pattern_firstname": "Must be a valid first name (at least 3 chars)", "pattern_fullname": "Must be a valid full name (at least 3 chars)", - "pattern_lastname": "Must be a valid last name (at least 3 chars)", "pattern_mailbox_quota": "Must be a size with b/k/M/G/T suffix or 0 to not have a quota", "pattern_password": "Must be at least 3 characters long", "pattern_password_app": "Sorry, passwords can not contain the following characters: {forbidden_chars}", @@ -692,7 +700,22 @@ "port_already_closed": "Port {port} is already closed for {ip_version} connections", "port_already_opened": "Port {port} is already opened for {ip_version} connections", "postinstall_low_rootfsspace": "The root filesystem has a total space less than 10 GB, which is quite worrisome! You will likely run out of disk space very quickly! It's recommended to have at least 16GB for the root filesystem. If you want to install YunoHost despite this warning, re-run the postinstall with --force-diskspace", - "regenconf_dry_pending_applying": "Checking pending configuration which would have been applied for category '{category}'…", + "pydantic_type_error": "Invalid type.", + "pydantic_type_error_none_not_allowed": "Value is required.", + "pydantic_type_error_str": "Invalid type, string expected.", + "pydantic_value_error_color": "Not a valid color, value must be a named or hex color.", + "pydantic_value_error_const": "Unexpected value; choose between {permitted}", + "pydantic_value_error_date": "Invalid date format", + "pydantic_value_error_email": "Value is not a valid email address", + "pydantic_value_error_number_not_ge": "Value must be greater than or equal to {limit_value}.", + "pydantic_value_error_number_not_le": "Value must be less than or equal to {limit_value}.", + "pydantic_value_error_str_regex": "Invalid string; value doesn't respects the pattern '{pattern}'", + "pydantic_value_error_time": "Invalid time format", + "pydantic_value_error_url_extra": "URL invalid, extra characters found after valid URL: '{extra}'", + "pydantic_value_error_url_host": "URL host invalid", + "pydantic_value_error_url_port": "URL port invalid, port cannot exceed 65535", + "pydantic_value_error_url_scheme": "Invalid or missing URL scheme", + "regenconf_dry_pending_applying": "Checking pending configuration which would have been applied for category '{category}'...", "regenconf_failed": "Could not regenerate the configuration for category(s): {categories}", "regenconf_file_backed_up": "Configuration file '{conf}' backed up to '{backup}'", "regenconf_file_copy_failed": "Could not copy the new configuration file '{new}' to '{conf}'", @@ -741,17 +764,17 @@ "service_description_dnsmasq": "Handles domain name resolution (DNS)", "service_description_dovecot": "Allows e-mail clients to access/fetch email (via IMAP and POP3)", "service_description_fail2ban": "Protects against brute-force and other kinds of attacks from the Internet", - "service_description_metronome": "Manage XMPP instant messaging accounts", "service_description_mysql": "Stores app data (SQL database)", "service_description_nginx": "Serves or provides access to all the websites hosted on your server", + "service_description_opendkim": "Signs outgoing emails using DKIM such that they are less likely to be flagged as spam", "service_description_postfix": "Used to send and receive e-mails", "service_description_postgresql": "Stores app data (SQL database)", "service_description_redis-server": "A specialized database used for rapid data access, task queue, and communication between programs", - "service_description_rspamd": "Filters spam, and other e-mail related features", "service_description_slapd": "Stores users, domains and related info", "service_description_ssh": "Allows you to connect remotely to your server via a terminal (SSH protocol)", "service_description_yunohost-api": "Manages interactions between the YunoHost web interface and the system", "service_description_yunohost-firewall": "Manages open and close connection ports to services", + "service_description_yunohost-portal-api": "Manages interactions between the different user portal web interfaces and the system", "service_description_yunomdns": "Allows you to reach your server using 'yunohost.local' in your local network", "service_disable_failed": "Could not make the service '{service}' not start at boot.\n\nRecent service logs:{logs}", "service_disabled": "The service '{service}' will not be started anymore when system boots.", @@ -771,14 +794,18 @@ "service_stop_failed": "Unable to stop the service '{service}'\n\nRecent service logs:{logs}", "service_stopped": "Service '{service}' stopped", "service_unknown": "Unknown service '{service}'", + "session_expired": "Session expired", "show_tile_cant_be_enabled_for_regex": "You cannot enable 'show_tile' right now, because the URL for the permission '{permission}' is a regex", "show_tile_cant_be_enabled_for_url_not_defined": "You cannot enable 'show_tile' right now, because you must first define an URL for the permission '{permission}'", - "ssowat_conf_generated": "SSOwat configuration regenerated", + "ssowat_conf_generated": "SSO and portal configurations regenerated", "system_upgraded": "System upgraded", "system_username_exists": "Username already exists in the list of system users", "this_action_broke_dpkg": "This action broke dpkg/APT (the system package managers)… You can try to solve this issue by connecting through SSH and running `sudo apt install --fix-broken` and/or `sudo dpkg --configure -a`.", "tools_upgrade": "Upgrading system packages", "tools_upgrade_failed": "Could not upgrade packages: {packages_list}", + "tos_dyndns_acknowledgement": "You chose to register a DynDNS domain which is a service provided by the YunoHost project. Considering that domain names are an important aspect of long-term digital services, we remind you to read carefully the corresponding Terms of Services, in particular the section regarding those free domain names: .", + "tos_postinstall_acknowledgement": "The YunoHost project is a team of volunteers who have made common cause to create a free operating system for servers, called YunoHost. The YunoHost software is published under the AGPLv3 license (). In connection with this software, the project administers and makes available several technical and community services for various purposes. By using these services, you agree to be bound by the following Terms of Services: .", + "unable_authenticate": "Failed to authenticate session", "unbackup_app": "{app} will not be saved", "unexpected_error": "Something unexpected went wrong: {error}", "unknown_main_domain_path": "Unknown domain or path for '{app}'. You need to specify a domain and a path to be able to specify a URL for permission.", diff --git a/maintenance/make_changelog.sh b/maintenance/make_changelog.sh index a24ccb08ca..b67c1dedd0 100644 --- a/maintenance/make_changelog.sh +++ b/maintenance/make_changelog.sh @@ -5,7 +5,7 @@ REPO_URL=$(git remote get-url origin) ME=$(git config --get user.name) EMAIL=$(git config --get user.email) -LAST_RELEASE=$(git tag --list 'debian/11.*' --sort="v:refname" | tail -n 1) +LAST_RELEASE=$(git tag --list 'debian/12.*' --sort="v:refname" | tail -n 1) echo "$REPO ($VERSION) $RELEASE; urgency=low" echo "" diff --git a/maintenance/missing_i18n_keys.py b/maintenance/missing_i18n_keys.py index 1226ea6002..3248aba18a 100644 --- a/maintenance/missing_i18n_keys.py +++ b/maintenance/missing_i18n_keys.py @@ -26,7 +26,7 @@ def find_expected_string_keys(): # # i18n: foo p1 = re.compile(r"m18n\.n\(\n*\s*[\"\'](\w+)[\"\']") p2 = re.compile(r"YunohostError\(\n*\s*[\'\"](\w+)[\'\"]") - p3 = re.compile(r"YunohostValidationError\(\n*\s*[\'\"](\w+)[\'\"]") + p3 = re.compile(r"Yunohost(Validation|Authentication)Error\(\n*\s*[\'\"](\w+)[\'\"]") p4 = re.compile(r"# i18n: [\'\"]?(\w+)[\'\"]?") python_files = glob.glob(ROOT + "src/*.py") @@ -47,7 +47,7 @@ def find_expected_string_keys(): if m.endswith("_"): continue yield m - for m in p3.findall(content): + for _, m in p3.findall(content): if m.endswith("_"): continue yield m @@ -76,9 +76,6 @@ def find_expected_string_keys(): continue yield "migration_description_" + os.path.basename(path)[:-3] - # FIXME: to be removed in bookworm branch - yield "migration_description_0027_migrate_to_bookworm" - # For each default service, expect to find "service_description_" for service, info in yaml.safe_load( open(ROOT + "conf/yunohost/services.yml") @@ -139,16 +136,32 @@ def find_expected_string_keys(): # Domain config panel domain_config = toml.load(open(ROOT + "share/config_domain.toml")) - for panel in domain_config.values(): + domain_settings_with_help_key = [ + "portal_logo", + "portal_public_intro", + "portal_theme", + "portal_user_intro", + "search_engine", + "custom_css", + "dns", + "enable_public_apps_page", + ] + domain_section_with_no_name = ["app", "cert_", "mail", "registrar"] + for panel_key, panel in domain_config.items(): if not isinstance(panel, dict): continue - for section in panel.values(): + yield f"domain_config_{panel_key}_name" + for section_key, section in panel.items(): if not isinstance(section, dict): continue + if section_key not in domain_section_with_no_name: + yield f"domain_config_{section_key}_name" for key, values in section.items(): if not isinstance(values, dict): continue yield f"domain_config_{key}" + if key in domain_settings_with_help_key: + yield f"domain_config_{key}_help" # Global settings global_config = toml.load(open(ROOT + "share/config_global.toml")) @@ -166,12 +179,14 @@ def find_expected_string_keys(): "tls_passthrough_explain", ] - for panel in global_config.values(): + for panel_key, panel in global_config.items(): if not isinstance(panel, dict): continue - for section in panel.values(): + yield f"global_settings_setting_{panel_key}_name" + for section_key, section in panel.items(): if not isinstance(section, dict): continue + yield f"global_settings_setting_{section_key}_name" for key, values in section.items(): if not isinstance(values, dict): continue diff --git a/share/actionsmap-portal.yml b/share/actionsmap-portal.yml new file mode 100644 index 0000000000..6b02a061d1 --- /dev/null +++ b/share/actionsmap-portal.yml @@ -0,0 +1,92 @@ +_global: + namespace: yunohost + authentication: + api: ldap_ynhuser + cli: null + lock: false + cache: false + +portal: + category_help: Portal routes + actions: + + ### portal_me() + me: + action_help: Allow user to fetch their own infos + api: GET /me + + ### portal_apps() + apps: + action_help: Allow users to fetch lit of apps they have access to + api: GET /me/apps + + ### portal_update() + update: + action_help: Allow user to update their infos (display name, mail aliases/forward, password, ...) + api: PUT /update + arguments: + --fullname: + help: The full name of the user. For example 'Camille Dupont' + extra: + pattern: &pattern_fullname + - !!str ^([^\W_]{1,30}[ ,.'-]{0,3})+$ + - "pattern_fullname" + --mailforward: + help: Mailforward addresses to add + nargs: "*" + metavar: MAIL + extra: + pattern: &pattern_email_forward + - !!str ^[\w\+.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$ + - "pattern_email_forward" + --mailalias: + help: Mail aliases to add + nargs: "*" + metavar: MAIL + extra: + pattern: &pattern_email + - !!str ^[\w.-]+@([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$ + - "pattern_email" + --currentpassword: + help: Current password + nargs: "?" + --newpassword: + help: New password to set + nargs: "?" + + ### portal_update_password() + # update_password: + # action_help: Allow user to change their password + # api: PUT /me/update_password + # arguments: + # -c: + # full: --current + # help: Current password + # -p: + # full: --password + # help: New password to set + + ### portal_reset_password() + reset_password: + action_help: Allow user to update their infos (display name, mail aliases/forward, ...) + api: PUT /me/reset_password + authentication: + # FIXME: to be implemented ? + api: reset_password_token + # FIXME: add args etc + + ### portal_register() + register: + action_help: Allow user to register using an invite token or ??? + api: POST /me + authentication: + # FIXME: to be implemented ? + api: register_invite_token + # FIXME: add args etc + + ### portal_public() + public: + action_help: Allow anybody to list public apps and other infos regarding the public portal + api: GET /public + authentication: + api: null diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 7e61994ea0..73528e4e6b 100755 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -70,26 +70,10 @@ user: help: The full name of the user. For example 'Camille Dupont' extra: ask: ask_fullname - required: False + required: True pattern: &pattern_fullname - !!str ^([^\W_]{1,30}[ ,.'-]{0,3})+$ - "pattern_fullname" - -f: - full: --firstname - help: Deprecated. Use --fullname instead. - extra: - required: False - pattern: &pattern_firstname - - !!str ^([^\W\d_]{1,30}[ ,.'-]{0,3})+$ - - "pattern_firstname" - -l: - full: --lastname - help: Deprecated. Use --fullname instead. - extra: - required: False - pattern: &pattern_lastname - - !!str ^([^\W\d_]{1,30}[ ,.'-]{0,3})+$ - - "pattern_lastname" -p: full: --password help: User password @@ -102,7 +86,7 @@ user: comment: good_practices_about_user_password -d: full: --domain - help: Domain for the email address and xmpp account + help: Domain for the email address extra: pattern: &pattern_domain - !!str ^([^\W_A-Z]+([-]*[^\W_A-Z]+)*\.)+((xn--)?[^\W_]{2,})$ @@ -120,7 +104,7 @@ user: full: --loginShell help: The login shell used default: "/bin/bash" - + ### user_delete() delete: @@ -147,16 +131,6 @@ user: help: The full name of the user. For example 'Camille Dupont' extra: pattern: *pattern_fullname - -f: - full: --firstname - help: Deprecated. Use --fullname instead. - extra: - pattern: *pattern_firstname - -l: - full: --lastname - help: Deprecated. Use --fullname instead. - extra: - pattern: *pattern_lastname -m: full: --mail extra: @@ -212,12 +186,12 @@ user: arguments: username: help: Username or email to get information - + ### user_export() export: action_help: Export users into CSV api: GET /users/export - + ### user_import() import: action_help: Import several users from CSV @@ -244,10 +218,6 @@ user: action_help: List existing groups api: GET /users/groups arguments: - -s: - full: --short - help: List only the names of groups - action: store_true -f: full: --full help: Display all informations known about each groups @@ -500,7 +470,7 @@ domain: help: Display domains as a tree action: store_true --features: - help: List only domains with features enabled (xmpp, mail_in, mail_out) + help: List only domains with features enabled (mail_in, mail_out) nargs: "*" ### domain_info() @@ -532,6 +502,9 @@ domain: help: If adding a DynDNS domain, subscribe to the DynDNS service with a password, used to later delete the domain extra: pattern: *pattern_password + --install-letsencrypt-cert: + help: If adding a subdomain of an already added domain, try to install a Let's Encrypt certificate + action: store_true ### domain_remove() remove: @@ -561,17 +534,6 @@ domain: extra: pattern: *pattern_password - - ### domain_dns_conf() - dns-conf: - deprecated: true - action_help: Generate sample DNS configuration for a domain - arguments: - domain: - help: Target domain - extra: - pattern: *pattern_domain - ### domain_maindomain() main-domain: action_help: Check the current main domain, or change it @@ -585,54 +547,6 @@ domain: extra: pattern: *pattern_domain - ### certificate_status() - cert-status: - deprecated: true - action_help: List status of current certificates (all by default). - arguments: - domain_list: - help: Domains to check - nargs: "*" - --full: - help: Show more details - action: store_true - - ### certificate_install() - cert-install: - deprecated: true - action_help: Install Let's Encrypt certificates for given domains (all by default). - arguments: - domain_list: - help: Domains for which to install the certificates - nargs: "*" - --force: - help: Install even if current certificate is not self-signed - action: store_true - --no-checks: - help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to install. (Not recommended) - action: store_true - --self-signed: - help: Install self-signed certificate instead of Let's Encrypt - action: store_true - - ### certificate_renew() - cert-renew: - deprecated: true - action_help: Renew the Let's Encrypt certificates for given domains (all by default). - arguments: - domain_list: - help: Domains for which to renew the certificates - nargs: "*" - --force: - help: Ignore the validity threshold (15 days) - action: store_true - --email: - help: Send an email to root with logs if some renewing fails - action: store_true - --no-checks: - help: Does not perform any check that your domain seems correctly configured (DNS, reachability) before attempting to renew. (Not recommended) - action: store_true - ### domain_url_available() url-available: hide_in_help: True @@ -645,7 +559,7 @@ domain: pattern: *pattern_domain path: help: The path to check (e.g. /coffee) - + ### domain_action_run() action-run: @@ -680,7 +594,7 @@ domain: help: Password used to later recover the domain if needed extra: pattern: *pattern_password - + ### domain_dyndns_unsubscribe() unsubscribe: action_help: Unsubscribe from a DynDNS service @@ -746,7 +660,7 @@ domain: domain: help: Domain name key: - help: The question or form key + help: The question or form key nargs: '?' -v: full: --value @@ -761,7 +675,7 @@ domain: ### domain_dns_conf() suggest: action_help: Generate sample DNS configuration for a domain - api: + api: - GET /domains//dns - GET /domains//dns/suggest arguments: @@ -769,7 +683,7 @@ domain: help: Target domain extra: pattern: *pattern_domain - + ### domain_dns_push() push: action_help: Push DNS records to registrar @@ -796,7 +710,9 @@ domain: ### certificate_status() status: action_help: List status of current certificates (all by default). - api: GET /domains//cert + api: + - GET /domains//cert + - GET /domains/*/cert arguments: domain_list: help: Domains to check @@ -832,7 +748,7 @@ domain: help: Domains for which to renew the certificates nargs: "*" --force: - help: Ignore the validity threshold (30 days) + help: Ignore the validity threshold (15 days) action: store_true --email: help: Send an email to root with logs if some renewing fails @@ -1151,7 +1067,7 @@ app: app: help: App name key: - help: The question or panel key + help: The question or panel key nargs: '?' -v: full: --value @@ -1308,7 +1224,7 @@ settings: api: PUT /settings/ arguments: key: - help: The question or form key + help: The question or form key nargs: '?' -v: full: --value @@ -1610,7 +1526,7 @@ dyndns: help: Password used to later recover the domain if needed extra: pattern: *pattern_password - + ### dyndns_update() update: action_help: Update IP on DynDNS platform @@ -1710,6 +1626,9 @@ tools: --force-diskspace: help: Use this if you really want to install YunoHost on a setup with less than 10 GB on the root filesystem action: store_true + --i-have-read-terms-of-services: + help: Automatically reply to the terms of services prompt, for example for non-interactive installations + action: store_true ### tools_update() update: @@ -2074,7 +1993,7 @@ diagnosis: api: PUT /diagnosis/ignore arguments: --filter: - help: "Add a filter. The first element should be a diagnosis category, and other criterias can be provided using the infos from the 'meta' sections in 'yunohost diagnosis show'. For example: 'dnsrecords domain=yolo.test category=xmpp'" + help: "Add a filter. The first element should be a diagnosis category, and other criterias can be provided using the infos from the 'meta' sections in 'yunohost diagnosis show'. For example: 'dnsrecords domain=yolo.test category=mail'" nargs: "*" metavar: CRITERIA --list: diff --git a/share/config_domain.toml b/share/config_domain.toml index 82ef90c323..2bc4e9661f 100644 --- a/share/config_domain.toml +++ b/share/config_domain.toml @@ -2,7 +2,15 @@ version = "1.0" i18n = "domain_config" [feature] -name = "Features" + + [feature.mail] + [feature.mail.mail_out] + type = "boolean" + default = 1 + + [feature.mail.mail_in] + type = "boolean" + default = 1 [feature.app] [feature.app.default_app] @@ -10,71 +18,106 @@ name = "Features" filter = "is_webapp" default = "_none" - [feature.mail] + [feature.portal] + # Only available for "topest" domains - [feature.mail.mail_out] + [feature.portal.enable_public_apps_page] type = "boolean" - default = 1 + default = false - [feature.mail.mail_in] + [feature.portal.show_other_domains_apps] type = "boolean" - default = 1 + default = false - [feature.xmpp] + [feature.portal.portal_title] + type = "string" + default = "YunoHost" + + [feature.portal.portal_logo] + type = "file" + accept = ["image/png", "image/jpeg", "image/svg+xml"] + mode = "python" + bind = "/usr/share/yunohost/portal/customassets/{filename}{ext}" + + [feature.portal.portal_theme] + type = "select" + choices = ["system", "light", "dark", "omg", "legacy", "black", "synthwave", "halloween", "coffee", "cupcake", "cyberpunk", "valentine", "nord"] + default = "system" + + [feature.portal.portal_tile_theme] + type = "select" + optional = false + choices = ["descriptive", "simple", "periodic"] + default = "simple" + + [feature.portal.search_engine] + type = "url" + default = "" + + [feature.portal.search_engine_name] + type = "string" + visible = "search_engine" - [feature.xmpp.xmpp] - type = "boolean" - default = 0 + [feature.portal.portal_user_intro] + type = "text" + + [feature.portal.portal_public_intro] + type = "text" + + # FIXME link to GCU + + [feature.portal.custom_css] + # NB: this is wrote into "/usr/share/yunohost/portal/customassets/{domain}.custom.css" + type = "text" [dns] -name = "DNS" [dns.registrar] # This part is automatically generated in DomainConfigPanel [cert] -name = "Certificate" - [cert.cert] + [cert.cert_] + # The section has a different id than 'cert' otherwise it ends up with an unecessary "name" because it's defined for the panel (in i18n.json) - [cert.cert.cert_summary] + [cert.cert_.cert_summary] type = "alert" # Automatically filled by DomainConfigPanel - [cert.cert.cert_validity] + [cert.cert_.cert_validity] type = "number" readonly = true visible = "false" # Automatically filled by DomainConfigPanel - [cert.cert.cert_issuer] + [cert.cert_.cert_issuer] type = "string" visible = false # Automatically filled by DomainConfigPanel - [cert.cert.acme_eligible] + [cert.cert_.acme_eligible] type = "boolean" visible = false # Automatically filled by DomainConfigPanel - [cert.cert.acme_eligible_explain] + [cert.cert_.acme_eligible_explain] type = "alert" style = "warning" visible = "acme_eligible == false || acme_eligible == null" - [cert.cert.cert_no_checks] + [cert.cert_.cert_no_checks] type = "boolean" default = false visible = "acme_eligible == false || acme_eligible == null" - [cert.cert.cert_install] + [cert.cert_.cert_install] type = "button" icon = "star" style = "success" visible = "cert_issuer != 'letsencrypt'" enabled = "acme_eligible || cert_no_checks" - [cert.cert.cert_renew] + [cert.cert_.cert_renew] type = "button" icon = "refresh" style = "warning" diff --git a/share/config_global.toml b/share/config_global.toml index 5c4eacbb06..2b2795a5cd 100644 --- a/share/config_global.toml +++ b/share/config_global.toml @@ -2,9 +2,7 @@ version = "1.0" i18n = "global_settings_setting" [security] -name = "Security" [security.password] - name = "Passwords" [security.password.admin_strength] type = "select" @@ -28,7 +26,7 @@ name = "Security" default = false [security.ssh] - name = "SSH" + [security.ssh.ssh_compatibility] type = "select" choices.intermediate = "Intermediate (compatible with older softwares)" @@ -44,7 +42,6 @@ name = "Security" default = true [security.nginx] - name = "NGINX (web server)" [security.nginx.nginx_redirect_to_https] type = "boolean" default = true @@ -56,7 +53,7 @@ name = "Security" default = "intermediate" [security.postfix] - name = "Postfix (SMTP email server)" + [security.postfix.postfix_compatibility] type = "select" choices.intermediate = "Intermediate (allows TLS 1.2)" @@ -64,7 +61,6 @@ name = "Security" default = "intermediate" [security.webadmin] - name = "Webadmin" [security.webadmin.webadmin_allowlist_enabled] type = "boolean" default = false @@ -76,8 +72,6 @@ name = "Security" default = "" [security.root_access] - name = "Change root password" - [security.root_access.root_access_explain] type = "alert" style = "info" @@ -94,22 +88,17 @@ name = "Security" default = "" [security.experimental] - name = "Experimental" [security.experimental.security_experimental_enabled] type = "boolean" default = false - [email] -name = "Email" [email.pop3] - name = "POP3" [email.pop3.pop3_enabled] type = "boolean" default = false [email.smtp] - name = "SMTP" [email.smtp.smtp_allow_ipv6] type = "boolean" default = true @@ -154,26 +143,13 @@ name = "Email" visible = "smtp_backup_mx_domains" [misc] -name = "Other" - [misc.portal] - name = "User portal" - [misc.portal.ssowat_panel_overlay_enabled] - type = "boolean" - default = true - - [misc.portal.portal_theme] - type = "select" - # Choices are loaded dynamically in the python code - default = "default" [misc.backup] - name = "Backup" [misc.backup.backup_compress_tar_archives] type = "boolean" default = false [misc.network] - name = "Network" [misc.network.dns_exposure] type = "select" choices.both = "Both" @@ -182,7 +158,6 @@ name = "Other" default = "both" [misc.tls_passthrough] - name = "TLS-passthrough / SNI-based forwarding" visible = "security_experimental_enabled" [misc.tls_passthrough.tls_passthrough_enabled] diff --git a/src/__init__.py b/src/__init__.py index 8bb192e6a7..7eb4f16cf9 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -50,6 +50,13 @@ def cli(debug, quiet, output_as, timeout, args, parser): def api(debug, host, port): + + allowed_cors_origins = [] + allowed_cors_origins_file = "/etc/yunohost/.admin-api-allowed-cors-origins" + + if os.path.exists(allowed_cors_origins_file): + allowed_cors_origins = open(allowed_cors_origins_file).read().strip().split(",") + init_logging(interface="api", debug=debug) def is_installed_api(): @@ -64,6 +71,28 @@ def is_installed_api(): actionsmap="/usr/share/yunohost/actionsmap.yml", locales_dir="/usr/share/yunohost/locales/", routes={("GET", "/installed"): is_installed_api}, + allowed_cors_origins=allowed_cors_origins, + ) + sys.exit(ret) + + +def portalapi(debug, host, port): + + allowed_cors_origins = [] + allowed_cors_origins_file = "/etc/yunohost/.portal-api-allowed-cors-origins" + + if os.path.exists(allowed_cors_origins_file): + allowed_cors_origins = open(allowed_cors_origins_file).read().strip().split(",") + + # FIXME : is this the logdir we want ? (yolo to work around permission issue) + init_logging(interface="portalapi", debug=debug, logdir="/var/log") + + ret = moulinette.api( + host=host, + port=port, + actionsmap="/usr/share/yunohost/actionsmap-portal.yml", + locales_dir="/usr/share/yunohost/locales/", + allowed_cors_origins=allowed_cors_origins, ) sys.exit(ret) @@ -115,17 +144,11 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun "version": 1, "disable_existing_loggers": True, "formatters": { - "console": { - "format": "%(relativeCreated)-5d %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" + "tty-debug": { + "format": "%(relativeCreated)-4d %(level_with_color)s %(message)s" }, - "tty-debug": {"format": "%(relativeCreated)-4d %(fmessage)s"}, "precise": { - "format": "%(asctime)-15s %(levelname)-8s %(name)s %(funcName)s - %(fmessage)s" - }, - }, - "filters": { - "action": { - "()": "moulinette.utils.log.ActionFilter", + "format": "%(asctime)-15s %(levelname)-8s %(name)s.%(funcName)s - %(message)s" }, }, "handlers": { @@ -138,11 +161,14 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun "level": "DEBUG" if debug else "INFO", "class": "moulinette.interfaces.api.APIQueueHandler", }, + "portalapi": { + "level": "DEBUG" if debug else "INFO", + "class": "moulinette.interfaces.api.APIQueueHandler", + }, "file": { "class": "logging.FileHandler", "formatter": "precise", "filename": logfile, - "filters": ["action"], }, }, "loggers": { @@ -164,7 +190,7 @@ def init_logging(interface="cli", debug=False, quiet=False, logdir="/var/log/yun } # Logging configuration for CLI (or any other interface than api...) # - if interface != "api": + if interface not in ["api", "portalapi"]: configure_logging(logging_configuration) # Logging configuration for API # diff --git a/src/app.py b/src/app.py index c2d930bc97..ec274b8e70 100644 --- a/src/app.py +++ b/src/app.py @@ -17,26 +17,26 @@ # along with this program. If not, see . # +import time import glob import os import shutil import yaml -import time import re import subprocess import tempfile import copy -from typing import List, Tuple, Dict, Any, Iterator, Optional +from typing import TYPE_CHECKING, List, Tuple, Dict, Any, Iterator, Optional, Union from packaging import version +from logging import getLogger +from pathlib import Path from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import run_commands, check_output from moulinette.utils.filesystem import ( read_file, read_json, read_toml, - read_yaml, write_to_file, write_to_json, cp, @@ -45,12 +45,6 @@ chmod, ) -from yunohost.utils.configpanel import ConfigPanel, ask_questions_and_parse_answers -from yunohost.utils.form import ( - DomainOption, - WebPathOption, - hydrate_questions_with_choices, -) from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.system import ( @@ -71,7 +65,13 @@ APPS_CATALOG_LOGOS, ) -logger = getActionLogger("yunohost.app") +if TYPE_CHECKING: + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + + from yunohost.utils.configpanel import RawSettings, ConfigPanelModel + from yunohost.utils.form import FormModel + +logger = getLogger("yunohost.app") APPS_SETTING_PATH = "/etc/yunohost/apps/" APP_TMP_WORKDIRS = "/var/cache/yunohost/app_tmp_work_dirs" @@ -96,6 +96,8 @@ "doc", ] +PORTAL_SETTINGS_DIR = "/etc/yunohost/portal" + def app_list(full=False, upgradable=False): """ @@ -121,8 +123,8 @@ def app_info(app, full=False, upgradable=False): """ Get info for a specific app """ + from yunohost.domain import _get_raw_domain_settings from yunohost.permission import user_permission_list - from yunohost.domain import domain_config_get _assert_is_installed(app) @@ -218,11 +220,11 @@ def app_info(app, full=False, upgradable=False): rendered_notifications[name][lang] = rendered_content ret["manifest"]["notifications"][step] = rendered_notifications - ret["is_webapp"] = "domain" in settings and "path" in settings + ret["is_webapp"] = "domain" in settings and settings["domain"] and "path" in settings if ret["is_webapp"]: ret["is_default"] = ( - domain_config_get(settings["domain"], "feature.app.default_app") == app + _get_raw_domain_settings(settings["domain"]).get("default_app") == app ) ret["supports_change_url"] = os.path.exists( @@ -255,8 +257,8 @@ def _app_upgradable(app_infos): # Determine upgradability app_in_catalog = app_infos.get("from_catalog") - installed_version = version.parse(app_infos.get("version", "0~ynh0")) - version_in_catalog = version.parse( + installed_version = _parse_app_version(app_infos.get("version", "0~ynh0")) + version_in_catalog = _parse_app_version( app_infos.get("from_catalog", {}).get("manifest", {}).get("version", "0~ynh0") ) @@ -271,25 +273,7 @@ def _app_upgradable(app_infos): ): return "bad_quality" - # If the app uses the standard version scheme, use it to determine - # upgradability - if "~ynh" in str(installed_version) and "~ynh" in str(version_in_catalog): - if installed_version < version_in_catalog: - return "yes" - else: - return "no" - - # Legacy stuff for app with old / non-standard version numbers... - - # In case there is neither update_time nor install_time, we assume the app can/has to be upgraded - if not app_infos["from_catalog"].get("lastUpdate") or not app_infos[ - "from_catalog" - ].get("git"): - return "url_required" - - settings = app_infos["settings"] - local_update_time = settings.get("update_time", settings.get("install_time", 0)) - if app_infos["from_catalog"]["lastUpdate"] > local_update_time: + if installed_version < version_in_catalog: return "yes" else: return "no" @@ -422,6 +406,7 @@ def app_change_url(operation_logger, app, domain, path): path -- New path at which the application will be move """ + from yunohost.utils.form import DomainOption, WebPathOption from yunohost.hook import hook_exec_with_script_debug_if_failure, hook_callback from yunohost.service import service_reload_or_restart @@ -573,7 +558,7 @@ def app_upgrade( ) from yunohost.permission import permission_sync_to_user from yunohost.regenconf import manually_modified_files - from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers + from yunohost.utils.legacy import _patch_legacy_helpers from yunohost.backup import ( backup_list, backup_create, @@ -634,9 +619,11 @@ def app_upgrade( # Manage upgrade type and avoid any upgrade if there is nothing to do upgrade_type = "UNKNOWN" # Get current_version and new version - app_new_version = version.parse(manifest.get("version", "?")) - app_current_version = version.parse(app_dict.get("version", "?")) - if "~ynh" in str(app_current_version) and "~ynh" in str(app_new_version): + app_new_version_raw = manifest.get("version", "?") + app_current_version_raw = app_dict.get("version", "?") + app_new_version = _parse_app_version(app_new_version_raw) + app_current_version = _parse_app_version(app_current_version_raw) + if "~ynh" in str(app_current_version_raw) and "~ynh" in str(app_new_version_raw): if app_current_version >= app_new_version and not force: # In case of upgrade from file or custom repository # No new version available @@ -656,8 +643,8 @@ def app_upgrade( elif app_current_version == app_new_version: upgrade_type = "UPGRADE_SAME" else: - app_current_version_upstream, _ = str(app_current_version).split("~ynh") - app_new_version_upstream, _ = str(app_new_version).split("~ynh") + app_current_version_upstream, _ = str(app_current_version_raw).split("~ynh") + app_new_version_upstream, _ = str(app_new_version_raw).split("~ynh") if app_current_version_upstream == app_new_version_upstream: upgrade_type = "UPGRADE_PACKAGE" else: @@ -684,7 +671,7 @@ def app_upgrade( settings = _get_app_settings(app_instance_name) notifications = _filter_and_hydrate_notifications( manifest["notifications"]["PRE_UPGRADE"], - current_version=app_current_version, + current_version=app_current_version_raw, data=settings, ) _display_notifications(notifications, force=force) @@ -739,9 +726,6 @@ def app_upgrade( # Attempt to patch legacy helpers ... _patch_legacy_helpers(extracted_app_folder) - # Apply dirty patch to make php5 apps compatible with php7 - _patch_legacy_php_versions(extracted_app_folder) - # Prepare env. var. to pass to script env_dict = _make_environment_for_app_script( app_instance_name, workdir=extracted_app_folder, action="upgrade" @@ -749,8 +733,8 @@ def app_upgrade( env_dict_more = { "YNH_APP_UPGRADE_TYPE": upgrade_type, - "YNH_APP_MANIFEST_VERSION": str(app_new_version), - "YNH_APP_CURRENT_VERSION": str(app_current_version), + "YNH_APP_MANIFEST_VERSION": str(app_new_version_raw), + "YNH_APP_CURRENT_VERSION": str(app_current_version_raw), } if manifest["packaging_format"] < 2: @@ -941,7 +925,7 @@ def app_upgrade( settings = _get_app_settings(app_instance_name) notifications = _filter_and_hydrate_notifications( manifest["notifications"]["POST_UPGRADE"], - current_version=app_current_version, + current_version=app_current_version_raw, data=settings, ) if Moulinette.interface.type == "cli": @@ -976,10 +960,11 @@ def app_upgrade( def app_manifest(app, with_screenshot=False): + from yunohost.utils.form import parse_raw_options + manifest, extracted_app_folder = _extract_app(app) - raw_questions = manifest.get("install", {}).values() - manifest["install"] = hydrate_questions_with_choices(raw_questions) + manifest["install"] = parse_raw_options(manifest.get("install", {}), serialize=True) # Add a base64 image to be displayed in web-admin if with_screenshot and Moulinette.interface.type == "api": @@ -1072,7 +1057,8 @@ def app_install( permission_sync_to_user, ) from yunohost.regenconf import manually_modified_files - from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers + from yunohost.utils.legacy import _patch_legacy_helpers + from yunohost.utils.form import ask_questions_and_parse_answers from yunohost.user import user_list # Check if disk space available @@ -1127,13 +1113,9 @@ def app_install( app_setting_path = os.path.join(APPS_SETTING_PATH, app_instance_name) # Retrieve arguments list for install script - raw_questions = manifest["install"] - questions = ask_questions_and_parse_answers(raw_questions, prefilled_answers=args) - args = { - question.id: question.value - for question in questions - if not question.readonly and question.value is not None - } + raw_options = manifest["install"] + options, form = ask_questions_and_parse_answers(raw_options, prefilled_answers=args) + args = form.dict(exclude_none=True) # Validate domain / path availability for webapps # (ideally this should be handled by the resource system for manifest v >= 2 @@ -1144,9 +1126,6 @@ def app_install( # Attempt to patch legacy helpers ... _patch_legacy_helpers(extracted_app_folder) - # Apply dirty patch to make php5 apps compatible with php7 - _patch_legacy_php_versions(extracted_app_folder) - # We'll check that the app didn't brutally edit some system configuration manually_modified_files_before_install = manually_modified_files() @@ -1170,18 +1149,18 @@ def app_install( "current_revision": manifest.get("remote", {}).get("revision", "?"), } - # If packaging_format v2+, save all install questions as settings + # If packaging_format v2+, save all install options as settings if packaging_format >= 2: - for question in questions: + for option in options: # Except readonly "questions" that don't even have a value - if question.readonly: + if option.readonly: continue # Except user-provider passwords # ... which we need to reinject later in the env_dict - if question.type == "password": + if option.type == "password": continue - app_settings[question.id] = question.value + app_settings[option.id] = form[option.id] _set_app_settings(app_instance_name, app_settings) @@ -1234,23 +1213,23 @@ def app_install( app_instance_name, args=args, workdir=extracted_app_folder, action="install" ) - # If packaging_format v2+, save all install questions as settings + # If packaging_format v2+, save all install options as settings if packaging_format >= 2: - for question in questions: + for option in options: # Reinject user-provider passwords which are not in the app settings # (cf a few line before) - if question.type == "password": - env_dict[question.id] = question.value + if option.type == "password": + env_dict[option.id] = form[option.id] # We want to hav the env_dict in the log ... but not password values env_dict_for_logging = env_dict.copy() - for question in questions: - # Or should it be more generally question.redact ? - if question.type == "password": - if f"YNH_APP_ARG_{question.id.upper()}" in env_dict_for_logging: - del env_dict_for_logging[f"YNH_APP_ARG_{question.id.upper()}"] - if question.id in env_dict_for_logging: - del env_dict_for_logging[question.id] + for option in options: + # Or should it be more generally option.redact ? + if option.type == "password": + if f"YNH_APP_ARG_{option.id.upper()}" in env_dict_for_logging: + del env_dict_for_logging[f"YNH_APP_ARG_{option.id.upper()}"] + if option.id in env_dict_for_logging: + del env_dict_for_logging[option.id] operation_logger.extra.update({"env": env_dict_for_logging}) @@ -1412,14 +1391,14 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): purge -- Remove with all app data force_workdir -- Special var to force the working directoy to use, in context such as remove-after-failed-upgrade or remove-after-failed-restore """ - from yunohost.utils.legacy import _patch_legacy_php_versions, _patch_legacy_helpers + from yunohost.utils.legacy import _patch_legacy_helpers from yunohost.hook import hook_exec, hook_remove, hook_callback from yunohost.permission import ( user_permission_list, permission_delete, permission_sync_to_user, ) - from yunohost.domain import domain_list, domain_config_get, domain_config_set + from yunohost.domain import domain_list, domain_config_set, _get_raw_domain_settings _assert_is_installed(app) @@ -1431,10 +1410,6 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): # Attempt to patch legacy helpers ... _patch_legacy_helpers(app_setting_path) - # Apply dirty patch to make php5 apps compatible with php7 (e.g. the remove - # script might date back from jessie install) - _patch_legacy_php_versions(app_setting_path) - if force_workdir: # This is when e.g. calling app_remove() from the upgrade-failed case # where we want to remove using the *new* remove script and not the old one @@ -1497,7 +1472,7 @@ def app_remove(operation_logger, app, purge=False, force_workdir=None): hook_remove(app) for domain in domain_list()["domains"]: - if domain_config_get(domain, "feature.app.default_app") == app: + if _get_raw_domain_settings(domain).get("default_app") == app: domain_config_set(domain, "feature.app.default_app", "_none") if ret == 0: @@ -1553,121 +1528,6 @@ def app_setting(app, key, value=None, delete=False): """ app_settings = _get_app_settings(app) or {} - # - # Legacy permission setting management - # (unprotected, protected, skipped_uri/regex) - # - - is_legacy_permission_setting = any( - key.startswith(word + "_") for word in ["unprotected", "protected", "skipped"] - ) - - if is_legacy_permission_setting: - from yunohost.permission import ( - user_permission_list, - user_permission_update, - permission_create, - permission_delete, - permission_url, - ) - - permissions = user_permission_list(full=True, apps=[app])["permissions"] - key_ = key.split("_")[0] - permission_name = f"{app}.legacy_{key_}_uris" - permission = permissions.get(permission_name) - - # GET - if value is None and not delete: - return ( - ",".join(permission.get("uris", []) + permission["additional_urls"]) - if permission - else None - ) - - # DELETE - if delete: - # If 'is_public' setting still exists, we interpret this as - # coming from a legacy app (because new apps shouldn't manage the - # is_public state themselves anymore...) - # - # In that case, we interpret the request for "deleting - # unprotected/skipped" setting as willing to make the app - # private - if ( - "is_public" in app_settings - and "visitors" in permissions[app + ".main"]["allowed"] - ): - if key.startswith("unprotected_") or key.startswith("skipped_"): - user_permission_update(app + ".main", remove="visitors") - - if permission: - permission_delete(permission_name) - - # SET - else: - urls = value - # If the request is about the root of the app (/), ( = the vast majority of cases) - # we interpret this as a change for the main permission - # (i.e. allowing/disallowing visitors) - if urls == "/": - if key.startswith("unprotected_") or key.startswith("skipped_"): - permission_url(app + ".main", url="/", sync_perm=False) - user_permission_update(app + ".main", add="visitors") - else: - user_permission_update(app + ".main", remove="visitors") - else: - urls = urls.split(",") - if key.endswith("_regex"): - urls = ["re:" + url for url in urls] - - if permission: - # In case of new regex, save the urls, to add a new time in the additional_urls - # In case of new urls, we do the same thing but inversed - if key.endswith("_regex"): - # List of urls to save - current_urls_or_regex = [ - url - for url in permission["additional_urls"] - if not url.startswith("re:") - ] - else: - # List of regex to save - current_urls_or_regex = [ - url - for url in permission["additional_urls"] - if url.startswith("re:") - ] - - new_urls = urls + current_urls_or_regex - # We need to clear urls because in the old setting the new setting override the old one and dont just add some urls - permission_url(permission_name, clear_urls=True, sync_perm=False) - permission_url(permission_name, add_url=new_urls) - else: - from yunohost.utils.legacy import legacy_permission_label - - # Let's create a "special" permission for the legacy settings - permission_create( - permission=permission_name, - # FIXME find a way to limit to only the user allowed to the main permission - allowed=( - ["all_users"] - if key.startswith("protected_") - else ["all_users", "visitors"] - ), - url=None, - additional_urls=urls, - auth_header=not key.startswith("skipped_"), - label=legacy_permission_label(app, key.split("_")[0]), - show_tile=False, - protected=True, - ) - - return - - # - # Regular setting management - # - # GET if value is None and not delete: return app_settings.get(key, None) @@ -1682,8 +1542,6 @@ def app_setting(app, key, value=None, delete=False): # SET else: - if key in ["redirected_urls", "redirected_regex"]: - value = yaml.safe_load(value) app_settings[key] = value _set_app_settings(app, app_settings) @@ -1715,6 +1573,7 @@ def app_register_url(app, domain, path): domain -- The domain on which the app should be registered (e.g. your.domain.tld) path -- The path to be registered (e.g. /coffee) """ + from yunohost.utils.form import DomainOption, WebPathOption from yunohost.permission import ( permission_url, user_permission_update, @@ -1755,12 +1614,17 @@ def app_ssowatconf(): """ - from yunohost.domain import domain_list, _get_maindomain, domain_config_get + from yunohost.domain import ( + domain_list, + _get_raw_domain_settings, + _get_domain_portal_dict, + ) from yunohost.permission import user_permission_list - from yunohost.settings import settings_get - main_domain = _get_maindomain() + domain_portal_dict = _get_domain_portal_dict() + domains = domain_list()["domains"] + portal_domains = domain_list(exclude_subdomains=True)["domains"] all_permissions = user_permission_list( full=True, ignore_system_perms=True, absolute_urls=True )["permissions"] @@ -1768,49 +1632,26 @@ def app_ssowatconf(): permissions = { "core_skipped": { "users": [], - "label": "Core permissions - skipped", - "show_tile": False, "auth_header": False, "public": True, "uris": [domain + "/yunohost/admin" for domain in domains] + [domain + "/yunohost/api" for domain in domains] + + [domain + "/yunohost/portalapi" for domain in domains] + [ - "re:^[^/]/502%.html$", - "re:^[^/]*/%.well%-known/ynh%-diagnosis/.*$", - "re:^[^/]*/%.well%-known/acme%-challenge/.*$", - "re:^[^/]*/%.well%-known/autoconfig/mail/config%-v1%.1%.xml.*$", + r"re:^[^/]*/502\.html$", + r"re:^[^/]*/\.well-known/ynh-diagnosis/.*$", + r"re:^[^/]*/\.well-known/acme-challenge/.*$", + r"re:^[^/]*/\.well-known/autoconfig/mail/config-v1\.1\.xml.*$", ], } } - redirected_regex = { - main_domain + r"/yunohost[\/]?$": "https://" + main_domain + "/yunohost/sso/" - } - redirected_urls = {} - - apps_using_remote_user_var_in_nginx = ( - check_output( - "grep -nri '$remote_user' /etc/yunohost/apps/*/conf/*nginx*conf | awk -F/ '{print $5}' || true" - ) - .strip() - .split("\n") - ) - - for app in _installed_apps(): - app_settings = read_yaml(APPS_SETTING_PATH + app + "/settings.yml") or {} - - # Redirected - redirected_urls.update(app_settings.get("redirected_urls", {})) - redirected_regex.update(app_settings.get("redirected_regex", {})) - - from .utils.legacy import ( - translate_legacy_default_app_in_ssowant_conf_json_persistent, - ) - - translate_legacy_default_app_in_ssowant_conf_json_persistent() + # FIXME : this could be handled by nginx's regen conf to further simplify ssowat's code ... + redirected_urls = {} for domain in domains: - default_app = domain_config_get(domain, "feature.app.default_app") - if default_app != "_none" and _is_installed(default_app): + default_app = _get_raw_domain_settings(domain).get("default_app") + + if default_app not in ["_none", None] and _is_installed(default_app): app_settings = _get_app_settings(default_app) app_domain = app_settings["domain"] app_path = app_settings["path"] @@ -1818,6 +1659,12 @@ def app_ssowatconf(): # Prevent infinite redirect loop... if domain + "/" != app_domain + app_path: redirected_urls[domain + "/"] = app_domain + app_path + elif bool(_get_raw_domain_settings(domain).get("enable_public_apps_page", False)): + redirected_urls[domain + "/"] = domain_portal_dict[domain] + + # Will organize apps by portal domain + portal_domains_apps = {domain: {} for domain in portal_domains} + apps_catalog = _load_apps_catalog()["apps"] # New permission system for perm_name, perm_info in all_permissions.items(): @@ -1832,38 +1679,98 @@ def app_ssowatconf(): continue app_id = perm_name.split(".")[0] + app_settings = _get_app_settings(app_id) + + if perm_info["auth_header"]: + if app_settings.get("auth_header"): + auth_header = app_settings.get("auth_header") + assert auth_header in ["basic-with-password", "basic-without-password"] + else: + auth_header = "basic-with-password" + else: + auth_header = False permissions[perm_name] = { - "use_remote_user_var_in_nginx_conf": app_id - in apps_using_remote_user_var_in_nginx, "users": perm_info["corresponding_users"], - "label": perm_info["label"], - "show_tile": perm_info["show_tile"] - and perm_info["url"] - and (not perm_info["url"].startswith("re:")), - "auth_header": perm_info["auth_header"], + "auth_header": auth_header, "public": "visitors" in perm_info["allowed"], "uris": uris, } + # Apps can opt out of the auth spoofing protection using this if they really need to, + # but that's a huge security hole and ultimately should never happen... + # ... But some apps live caldav/webdav need this to not break external clients x_x + apps_that_need_external_auth_maybe = ["agendav", "baikal", "ihatemoney", "keeweb", "monica", "nextcloud", "owncloud", "paheko", "radicale", "tracim", "vikunja", "z-push"] + protect_against_basic_auth_spoofing = app_settings.get("protect_against_basic_auth_spoofing") + if protect_against_basic_auth_spoofing is not None: + permissions[perm_name]["protect_against_basic_auth_spoofing"] = protect_against_basic_auth_spoofing not in [False, "False", "false", "0", 0] + elif app_id.split("__")[0] in apps_that_need_external_auth_maybe: + permissions[perm_name]["protect_against_basic_auth_spoofing"] = False + + # Next: portal related + # No need to keep apps that aren't supposed to be displayed in portal + if not perm_info.get("show_tile", False): + continue + + setting_path = os.path.join(APPS_SETTING_PATH, app_id) + local_manifest = _get_manifest_of_app(setting_path) + + app_domain = uris[0].split("/")[0] + # get "topest" domain + app_portal_domain = next( + domain for domain in portal_domains if domain in app_domain + ) + app_portal_info = { + "label": perm_info["label"], + "users": perm_info["corresponding_users"], + "public": "visitors" in perm_info["allowed"], + "url": uris[0], + "description": local_manifest["description"], + } + + # FIXME : find a smarter way to get this info ? (in the settings maybe..) + # Also ideally we should not rely on the webadmin route for this, maybe expose these through a different route in nginx idk + # Also related to "people will want to customize those.." + app_catalog_info = apps_catalog.get(app_id.split("__")[0]) + if app_catalog_info and "logo_hash" in app_catalog_info: + app_portal_info["logo"] = f"/yunohost/sso/applogos/{app_catalog_info['logo_hash']}.png" + + portal_domains_apps[app_portal_domain][perm_name] = app_portal_info + conf_dict = { - "theme": settings_get("misc.portal.portal_theme"), - "portal_domain": main_domain, - "portal_path": "/yunohost/sso/", - "additional_headers": { - "Auth-User": "uid", - "Remote-User": "uid", - "Name": "cn", - "Email": "mail", - }, - "domains": domains, + "cookie_secret_file": "/etc/yunohost/.ssowat_cookie_secret", + "session_folder": "/var/cache/yunohost-portal/sessions", + "cookie_name": "yunohost.portal", "redirected_urls": redirected_urls, - "redirected_regex": redirected_regex, + "domain_portal_urls": domain_portal_dict, "permissions": permissions, } write_to_json("/etc/ssowat/conf.json", conf_dict, sort_keys=True, indent=4) + # Generate a file per possible portal with available apps + for domain, apps in portal_domains_apps.items(): + portal_settings = {} + + portal_settings_path = Path(PORTAL_SETTINGS_DIR) / f"{domain}.json" + if portal_settings_path.exists(): + portal_settings.update(read_json(str(portal_settings_path))) + + # Do no override anything else than "apps" since the file is shared + # with domain's config panel "portal" options + portal_settings["apps"] = apps + + write_to_json( + str(portal_settings_path), portal_settings, sort_keys=True, indent=4 + ) + + # Cleanup old files from possibly old domains + for setting_file in Path(PORTAL_SETTINGS_DIR).iterdir(): + if setting_file.name.endswith(".json"): + domain = setting_file.name[:-len(".json")] + if domain not in portal_domains_apps: + setting_file.unlink() + logger.debug(m18n.n("ssowat_conf_generated")) @@ -1884,11 +1791,13 @@ def app_change_label(app, new_label): def app_action_list(app): + AppConfigPanel = _get_AppConfigPanel() return AppConfigPanel(app).list_actions() @is_unit_operation() def app_action_run(operation_logger, app, action, args=None, args_file=None): + AppConfigPanel = _get_AppConfigPanel() return AppConfigPanel(app).run_action( action, args=args, args_file=args_file, operation_logger=operation_logger ) @@ -1910,6 +1819,7 @@ def app_config_get(app, key="", full=False, export=False): else: mode = "classic" + AppConfigPanel = _get_AppConfigPanel() try: config_ = AppConfigPanel(app) return config_.get(key, mode) @@ -1929,149 +1839,162 @@ def app_config_set( Apply a new app configuration """ + AppConfigPanel = _get_AppConfigPanel() config_ = AppConfigPanel(app) return config_.set(key, value, args, args_file, operation_logger=operation_logger) -class AppConfigPanel(ConfigPanel): - entity_type = "app" - save_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/settings.yml") - config_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/config_panel.toml") - - def _run_action(self, action): - env = {key: str(value) for key, value in self.new_values.items()} - self._call_config_script(action, env=env) - - def _get_raw_settings(self): - self.values = self._call_config_script("show") +def _get_AppConfigPanel(): + from yunohost.utils.configpanel import ConfigPanel + + class AppConfigPanel(ConfigPanel): + entity_type = "app" + save_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/settings.yml") + config_path_tpl = os.path.join(APPS_SETTING_PATH, "{entity}/config_panel.toml") + settings_must_be_defined: bool = True + + def _get_raw_settings(self) -> "RawSettings": + return self._call_config_script("show") + + def _apply( + self, + form: "FormModel", + config: "ConfigPanelModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ) -> None: + env = {key: str(value) for key, value in form.dict().items()} + return_content = self._call_config_script("apply", env=env) + + # If the script returned validation error + # raise a ValidationError exception using + # the first key + errors = return_content.get("validation_errors") + if errors: + for key, message in errors.items(): + raise YunohostValidationError( + "app_argument_invalid", + name=key, + error=message, + ) - def _apply(self): - env = {key: str(value) for key, value in self.new_values.items()} - return_content = self._call_config_script("apply", env=env) + def _run_action(self, form: "FormModel", action_id: str) -> None: + env = {key: str(value) for key, value in form.dict().items()} + self._call_config_script(action_id, env=env) - # If the script returned validation error - # raise a ValidationError exception using - # the first key - if return_content: - for key, message in return_content.get("validation_errors").items(): - raise YunohostValidationError( - "app_argument_invalid", - name=key, - error=message, - ) + def _call_config_script( + self, action: str, env: Union[dict[str, Any], None] = None + ) -> dict[str, Any]: + from yunohost.hook import hook_exec - def _call_config_script(self, action, env=None): - from yunohost.hook import hook_exec + if env is None: + env = {} - if env is None: - env = {} - - # Add default config script if needed - config_script = os.path.join( - APPS_SETTING_PATH, self.entity, "scripts", "config" - ) - if not os.path.exists(config_script): - logger.debug("Adding a default config script") - default_script = """#!/bin/bash + # Add default config script if needed + config_script = os.path.join( + APPS_SETTING_PATH, self.entity, "scripts", "config" + ) + if not os.path.exists(config_script): + logger.debug("Adding a default config script") + default_script = """#!/bin/bash source /usr/share/yunohost/helpers ynh_abort_if_errors ynh_app_config_run $1 """ - write_to_file(config_script, default_script) - - # Call config script to extract current values - logger.debug(f"Calling '{action}' action from config script") - app = self.entity - app_id, app_instance_nb = _parse_app_instance_name(app) - settings = _get_app_settings(app) - app_setting_path = os.path.join(APPS_SETTING_PATH, self.entity) - manifest = _get_manifest_of_app(app_setting_path) - # FIXME: this is inconsistent with other script call ... - # this should be based on _make_environment_for_app_script ... - env.update( - { - "app_id": app_id, - "app": app, - "app_instance_nb": str(app_instance_nb), - "final_path": settings.get("final_path", ""), - "install_dir": settings.get("install_dir", ""), - "YNH_APP_BASEDIR": os.path.join(APPS_SETTING_PATH, app), - "YNH_APP_PACKAGING_FORMAT": str(manifest["packaging_format"]), - "YNH_APP_CONFIG_PANEL_OPTIONS_TYPES_AND_BINDS": self._dump_options_types_and_binds(), - } - ) - app_script_env = _make_environment_for_app_script(app) - # Note that we only need to update settings wich are not already set - # The settings from config panel should be keep as it is - app_script_env.update(env) - env = app_script_env - - ret, values = hook_exec(config_script, args=[action], env=env) - if ret != 0: - if action == "show": - raise YunohostError("app_config_unable_to_read") - elif action == "apply": - raise YunohostError("app_config_unable_to_apply") - else: - raise YunohostError("app_action_failed", action=action, app=app) - return values - - def _get_config_panel(self): - - ret = super()._get_config_panel() - - self._compute_binds() + write_to_file(config_script, default_script) + + # Call config script to extract current values + logger.debug(f"Calling '{action}' action from config script") + app = self.entity + app_setting_path = os.path.join(APPS_SETTING_PATH, self.entity) + app_script_env = _make_environment_for_app_script(app, workdir=app_setting_path) + app_script_env.update(env) + app_script_env["YNH_APP_CONFIG_PANEL_OPTIONS_TYPES_AND_BINDS"] = self._dump_options_types_and_binds() + + ret, values = hook_exec(config_script, args=[action], env=app_script_env) + if ret != 0: + if action == "show": + raise YunohostError("app_config_unable_to_read") + elif action == "apply": + raise YunohostError("app_config_unable_to_apply") + else: + raise YunohostError("app_action_failed", action=action, app=app) + return values - return ret + def _get_partial_raw_config(self): - def _compute_binds(self): - """ - This compute the 'bind' statement for every option - In particular to handle __FOOBAR__ syntax - and to handle the fact that bind statements may be defined panel-wide or section-wide - """ + raw_config = super()._get_partial_raw_config() - settings = _get_app_settings(self.entity) + self._compute_binds(raw_config) - for panel, section, option in self._iterate(): + return raw_config - bind_panel = panel.get("bind") + def _compute_binds(self, raw_config): + """ + This compute the 'bind' statement for every option + In particular to handle __FOOBAR__ syntax + and to handle the fact that bind statements may be defined panel-wide or section-wide + """ - bind_section = section.get("bind") - if not bind_section: - bind_section = bind_panel - elif bind_section[-1] == ":" and bind_panel and ":" in bind_panel: - selector, bind_panel_file = bind_panel.split(":") - if ">" in bind_section: - bind_section = bind_section + bind_panel_file - else: - bind_section = selector + bind_section + bind_panel_file + settings = _get_app_settings(self.entity) - bind = option.get("bind") - if not bind: - if bind_section: - bind = bind_section - else: - bind = "settings" - elif bind[-1] == ":" and bind_section and ":" in bind_section: - selector, bind_file = bind_section.split(":") - if ">" in bind: - bind = bind + bind_file - else: - bind = selector + bind + bind_file - if bind == "settings" and option.get("type", "string") == "file": - bind = "null" - - option["bind"] = _hydrate_app_template(bind, settings) + for panel_id, panel in raw_config.items(): + if not isinstance(panel, dict): + continue + bind_panel = panel.get("bind") + for section_id, section in panel.items(): + if not isinstance(section, dict): + continue + bind_section = section.get("bind") + if not bind_section: + bind_section = bind_panel + elif bind_section[-1] == ":" and bind_panel and ":" in bind_panel: + selector, bind_panel_file = bind_panel.split(":") + if ">" in bind_section: + bind_section = bind_section + bind_panel_file + else: + bind_section = selector + bind_section + bind_panel_file + for option_id, option in section.items(): + if not isinstance(option, dict): + continue + bind = option.get("bind") + if not bind: + if bind_section: + bind = bind_section + else: + bind = "settings" + elif bind[-1] == ":" and bind_section and ":" in bind_section: + selector, bind_file = bind_section.split(":") + if ">" in bind: + bind = bind + bind_file + else: + bind = selector + bind + bind_file + if bind == "settings" and option.get("type", "string") == "file": + bind = "null" + if option.get("type", "string") == "button": + bind = "null" + + option["bind"] = _hydrate_app_template(bind, settings) + + def _dump_options_types_and_binds(self): + raw_config = self._get_partial_raw_config() + lines = [] + for panel_id, panel in raw_config.items(): + if not isinstance(panel, dict): + continue + for section_id, section in panel.items(): + if not isinstance(section, dict): + continue + for option_id, option in section.items(): + if not isinstance(option, dict): + continue + lines.append( + "|".join([option_id, option.get("type", "string"), option["bind"]]) + ) + return "\n".join(lines) - def _dump_options_types_and_binds(self): - lines = [] - for _, _, option in self._iterate(): - lines.append( - "|".join([option["id"], option.get("type", "string"), option["bind"]]) - ) - return "\n".join(lines) + return AppConfigPanel app_settings_cache: Dict[str, Dict[str, Any]] = {} @@ -2116,20 +2039,6 @@ def _get_app_settings(app: str) -> Dict[str, Any]: logger.error(m18n.n("app_not_correctly_installed", app=app)) return {} - # Stupid fix for legacy bullshit - # In the past, some setups did not have proper normalization for app domain/path - # Meaning some setups (as of January 2021) still have path=/foobar/ (with a trailing slash) - # resulting in stupid issue unless apps using ynh_app_normalize_path_stuff - # So we yolofix the settings if such an issue is found >_> - # A simple call to `yunohost app list` (which happens quite often) should be enough - # to migrate all app settings ... so this can probably be removed once we're past Bullseye... - if settings.get("path") != "/" and ( - settings.get("path", "").endswith("/") - or not settings.get("path", "/").startswith("/") - ): - settings["path"] = "/" + settings["path"].strip("/") - _set_app_settings(app, settings) - # Make the app id available as $app too settings["app"] = app @@ -2165,6 +2074,20 @@ def _set_app_settings(app, settings): del app_settings_cache[app] +def _parse_app_version(v): + + if v in ["?", "-"]: + return (0, 0) + + try: + if "~" in v: + return (version.parse(v.split("~")[0]), int(v.split("~")[1].replace("ynh", ""))) + else: + return (version.parse(v), 0) + except Exception as e: + raise YunohostError(f"Failed to parse app version '{v}' : {e}", raw_msg=True) + + def _get_manifest_of_app(path): "Get app manifest stored in json or in toml" @@ -2987,6 +2910,7 @@ def _get_conflicting_apps(domain, path, ignore_app=None): """ from yunohost.domain import _assert_domain_exists + from yunohost.utils.form import DomainOption, WebPathOption domain = DomainOption.normalize(domain) path = WebPathOption.normalize(path) @@ -3043,6 +2967,7 @@ def _make_environment_for_app_script( app_id, app_instance_nb = _parse_app_instance_name(app) env_dict = { + "YNH_DEFAULT_PHP_VERSION": "8.2", "YNH_APP_ID": app_id, "YNH_APP_INSTANCE_NAME": app, "YNH_APP_INSTANCE_NUMBER": str(app_instance_nb), @@ -3207,27 +3132,12 @@ def _assert_system_is_sane_for_app(manifest, when): logger.debug("Checking that required services are up and running...") - services = manifest.get("services", []) - - # Some apps use php-fpm, php5-fpm or php7.x-fpm which is now php7.4-fpm - def replace_alias(service): - if service in ["php-fpm", "php5-fpm", "php7.0-fpm", "php7.3-fpm"]: - return "php7.4-fpm" - else: - return service - - services = [replace_alias(s) for s in services] + # FIXME: in the past we had more elaborate checks about mariadb/php/postfix + # though they werent very formalized. Ideally we should rework this in the + # context of packaging v2, which implies deriving what services are + # relevant to check from the manifst - # We only check those, mostly to ignore "custom" services - # (added by apps) and because those are the most popular - # services - service_filter = ["nginx", "php7.4-fpm", "mysql", "postfix"] - services = [str(s) for s in services if s in service_filter] - - if "nginx" not in services: - services = ["nginx"] + services - if "fail2ban" not in services: - services.append("fail2ban") + services = ["nginx", "fail2ban"] # Wait if a service is reloading test_nb = 0 @@ -3297,12 +3207,7 @@ def _notification_is_dismissed(name, settings): def _filter_and_hydrate_notifications(notifications, current_version=None, data={}): def is_version_more_recent_than_current_version(name, current_version): current_version = str(current_version) - # Boring code to handle the fact that "0.1 < 9999~ynh1" is False - - if "~" in name: - return version.parse(name) > version.parse(current_version) - else: - return version.parse(name) > version.parse(current_version.split("~")[0]) + return _parse_app_version(name) > _parse_app_version(current_version) out = { # Should we render the markdown maybe? idk @@ -3381,7 +3286,7 @@ def regen_mail_app_user_config_for_dovecot_and_postfix(only=None): dovecot = True if only in [None, "dovecot"] else False postfix = True if only in [None, "postfix"] else False - from yunohost.user import _hash_user_password + from yunohost.utils.password import _hash_user_password postfix_map = [] dovecot_passwd = [] diff --git a/src/app_catalog.py b/src/app_catalog.py index 00419be6a3..2f3076eed2 100644 --- a/src/app_catalog.py +++ b/src/app_catalog.py @@ -19,28 +19,28 @@ import os import re import hashlib +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.network import download_json from moulinette.utils.filesystem import ( read_json, read_yaml, write_to_json, - write_to_yaml, mkdir, ) from yunohost.utils.i18n import _value_for_locale from yunohost.utils.error import YunohostError -logger = getActionLogger("yunohost.app_catalog") +logger = getLogger("yunohost.app_catalog") APPS_CATALOG_CACHE = "/var/cache/yunohost/repo" APPS_CATALOG_LOGOS = "/usr/share/yunohost/applogos" APPS_CATALOG_CONF = "/etc/yunohost/apps_catalog.yml" APPS_CATALOG_API_VERSION = 3 APPS_CATALOG_DEFAULT_URL = "https://app.yunohost.org/default" +DEFAULT_APPS_CATALOG_LIST = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}] def app_catalog(full=False, with_categories=False, with_antifeatures=False): @@ -120,33 +120,21 @@ def app_search(string): return matching_apps -def _initialize_apps_catalog_system(): - """ - This function is meant to intialize the apps_catalog system with YunoHost's default app catalog. - """ - - default_apps_catalog_list = [{"id": "default", "url": APPS_CATALOG_DEFAULT_URL}] - - try: - logger.debug( - "Initializing apps catalog system with YunoHost's default app list" - ) - write_to_yaml(APPS_CATALOG_CONF, default_apps_catalog_list) - except Exception as e: - raise YunohostError( - f"Could not initialize the apps catalog system... : {e}", raw_msg=True - ) - - logger.success(m18n.n("apps_catalog_init_success")) - - def _read_apps_catalog_list(): """ Read the json corresponding to the list of apps catalogs """ + if not os.path.exists(APPS_CATALOG_CONF): + return DEFAULT_APPS_CATALOG_LIST + try: list_ = read_yaml(APPS_CATALOG_CONF) + if list_ == DEFAULT_APPS_CATALOG_LIST: + try: + os.remove(APPS_CATALOG_CONF) + except Exception: + pass # Support the case where file exists but is empty # by returning [] if list_ is None return list_ if list_ else [] diff --git a/src/authenticators/ldap_admin.py b/src/authenticators/ldap_admin.py index b5f9eed070..36bde54526 100644 --- a/src/authenticators/ldap_admin.py +++ b/src/authenticators/ldap_admin.py @@ -16,11 +16,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # +import jwt import os import logging import ldap import ldap.sasl import time +import hashlib +from pathlib import Path from moulinette import m18n from moulinette.authentication import BaseAuthenticator @@ -29,14 +32,32 @@ from yunohost.utils.error import YunohostError, YunohostAuthenticationError from yunohost.utils.ldap import _get_ldap_interface -session_secret = random_ascii() logger = logging.getLogger("yunohost.authenticators.ldap_admin") + +def SESSION_SECRET(): + # Only load this once actually requested to avoid boring issues like + # "secret doesnt exists yet" (before postinstall) and therefore service + # miserably fail to start + if not SESSION_SECRET.value: + SESSION_SECRET.value = open("/etc/yunohost/.admin_cookie_secret").read().strip() + assert SESSION_SECRET.value + return SESSION_SECRET.value + + +SESSION_SECRET.value = None # type: ignore +SESSION_FOLDER = "/var/cache/yunohost/sessions" +SESSION_VALIDITY = 3 * 24 * 3600 # 3 days + LDAP_URI = "ldap://localhost:389" ADMIN_GROUP = "cn=admins,ou=groups" AUTH_DN = "uid={uid},ou=users,dc=yunohost,dc=org" +def short_hash(data): + return hashlib.shake_256(data.encode()).hexdigest(20) + + class Authenticator(BaseAuthenticator): name = "ldap_admin" @@ -122,55 +143,87 @@ def _reconnect(): if con: con.unbind_s() + return {"user": uid} + def set_session_cookie(self, infos): from bottle import response assert isinstance(infos, dict) + assert "user" in infos - # This allows to generate a new session id or keep the existing one - current_infos = self.get_session_cookie(raise_if_no_session_exists=False) - new_infos = {"id": current_infos["id"]} - new_infos.update(infos) + # Create a session id, built as + some random ascii + # Prefixing with the user hash is meant to provide the ability to invalidate all this user's session + # (eg because the user gets deleted, or password gets changed) + # User hashing not really meant for security, just to sort of anonymize/pseudonymize the session file name + infos["id"] = short_hash(infos['user']) + random_ascii(20) response.set_cookie( "yunohost.admin", - new_infos, + jwt.encode(infos, SESSION_SECRET(), algorithm="HS256"), secure=True, - secret=session_secret, httponly=True, - # samesite="strict", # Bottle 0.12 doesn't support samesite, to be added in next versions + path="/yunohost/api", + samesite="strict", ) + # Create the session file (expiration mechanism) + session_file = f'{SESSION_FOLDER}/{infos["id"]}' + os.system(f'touch "{session_file}"') + def get_session_cookie(self, raise_if_no_session_exists=True): - from bottle import request + from bottle import request, response try: - # N.B. : here we implicitly reauthenticate the cookie - # because it's signed via the session_secret - # If no session exists (or if session is invalid?) - # it's gonna return the default empty dict, - # which we interpret as an authentication failure - infos = request.get_cookie( - "yunohost.admin", secret=session_secret, default={} + token = request.get_cookie("yunohost.admin", default="").encode() + infos = jwt.decode( + token, + SESSION_SECRET(), + algorithms="HS256", + options={"require": ["id", "user"]}, ) except Exception: - if not raise_if_no_session_exists: - return {"id": random_ascii()} raise YunohostAuthenticationError("unable_authenticate") - if not infos and raise_if_no_session_exists: + if not infos: raise YunohostAuthenticationError("unable_authenticate") - if "id" not in infos: - infos["id"] = random_ascii() + self.purge_expired_session_files() + session_file = f'{SESSION_FOLDER}/{infos["id"]}' + if not os.path.exists(session_file): + response.delete_cookie("yunohost.admin", path="/yunohost/api") + raise YunohostAuthenticationError("session_expired") - # FIXME: Here, maybe we want to re-authenticate the session via the authenticator - # For example to check that the username authenticated is still in the admin group... + # Otherwise, we 'touch' the file to extend the validity + os.system(f'touch "{session_file}"') return infos def delete_session_cookie(self): from bottle import response - response.set_cookie("yunohost.admin", "", max_age=-1) - response.delete_cookie("yunohost.admin") + try: + infos = self.get_session_cookie() + session_file = f'{SESSION_FOLDER}/{infos["id"]}' + os.remove(session_file) + except Exception as e: + logger.debug(f"User logged out, but failed to properly invalidate the session : {e}") + + response.delete_cookie("yunohost.admin", path="/yunohost/api") + + def purge_expired_session_files(self): + + for session_file in Path(SESSION_FOLDER).iterdir(): + if abs(session_file.stat().st_mtime - time.time()) > SESSION_VALIDITY: + try: + session_file.unlink() + except Exception as e: + logger.debug(f"Failed to delete session file {session_file} ? {e}") + + @staticmethod + def invalidate_all_sessions_for_user(user): + + for file in Path(SESSION_FOLDER).glob(f"{short_hash(user)}*"): + try: + file.unlink() + except Exception as e: + logger.debug(f"Failed to delete session file {file} ? {e}") diff --git a/src/authenticators/ldap_ynhuser.py b/src/authenticators/ldap_ynhuser.py new file mode 100644 index 0000000000..3407c3b02d --- /dev/null +++ b/src/authenticators/ldap_ynhuser.py @@ -0,0 +1,332 @@ +# -*- coding: utf-8 -*- + +import time +import jwt +import logging +import ldap +import ldap.sasl +import base64 +import os +import hashlib +from pathlib import Path + +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.backends import default_backend + +from moulinette import m18n +from moulinette.authentication import BaseAuthenticator +from moulinette.utils.text import random_ascii +from moulinette.utils.filesystem import read_json +from yunohost.utils.error import YunohostError, YunohostAuthenticationError +from yunohost.utils.ldap import _get_ldap_interface + +logger = logging.getLogger("yunohostportal.authenticators.ldap_ynhuser") + + +def SESSION_SECRET(): + # Only load this once actually requested to avoid boring issues like + # "secret doesnt exists yet" (before postinstall) and therefore service + # miserably fail to start + if not SESSION_SECRET.value: + SESSION_SECRET.value = open("/etc/yunohost/.ssowat_cookie_secret").read().strip() + assert SESSION_SECRET.value + return SESSION_SECRET.value + + +SESSION_SECRET.value = None # type: ignore +SESSION_FOLDER = "/var/cache/yunohost-portal/sessions" +SESSION_VALIDITY = 3 * 24 * 3600 # 3 days + +URI = "ldap://localhost:389" +USERDN = "uid={username},ou=users,dc=yunohost,dc=org" + +# Cache on-disk settings to RAM for faster access +DOMAIN_USER_ACL_DICT: dict[str, dict] = {} +PORTAL_SETTINGS_DIR = "/etc/yunohost/portal" + + +# Should a user have *minimal* access to a domain? +# - if the user has permission for an application with a URI on the domain, yes +# - if the user is an admin, yes +# - if the user has an email on the domain, yes +# - otherwise, no +def user_is_allowed_on_domain(user: str, domain: str) -> bool: + + assert "/" not in domain + + portal_settings_path = Path(PORTAL_SETTINGS_DIR) / f"{domain}.json" + + if not portal_settings_path.exists(): + if "." not in domain: + return False + parent_domain = domain.split(".", 1)[-1] + return user_is_allowed_on_domain(user, parent_domain) + + # Check that the domain permissions haven't changed on-disk since we read them + # by comparing file mtime. If we haven't read the file yet, read it for the first time. + # We compare mtime by equality not superiority because maybe the system clock has changed. + mtime = portal_settings_path.stat().st_mtime + if domain not in DOMAIN_USER_ACL_DICT or DOMAIN_USER_ACL_DICT[domain]["mtime"] != mtime: + users: set[str] = set() + for infos in read_json(str(portal_settings_path))["apps"].values(): + users = users.union(infos["users"]) + DOMAIN_USER_ACL_DICT[domain] = {} + DOMAIN_USER_ACL_DICT[domain]["mtime"] = mtime + DOMAIN_USER_ACL_DICT[domain]["users"] = users + + if user in DOMAIN_USER_ACL_DICT[domain]["users"]: + # A user with explicit permission to an application is certainly welcome + return True + + ADMIN_GROUP = "cn=admins,ou=groups" + try: + admins = ( + _get_ldap_interface() + .search(ADMIN_GROUP, attrs=["memberUid"])[0] + .get("memberUid", []) + ) + except Exception as e: + logger.error(f"Failed to list admin users: {e}") + return False + if user in admins: + # Admins can access everything + return True + + try: + user_result = _get_ldap_interface().search("ou=users", f"uid={user}", ["mail"]) + if len(user_result) != 1: + logger.error(f"User not found or many users found for {user}. How is this possible after so much validation?") + return False + + user_mail = user_result[0]["mail"] + if len(user_mail) != 1: + logger.error(f"User {user} found, but has the wrong number of email addresses: {user_mail}") + return False + + user_mail = user_mail[0] + if "@" not in user_mail: + logger.error(f"Invalid email address for {user}: {user_mail}") + return False + + if user_mail.split("@")[1] == domain: + # A user from that domain is welcome + return True + + # Users from other domains don't belong here + return False + except Exception as e: + logger.error(f"Failed to get email info for {user}: {e}") + return False + + +# We want to save the password in the cookie, but we should do so in an encrypted fashion +# This is needed because the SSO later needs to possibly inject the Basic Auth header +# which includes the user's password +# It's also needed because we need to be able to open LDAP sessions, authenticated as the user, +# which requires the user's password +# +# To do so, we use AES-256-CBC. As it's a block encryption algorithm, it requires an IV, +# which we need to keep around for decryption on SSOwat'side. +# +# SESSION_SECRET is used as the encryption key, which implies it must be exactly 32-char long (256/8) +# +# The result is a string formatted as | +# For example: ctl8kk5GevYdaA5VZ2S88Q==|yTAzCx0Gd1+MCit4EQl9lA== +def encrypt(data): + alg = algorithms.AES(SESSION_SECRET().encode()) + iv = os.urandom(int(alg.block_size / 8)) + + E = Cipher(alg, modes.CBC(iv), default_backend()).encryptor() + p = padding.PKCS7(alg.block_size).padder() + data_padded = p.update(data.encode()) + p.finalize() + data_enc = E.update(data_padded) + E.finalize() + data_enc_b64 = base64.b64encode(data_enc).decode() + iv_b64 = base64.b64encode(iv).decode() + return data_enc_b64 + "|" + iv_b64 + + +def decrypt(data_enc_and_iv_b64): + data_enc_b64, iv_b64 = data_enc_and_iv_b64.split("|") + data_enc = base64.b64decode(data_enc_b64) + iv = base64.b64decode(iv_b64) + + alg = algorithms.AES(SESSION_SECRET().encode()) + D = Cipher(alg, modes.CBC(iv), default_backend()).decryptor() + p = padding.PKCS7(alg.block_size).unpadder() + data_padded = D.update(data_enc) + data = p.update(data_padded) + p.finalize() + return data.decode() + + +def short_hash(data): + return hashlib.shake_256(data.encode()).hexdigest(20) + + +class Authenticator(BaseAuthenticator): + name = "ldap_ynhuser" + + def _authenticate_credentials(self, credentials=None): + from bottle import request + + try: + username, password = credentials.split(":", 1) + except ValueError: + raise YunohostError("invalid_credentials") + + def _reconnect(): + con = ldap.ldapobject.ReconnectLDAPObject(URI, retry_max=2, retry_delay=0.5) + con.simple_bind_s(USERDN.format(username=username), password) + return con + + try: + con = _reconnect() + except ldap.INVALID_CREDENTIALS: + # FIXME FIXME FIXME : this should be properly logged and caught by Fail2ban ! ! ! ! ! ! ! + raise YunohostError("invalid_password") + except ldap.SERVER_DOWN: + logger.warning(m18n.n("ldap_server_down")) + + # Check that we are indeed logged in with the expected identity + try: + # whoami_s return dn:..., then delete these 3 characters + who = con.whoami_s()[3:] + except Exception as e: + logger.warning("Error during ldap authentication process: %s", e) + raise + else: + if who != USERDN.format(username=username): + raise YunohostError( + "Not logged with the appropriate identity ?!", + raw_msg=True, + ) + finally: + # Free the connection, we don't really need it to keep it open as the point is only to check authentication... + if con: + con.unbind_s() + + ldap_user_infos = _get_ldap_interface().search("ou=users", f"uid={username}", attrs=["cn", "mail"])[0] + + if not user_is_allowed_on_domain(username, request.get_header("host")): + raise YunohostAuthenticationError("unable_authenticate") + + return {"user": username, + "pwd": encrypt(password), + "email": ldap_user_infos["mail"][0], + "fullname": ldap_user_infos["cn"][0] + } + + def set_session_cookie(self, infos): + from bottle import response, request + + assert isinstance(infos, dict) + assert "user" in infos + assert "pwd" in infos + assert "email" in infos + assert "fullname" in infos + + # Create a session id, built as + some random ascii + # Prefixing with the user hash is meant to provide the ability to invalidate all this user's session + # (eg because the user gets deleted, or password gets changed) + # User hashing not really meant for security, just to sort of anonymize/pseudonymize the session file name + infos["id"] = short_hash(infos['user']) + random_ascii(20) + infos["host"] = request.get_header("host") + + is_dev = Path("/etc/yunohost/.portal-api-allowed-cors-origins").exists() + + response.set_cookie( + "yunohost.portal", + jwt.encode(infos, SESSION_SECRET(), algorithm="HS256"), + secure=True, + httponly=True, + path="/", + samesite="strict" if not is_dev else None, + domain=f".{request.get_header('host')}", + max_age=SESSION_VALIDITY - 600 # remove 1 minute such that cookie expires on the browser slightly sooner on browser side, just to help desimbuigate edge case near the expiration limit + ) + + # Create the session file (expiration mechanism) + session_file = f'{SESSION_FOLDER}/{infos["id"]}' + os.system(f'touch "{session_file}"') + + def get_session_cookie(self, decrypt_pwd=False): + from bottle import request, response + + try: + token = request.get_cookie("yunohost.portal", default="").encode() + infos = jwt.decode( + token, + SESSION_SECRET(), + algorithms="HS256", + options={"require": ["id", "host", "user", "pwd"]}, + ) + except Exception: + raise YunohostAuthenticationError("unable_authenticate") + + if not infos: + raise YunohostAuthenticationError("unable_authenticate") + + if infos["host"] != request.get_header("host"): + raise YunohostAuthenticationError("unable_authenticate") + + if not user_is_allowed_on_domain(infos["user"], infos["host"]): + raise YunohostAuthenticationError("unable_authenticate") + + self.purge_expired_session_files() + session_file = Path(SESSION_FOLDER) / infos["id"] + if not session_file.exists(): + response.delete_cookie("yunohost.portal", path="/") + raise YunohostAuthenticationError("session_expired") + + # Otherwise, we 'touch' the file to extend the validity + session_file.touch() + + is_dev = Path("/etc/yunohost/.portal-api-allowed-cors-origins").exists() + + # We also re-set the cookie such that validity is also extended on browser side + response.set_cookie( + "yunohost.portal", + request.get_cookie("yunohost.portal"), # Reuse the same token to avoid recomputing stuff (saves a bit of CPU / delay I suppose?) + secure=True, + httponly=True, + path="/", + samesite="strict" if not is_dev else None, + domain=f".{request.get_header('host')}", + max_age=SESSION_VALIDITY - 600 # remove 1 minute such that cookie expires on the browser slightly sooner on browser side, just to help desimbuigate edge case near the expiration limit + ) + + if decrypt_pwd: + infos["pwd"] = decrypt(infos["pwd"]) + + return infos + + def delete_session_cookie(self): + from bottle import response + + try: + infos = self.get_session_cookie() + session_file = Path(SESSION_FOLDER) / infos["id"] + session_file.unlink() + except Exception as e: + logger.debug(f"User logged out, but failed to properly invalidate the session : {e}") + + response.delete_cookie("yunohost.portal", path="/") + + def purge_expired_session_files(self): + + for session_file in Path(SESSION_FOLDER).iterdir(): + print(session_file.stat().st_mtime - time.time()) + if abs(session_file.stat().st_mtime - time.time()) > SESSION_VALIDITY: + try: + session_file.unlink() + except Exception as e: + logger.debug(f"Failed to delete session file {session_file} ? {e}") + + @staticmethod + def invalidate_all_sessions_for_user(user): + + for file in Path(SESSION_FOLDER).glob(f"{short_hash(user)}*"): + try: + file.unlink() + except Exception as e: + logger.debug(f"Failed to delete session file {file} ? {e}") diff --git a/src/backup.py b/src/backup.py index b9476f0a54..9fd3a266bb 100644 --- a/src/backup.py +++ b/src/backup.py @@ -30,10 +30,10 @@ from collections import OrderedDict from functools import reduce from packaging import version +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.utils.text import random_ascii -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, mkdir, @@ -78,7 +78,6 @@ binary_to_human, space_used_by_directory, ) -from yunohost.settings import settings_get BACKUP_PATH = "/home/yunohost.backup" ARCHIVES_PATH = f"{BACKUP_PATH}/archives" @@ -86,7 +85,7 @@ CONF_MARGIN_SPACE_SIZE = 10 # IN MB POSTINSTALL_ESTIMATE_SPACE_SIZE = 5 # In MB MB_ALLOWED_TO_ORGANIZE = 10 -logger = getActionLogger("yunohost.backup") +logger = getLogger("yunohost.backup") class BackupRestoreTargetsManager: @@ -1201,9 +1200,6 @@ def restore(self): try: self._postinstall_if_needed() - # Apply dirty patch to redirect php5 file on php7 - self._patch_legacy_php_versions_in_csv_file() - self._restore_system() self._restore_apps() except Exception as e: @@ -1214,39 +1210,6 @@ def restore(self): finally: self.clean() - def _patch_legacy_php_versions_in_csv_file(self): - """ - Apply dirty patch to redirect php5 and php7.0 files to php7.4 - """ - from yunohost.utils.legacy import LEGACY_PHP_VERSION_REPLACEMENTS - - backup_csv = os.path.join(self.work_dir, "backup.csv") - - if not os.path.isfile(backup_csv): - return - - replaced_something = False - with open(backup_csv) as csvfile: - reader = csv.DictReader(csvfile, fieldnames=["source", "dest"]) - newlines = [] - for row in reader: - for pattern, replace in LEGACY_PHP_VERSION_REPLACEMENTS: - if pattern in row["source"]: - replaced_something = True - row["source"] = row["source"].replace(pattern, replace) - - newlines.append(row) - - if not replaced_something: - return - - with open(backup_csv, "w") as csvfile: - writer = csv.DictWriter( - csvfile, fieldnames=["source", "dest"], quoting=csv.QUOTE_ALL - ) - for row in newlines: - writer.writerow(row) - def _restore_system(self): """Restore user and system parts""" @@ -1383,8 +1346,6 @@ def _restore_app(self, app_instance_name): name should be already install) """ from yunohost.utils.legacy import ( - _patch_legacy_php_versions, - _patch_legacy_php_versions_in_settings, _patch_legacy_helpers, ) from yunohost.user import user_group_list @@ -1423,10 +1384,6 @@ def copytree(src, dst, symlinks=False, ignore=None): # Attempt to patch legacy helpers... _patch_legacy_helpers(app_settings_in_archive) - # Apply dirty patch to make php5 apps compatible with php7 - _patch_legacy_php_versions(app_settings_in_archive) - _patch_legacy_php_versions_in_settings(app_settings_in_archive) - # Delete _common.sh file in backup common_file = os.path.join(app_backup_in_archive, "_common.sh") rm(common_file, force=True) @@ -1950,6 +1907,8 @@ class TarBackupMethod(BackupMethod): @property def _archive_file(self): + from yunohost.settings import settings_get + if isinstance(self.manager, RestoreManager): return self.manager.archive_path @@ -2581,9 +2540,6 @@ def backup_info(name, with_details=False, human_readable=False): for category in ["apps", "system"]: for name, key_info in info[category].items(): if category == "system": - # Stupid legacy fix for weird format between 3.5 and 3.6 - if isinstance(key_info, dict): - key_info = key_info.keys() info[category][name] = key_info = {"paths": key_info} else: info[category][name] = key_info diff --git a/src/certificate.py b/src/certificate.py index e31b991e50..00f653f2e2 100644 --- a/src/certificate.py +++ b/src/certificate.py @@ -21,11 +21,10 @@ import shutil import subprocess from glob import glob - +from logging import getLogger from datetime import datetime from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, chown, chmod from moulinette.utils.process import check_output @@ -38,7 +37,7 @@ from yunohost.regenconf import regen_conf from yunohost.log import OperationLogger -logger = getActionLogger("yunohost.certmanager") +logger = getLogger("yunohost.certmanager") CERT_FOLDER = "/etc/yunohost/certs/" TMP_FOLDER = "/var/www/.well-known/acme-challenge-private/" @@ -70,7 +69,11 @@ def certificate_status(domains, full=False): full -- Display more info about the certificates """ - from yunohost.domain import domain_list, _assert_domain_exists + from yunohost.domain import ( + domain_list, + _assert_domain_exists, + _get_parent_domain_of, + ) # If no domains given, consider all yunohost domains if domains == []: @@ -99,6 +102,16 @@ def certificate_status(domains, full=False): else: status["ACME_eligible"] = False + # Check if a wildcard is setup for the ipv4/ipv6 A/AAAA records on the topest domain + parent_domain = _get_parent_domain_of(domain, return_self=True, topest=True) + dns_extra = Diagnoser.get_cached_report( + "dnsrecords", item={"domain": parent_domain, "category": "extra"} + ).get("data", {}) + has_wildcards = [ + v == "OK" for k, v in dns_extra.items() if k.startswith("A") + ] + status["has_wildcards"] = len(has_wildcards) > 0 and all(has_wildcards) + del status["domain"] certificates[domain] = status @@ -557,6 +570,7 @@ def _fetch_and_enable_new_certificate(domain, no_checks=False): def _prepare_certificate_signing_request(domain, key_file, output_folder): from OpenSSL import crypto # lazy loading this module for performance reasons + from yunohost.hook import hook_callback # Init a request csr = crypto.X509Req() @@ -564,53 +578,34 @@ def _prepare_certificate_signing_request(domain, key_file, output_folder): # Set the domain csr.get_subject().CN = domain - from yunohost.domain import domain_config_get - - # If XMPP is enabled for this domain, add xmpp-upload and muc subdomains - # in subject alternate names - if domain_config_get(domain, key="feature.xmpp.xmpp") == 1: - subdomain = "xmpp-upload." + domain - xmpp_records = ( - Diagnoser.get_cached_report( - "dnsrecords", item={"domain": domain, "category": "xmpp"} - ).get("data") - or {} - ) - sanlist = [] - - # Handle the boring case where the domain is not the root of the dns zone etc... - from yunohost.dns import ( - _get_relative_name_for_dns_zone, - _get_dns_zone_for_domain, - ) - - base_dns_zone = _get_dns_zone_for_domain(domain) - basename = _get_relative_name_for_dns_zone(domain, base_dns_zone) - suffix = f".{basename}" if basename != "@" else "" - - for sub in ("xmpp-upload", "muc"): - subdomain = sub + "." + domain - if xmpp_records.get("CNAME:" + sub + suffix) == "OK": - sanlist.append(("DNS:" + subdomain)) - else: - logger.warning( - m18n.n( - "certmanager_warning_subdomain_dns_record", - subdomain=subdomain, - domain=domain, - ) + sanlist = [] + hook_results = hook_callback("cert_alternate_names", env={"domain": domain}) + for hook_name, results in hook_results.items(): + # + # There can be multiple results per hook name, so results look like + # {'/some/path/to/hook1': + # { 'state': 'succeed', + # 'stdreturn': ["foo", "bar"] + # }, + # '/some/path/to/hook2': + # { ... }, + # [...] + # + # Loop over the sub-results + for result in results.values(): + if result.get("stdreturn"): + sanlist += result["stdreturn"] + + if sanlist: + csr.add_extensions( + [ + crypto.X509Extension( + b"subjectAltName", + False, + (", ".join([f"DNS:{sub}.{domain}" for sub in sanlist])).encode("utf-8"), ) - - if sanlist: - csr.add_extensions( - [ - crypto.X509Extension( - b"subjectAltName", - False, - (", ".join(sanlist)).encode("utf-8"), - ) - ] - ) + ] + ) # Set the key with open(key_file, "rt") as f: @@ -744,15 +739,6 @@ def _enable_certificate(domain, new_cert_folder): logger.debug("Restarting services...") - for service in ("dovecot", "metronome"): - # Ugly trick to not restart metronome if it's not installed or no domain configured for XMPP - if service == "metronome" and ( - os.system("dpkg --list | grep -q 'ii *metronome'") != 0 - or not glob("/etc/metronome/conf.d/*.cfg.lua") - ): - continue - _run_service_command("restart", service) - if os.path.isfile("/etc/yunohost/installed"): # regen nginx conf to be sure it integrates OCSP Stapling # (We don't do this yet if postinstall is not finished yet) @@ -760,6 +746,7 @@ def _enable_certificate(domain, new_cert_folder): regen_conf(names=["nginx", "postfix"]) _run_service_command("reload", "nginx") + _run_service_command("restart", "dovecot") from yunohost.hook import hook_callback diff --git a/src/diagnosers/00-basesystem.py b/src/diagnosers/00-basesystem.py index bee2fcdb92..65e78783f9 100644 --- a/src/diagnosers/00-basesystem.py +++ b/src/diagnosers/00-basesystem.py @@ -19,9 +19,9 @@ import os import json import subprocess +import logging from typing import List -from moulinette.utils import log from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file, read_json, write_to_json from yunohost.diagnosis import Diagnoser @@ -31,7 +31,7 @@ system_arch, ) -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): diff --git a/src/diagnosers/10-ip.py b/src/diagnosers/10-ip.py index abee4add87..09199fb00e 100644 --- a/src/diagnosers/10-ip.py +++ b/src/diagnosers/10-ip.py @@ -19,9 +19,9 @@ import re import os import random +import logging from typing import List -from moulinette.utils import log from moulinette.utils.network import download_text from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_file @@ -30,7 +30,7 @@ from yunohost.utils.network import get_network_interfaces from yunohost.settings import settings_get -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): diff --git a/src/diagnosers/12-dnsrecords.py b/src/diagnosers/12-dnsrecords.py index d63e54c9c8..e6959898f2 100644 --- a/src/diagnosers/12-dnsrecords.py +++ b/src/diagnosers/12-dnsrecords.py @@ -18,11 +18,11 @@ # import os import re +import logging from typing import List from datetime import datetime, timedelta from publicsuffix2 import PublicSuffixList -from moulinette.utils import log from moulinette.utils.process import check_output from yunohost.utils.dns import ( @@ -39,7 +39,7 @@ _get_relative_name_for_dns_zone, ) -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): @@ -91,7 +91,7 @@ def check_domain(self, domain, is_main_domain): domain, include_empty_AAAA_if_no_ipv6=True ) - categories = ["basic", "mail", "xmpp", "extra"] + categories = ["basic", "mail", "extra"] for category in categories: records = expected_configuration[category] diff --git a/src/diagnosers/24-mail.py b/src/diagnosers/24-mail.py index 4c27af90d9..7ca5821476 100644 --- a/src/diagnosers/24-mail.py +++ b/src/diagnosers/24-mail.py @@ -19,11 +19,11 @@ import os import dns.resolver import re +import logging from typing import List from subprocess import CalledProcessError -from moulinette.utils import log from moulinette.utils.process import check_output from moulinette.utils.filesystem import read_yaml @@ -34,7 +34,7 @@ DEFAULT_DNS_BLACKLIST = "/usr/share/yunohost/dnsbl_list.yml" -logger = log.getActionLogger("yunohost.diagnosis") +logger = logging.getLogger("yunohost.diagnosis") class MyDiagnoser(Diagnoser): diff --git a/src/diagnosis.py b/src/diagnosis.py index 37d5c8d472..9f542ea47c 100644 --- a/src/diagnosis.py +++ b/src/diagnosis.py @@ -21,9 +21,9 @@ import time import glob from importlib import import_module +from logging import getLogger from moulinette import m18n, Moulinette -from moulinette.utils import log from moulinette.utils.filesystem import ( read_json, write_to_json, @@ -33,7 +33,7 @@ from yunohost.utils.error import YunohostError, YunohostValidationError -logger = log.getActionLogger("yunohost.diagnosis") +logger = getLogger("yunohost.diagnosis") DIAGNOSIS_CACHE = "/var/cache/yunohost/diagnosis/" DIAGNOSIS_CONFIG_FILE = "/etc/yunohost/diagnosis.yml" diff --git a/src/dns.py b/src/dns.py index c6a9036a56..9112dc6376 100644 --- a/src/dns.py +++ b/src/dns.py @@ -19,12 +19,11 @@ import os import re import time - +from logging import getLogger from difflib import SequenceMatcher from collections import OrderedDict from moulinette import m18n, Moulinette -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, write_to_file, read_toml, mkdir from yunohost.domain import ( @@ -38,11 +37,10 @@ from yunohost.utils.dns import dig, is_yunohost_dyndns_domain, is_special_use_tld from yunohost.utils.error import YunohostValidationError, YunohostError from yunohost.utils.network import get_public_ip -from yunohost.settings import settings_get from yunohost.log import is_unit_operation from yunohost.hook import hook_callback -logger = getActionLogger("yunohost.domain") +logger = getLogger("yunohost.domain") DOMAIN_REGISTRAR_LIST_PATH = "/usr/share/yunohost/registrar_list.toml" @@ -77,12 +75,6 @@ def domain_dns_suggest(domain): result += "\n{name} {ttl} IN {type} {value}".format(**record) result += "\n\n" - if dns_conf["xmpp"]: - result += "\n\n" - result += "; XMPP" - for record in dns_conf["xmpp"]: - result += "\n{name} {ttl} IN {type} {value}".format(**record) - if dns_conf["extra"]: result += "\n\n" result += "; Extra" @@ -90,7 +82,7 @@ def domain_dns_suggest(domain): result += "\n{name} {ttl} IN {type} {value}".format(**record) for name, record_list in dns_conf.items(): - if name not in ("basic", "xmpp", "mail", "extra") and record_list: + if name not in ("basic", "mail", "extra") and record_list: result += "\n\n" result += "; " + name for record in record_list: @@ -119,14 +111,6 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): # if ipv6 available {"type": "AAAA", "name": "@", "value": "valid-ipv6", "ttl": 3600}, ], - "xmpp": [ - {"type": "SRV", "name": "_xmpp-client._tcp", "value": "0 5 5222 domain.tld.", "ttl": 3600}, - {"type": "SRV", "name": "_xmpp-server._tcp", "value": "0 5 5269 domain.tld.", "ttl": 3600}, - {"type": "CNAME", "name": "muc", "value": "@", "ttl": 3600}, - {"type": "CNAME", "name": "pubsub", "value": "@", "ttl": 3600}, - {"type": "CNAME", "name": "vjud", "value": "@", "ttl": 3600} - {"type": "CNAME", "name": "xmpp-upload", "value": "@", "ttl": 3600} - ], "mail": [ {"type": "MX", "name": "@", "value": "10 domain.tld.", "ttl": 3600}, {"type": "TXT", "name": "@", "value": "\"v=spf1 a mx ip4:123.123.123.123 ipv6:valid-ipv6 -all\"", "ttl": 3600 }, @@ -146,9 +130,10 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): } """ + from yunohost.settings import settings_get + basic = [] mail = [] - xmpp = [] extra = [] ipv4 = get_public_ip() ipv6 = get_public_ip(6) @@ -211,29 +196,6 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): [f"_dmarc{suffix}", ttl, "TXT", '"v=DMARC1; p=none"'], ] - ######## - # XMPP # - ######## - if settings["xmpp"]: - xmpp += [ - [ - f"_xmpp-client._tcp{suffix}", - ttl, - "SRV", - f"0 5 5222 {domain}.", - ], - [ - f"_xmpp-server._tcp{suffix}", - ttl, - "SRV", - f"0 5 5269 {domain}.", - ], - [f"muc{suffix}", ttl, "CNAME", f"{domain}."], - [f"pubsub{suffix}", ttl, "CNAME", f"{domain}."], - [f"vjud{suffix}", ttl, "CNAME", f"{domain}."], - [f"xmpp-upload{suffix}", ttl, "CNAME", f"{domain}."], - ] - ######### # Extra # ######### @@ -259,10 +221,6 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): {"name": name, "ttl": ttl_, "type": type_, "value": value} for name, ttl_, type_, value in basic ], - "xmpp": [ - {"name": name, "ttl": ttl_, "type": type_, "value": value} - for name, ttl_, type_, value in xmpp - ], "mail": [ {"name": name, "ttl": ttl_, "type": type_, "value": value} for name, ttl_, type_, value in mail @@ -277,15 +235,8 @@ def _build_dns_conf(base_domain, include_empty_AAAA_if_no_ipv6=False): # Custom records # ################## - # Defined by custom hooks ships in apps for example ... - - # FIXME : this ain't practical for apps that may want to add - # custom dns records for a subdomain ... there's no easy way for - # an app to compare the base domain is the parent of the subdomain ? - # (On the other hand, in sep 2021, it looks like no app is using - # this mechanism...) - - hook_results = hook_callback("custom_dns_rules", args=[base_domain]) + # Defined by custom hooks shipped in apps for example ... + hook_results = hook_callback("custom_dns_rules", env={"base_domain": base_domain, "suffix": suffix}) for hook_name, results in hook_results.items(): # # There can be multiple results per hook name, so results look like @@ -512,11 +463,26 @@ def _get_relative_name_for_dns_zone(domain, base_dns_zone): def _get_registrar_config_section(domain): from lexicon.providers.auto import _relevant_provider_for_domain - registrar_infos = { - "name": m18n.n( - "registrar_infos" - ), # This is meant to name the config panel section, for proper display in the webadmin - } + registrar_infos = OrderedDict( + { + "name": m18n.n( + "registrar_infos" + ), # This is meant to name the config panel section, for proper display in the webadmin + "registrar": OrderedDict( + { + "readonly": True, + "visible": False, + "default": None, + } + ), + "infos": OrderedDict( + { + "type": "alert", + "style": "info", + } + ), + } + ) dns_zone = _get_dns_zone_for_domain(domain) @@ -529,31 +495,20 @@ def _get_registrar_config_section(domain): else: parent_domain_link = parent_domain - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "info", - "ask": m18n.n( - "domain_dns_registrar_managed_in_parent_domain", - parent_domain=parent_domain, - parent_domain_link=parent_domain_link, - ), - "value": "parent_domain", - } + registrar_infos["registrar"]["default"] = "parent_domain" + registrar_infos["infos"]["ask"] = m18n.n( + "domain_dns_registrar_managed_in_parent_domain", + parent_domain=parent_domain, + parent_domain_link=parent_domain_link, ) - return OrderedDict(registrar_infos) + return registrar_infos # TODO big project, integrate yunohost's dynette as a registrar-like provider # TODO big project, integrate other dyndns providers such as netlib.re, or cf the list of dyndns providers supported by cloudron... if is_yunohost_dyndns_domain(dns_zone): - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "success", - "ask": m18n.n("domain_dns_registrar_yunohost"), - "value": "yunohost", - } - ) + registrar_infos["registrar"]["default"] = "yunohost" + registrar_infos["infos"]["style"] = "success" + registrar_infos["infos"]["ask"] = m18n.n("domain_dns_registrar_yunohost") registrar_infos["recovery_password"] = OrderedDict( { "type": "password", @@ -561,36 +516,24 @@ def _get_registrar_config_section(domain): "default": "", } ) - return OrderedDict(registrar_infos) + + return registrar_infos + elif is_special_use_tld(dns_zone): - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "info", - "ask": m18n.n("domain_dns_conf_special_use_tld"), - "value": None, - } - ) + registrar_infos["infos"]["ask"] = m18n.n("domain_dns_conf_special_use_tld") + + return registrar_infos try: registrar = _relevant_provider_for_domain(dns_zone)[0] except ValueError: - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "warning", - "ask": m18n.n("domain_dns_registrar_not_supported"), - "value": None, - } - ) + registrar_infos["registrar"]["default"] = None + registrar_infos["infos"]["ask"] = m18n.n("domain_dns_registrar_not_supported") + registrar_infos["infos"]["style"] = "warning" else: - registrar_infos["registrar"] = OrderedDict( - { - "type": "alert", - "style": "info", - "ask": m18n.n("domain_dns_registrar_supported", registrar=registrar), - "value": registrar, - } + registrar_infos["registrar"]["default"] = registrar + registrar_infos["infos"]["ask"] = m18n.n( + "domain_dns_registrar_supported", registrar=registrar ) TESTED_REGISTRARS = ["ovh", "gandi"] @@ -613,12 +556,18 @@ def _get_registrar_config_section(domain): f"Registrar {registrar} unknown / Should be added to YunoHost's registrar_list.toml by the development team!" ) registrar_credentials = {} + else: + registrar_infos["use_auto_dns"] = { + "type": "boolean", + "ask": m18n.n("domain_dns_registrar_use_auto"), + "default": True + } for credential, infos in registrar_credentials.items(): infos["default"] = infos.get("default", "") - infos["optional"] = infos.get("optional", "False") + infos["visible"] = "use_auto_dns == true" registrar_infos.update(registrar_credentials) - return OrderedDict(registrar_infos) + return registrar_infos def _get_registar_settings(domain): @@ -648,8 +597,7 @@ def domain_dns_push(operation_logger, domain, dry_run=False, force=False, purge= _assert_domain_exists(domain) if is_special_use_tld(domain): - logger.info(m18n.n("domain_dns_conf_special_use_tld")) - return {} + raise YunohostValidationError("domain_dns_conf_special_use_tld") if not registrar or registrar == "None": # yes it's None as a string raise YunohostValidationError("domain_dns_push_not_applicable", domain=domain) diff --git a/src/domain.py b/src/domain.py index 8536337ba5..3291d48494 100644 --- a/src/domain.py +++ b/src/domain.py @@ -18,28 +18,35 @@ # import os import time -from typing import List, Optional +from pathlib import Path +from typing import TYPE_CHECKING, Any, List, Optional, Union from collections import OrderedDict +from logging import getLogger from moulinette import m18n, Moulinette from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import write_to_file, read_yaml, write_to_yaml, rm - -from yunohost.app import ( - app_ssowatconf, - _installed_apps, - _get_app_settings, - _get_conflicting_apps, +from moulinette.utils.filesystem import ( + read_json, + read_yaml, + rm, + read_file, + write_to_file, + write_to_json, + write_to_yaml, ) + from yunohost.regenconf import regen_conf, _force_clear_hashes, _process_regen_conf -from yunohost.utils.configpanel import ConfigPanel -from yunohost.utils.form import BaseOption from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.dns import is_yunohost_dyndns_domain from yunohost.log import is_unit_operation -logger = getActionLogger("yunohost.domain") +if TYPE_CHECKING: + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + + from yunohost.utils.configpanel import RawConfig + from yunohost.utils.form import FormModel, ConfigPanelModel + from yunohost.utils.configpanel import RawSettings + +logger = getLogger("yunohost.domain") DOMAIN_SETTINGS_DIR = "/etc/yunohost/domains" @@ -100,6 +107,30 @@ def cmp_domain(domain): return domain_list_cache +def _get_domain_portal_dict(): + + domains = _get_domains() + out = OrderedDict() + + for domain in domains: + + parent = None + + # Use the topest parent domain if any + for d in out.keys(): + if domain.endswith(f".{d}"): + parent = d + break + + out[domain] = f'{parent or domain}/yunohost/sso' + + # By default, redirect to $host/yunohost/admin for domains not listed in the dict + # maybe in the future, we can allow to tweak this + out["default"] = "/yunohost/admin" + + return dict(out) + + def domain_list(exclude_subdomains=False, tree=False, features=[]): """ List domains @@ -153,13 +184,14 @@ def domain_info(domain): domain -- Domain to be checked """ - from yunohost.app import app_info + from yunohost.app import app_info, _installed_apps, _get_app_settings from yunohost.dns import _get_registar_settings + from yunohost.certificate import certificate_status _assert_domain_exists(domain) registrar, _ = _get_registar_settings(domain) - certificate = domain_cert_status([domain], full=True)["certificates"][domain] + certificate = certificate_status([domain], full=True)["certificates"][domain] apps = [] for app in _installed_apps(): @@ -213,7 +245,12 @@ def _get_parent_domain_of(domain, return_self=False, topest=False): @is_unit_operation(exclude=["dyndns_recovery_password"]) def domain_add( - operation_logger, domain, dyndns_recovery_password=None, ignore_dyndns=False + operation_logger, + domain, + dyndns_recovery_password=None, + ignore_dyndns=False, + install_letsencrypt_cert=False, + skip_tos=False, ): """ Create a custom domain @@ -223,22 +260,22 @@ def domain_add( dyndns -- Subscribe to DynDNS dyndns_recovery_password -- Password used to later unsubscribe from DynDNS ignore_dyndns -- If we want to just add the DynDNS domain to the list, without subscribing + install_letsencrypt_cert -- If adding a subdomain of an already added domain, try to install a Let's Encrypt certificate """ from yunohost.hook import hook_callback from yunohost.app import app_ssowatconf from yunohost.utils.ldap import _get_ldap_interface from yunohost.utils.password import assert_password_is_strong_enough - from yunohost.certificate import _certificate_install_selfsigned + from yunohost.certificate import ( + _certificate_install_letsencrypt, + _certificate_install_selfsigned, + certificate_status, + ) + from yunohost.utils.dns import is_yunohost_dyndns_domain if dyndns_recovery_password: operation_logger.data_to_redact.append(dyndns_recovery_password) - if domain.startswith("xmpp-upload."): - raise YunohostValidationError("domain_cannot_add_xmpp_upload") - - if domain.startswith("muc."): - raise YunohostError("domain_cannot_add_muc_upload") - ldap = _get_ldap_interface() try: @@ -260,11 +297,17 @@ def domain_add( and len(domain.split(".")) == 3 ) if dyndns: + from yunohost.app import _ask_confirmation from yunohost.dyndns import is_subscribing_allowed # Do not allow to subscribe to multiple dyndns domains... if not is_subscribing_allowed(): raise YunohostValidationError("domain_dyndns_already_subscribed") + + if not skip_tos and Moulinette.interface.type == "cli" and os.isatty(1): + Moulinette.display(m18n.n("tos_dyndns_acknowledgement"), style="warning") + _ask_confirmation("confirm_tos_acknowledgement", kind="soft") + if dyndns_recovery_password: assert_password_is_strong_enough("admin", dyndns_recovery_password) @@ -275,7 +318,7 @@ def domain_add( domain=domain, recovery_password=dyndns_recovery_password ) - _certificate_install_selfsigned([domain], True) + _certificate_install_selfsigned([domain], force=True) try: attr_dict = { @@ -307,10 +350,8 @@ def domain_add( regen_conf( names=[ "nginx", - "metronome", "dnsmasq", "postfix", - "rspamd", "mdns", "dovecot", ] @@ -325,10 +366,32 @@ def domain_add( pass raise e + failed_letsencrypt_cert_install = False + if install_letsencrypt_cert: + parent_domain = _get_parent_domain_of(domain) + can_install_letsencrypt = ( + parent_domain + and certificate_status([parent_domain], full=True)["certificates"][ + parent_domain + ]["has_wildcards"] + ) + + if can_install_letsencrypt: + try: + _certificate_install_letsencrypt([domain], force=True, no_checks=True) + except Exception: + failed_letsencrypt_cert_install = True + else: + logger.warning("Skipping Let's Encrypt certificate attempt because there's no wildcard configured on the parent domain's DNS records.") + failed_letsencrypt_cert_install = True + hook_callback("post_domain_add", args=[domain]) logger.success(m18n.n("domain_created")) + if failed_letsencrypt_cert_install: + logger.warning(m18n.n("certmanager_cert_install_failed", domains=domain)) + @is_unit_operation(exclude=["dyndns_recovery_password"]) def domain_remove( @@ -352,8 +415,15 @@ def domain_remove( """ import glob from yunohost.hook import hook_callback - from yunohost.app import app_ssowatconf, app_info, app_remove + from yunohost.app import ( + app_ssowatconf, + app_info, + app_remove, + _get_app_settings, + _installed_apps, + ) from yunohost.utils.ldap import _get_ldap_interface + from yunohost.utils.dns import is_yunohost_dyndns_domain if dyndns_recovery_password: operation_logger.data_to_redact.append(dyndns_recovery_password) @@ -474,7 +544,7 @@ def domain_remove( f"/etc/nginx/conf.d/{domain}.conf", new_conf=None, save=True ) - regen_conf(names=["nginx", "metronome", "dnsmasq", "postfix", "rspamd", "mdns"]) + regen_conf(names=["nginx", "dnsmasq", "postfix", "mdns"]) app_ssowatconf() hook_callback("post_domain_remove", args=[domain]) @@ -560,9 +630,6 @@ def domain_main_domain(operation_logger, new_main_domain=None): logger.warning(str(e), exc_info=1) raise YunohostError("main_domain_change_failed") - # Generate SSOwat configuration file - app_ssowatconf() - # Regen configurations if os.path.exists("/etc/yunohost/installed"): regen_conf() @@ -585,9 +652,25 @@ def domain_url_available(domain, path): path -- The path to check (e.g. /coffee) """ + from yunohost.app import _get_conflicting_apps + return len(_get_conflicting_apps(domain, path)) == 0 +def _get_raw_domain_settings(domain): + """Get domain settings directly from file. + Be carefull, domain settings are saved in `"diff"` mode (i.e. default settings are not saved) + so the file may be completely empty + """ + _assert_domain_exists(domain) + # NB: this corresponds to save_path_tpl in DomainConfigPanel + path = f"{DOMAIN_SETTINGS_DIR}/{domain}.yml" + if os.path.exists(path): + return read_yaml(path) + + return {} + + def domain_config_get(domain, key="", full=False, export=False): """ Display a domain configuration @@ -605,6 +688,7 @@ def domain_config_get(domain, key="", full=False, export=False): else: mode = "classic" + DomainConfigPanel = _get_DomainConfigPanel() config = DomainConfigPanel(domain) return config.get(key, mode) @@ -616,161 +700,224 @@ def domain_config_set( """ Apply a new domain configuration """ + from yunohost.utils.form import BaseOption + + DomainConfigPanel = _get_DomainConfigPanel() BaseOption.operation_logger = operation_logger config = DomainConfigPanel(domain) return config.set(key, value, args, args_file, operation_logger=operation_logger) -class DomainConfigPanel(ConfigPanel): - entity_type = "domain" - save_path_tpl = f"{DOMAIN_SETTINGS_DIR}/{{entity}}.yml" - save_mode = "diff" +def _get_DomainConfigPanel(): + from yunohost.utils.configpanel import ConfigPanel + from yunohost.dns import _set_managed_dns_records_hashes - def get(self, key="", mode="classic"): - result = super().get(key=key, mode=mode) + class DomainConfigPanel(ConfigPanel): + entity_type = "domain" + save_path_tpl = f"{DOMAIN_SETTINGS_DIR}/{{entity}}.yml" + save_mode = "diff" - if mode == "full": - for panel, section, option in self._iterate(): - # This injects: - # i18n: domain_config_cert_renew_help - # i18n: domain_config_default_app_help - # i18n: domain_config_xmpp_help - if m18n.key_exists(self.config["i18n"] + "_" + option["id"] + "_help"): - option["help"] = m18n.n( - self.config["i18n"] + "_" + option["id"] + "_help" - ) - return self.config + # i18n: domain_config_cert_renew_help + # i18n: domain_config_default_app_help - return result + def _get_raw_config(self) -> "RawConfig": + # TODO add mechanism to share some settings with other domains on the same zone + raw_config = super()._get_raw_config() - def _get_raw_config(self): - toml = super()._get_raw_config() + any_filter = all(self.filter_key) + panel_id, section_id, option_id = self.filter_key - toml["feature"]["xmpp"]["xmpp"]["default"] = ( - 1 if self.entity == _get_maindomain() else 0 - ) + # Portal settings are only available on "topest" domains + if _get_parent_domain_of(self.entity, topest=True) is not None: + del raw_config["feature"]["portal"] - # Optimize wether or not to load the DNS section, - # e.g. we don't want to trigger the whole _get_registary_config_section - # when just getting the current value from the feature section - filter_key = self.filter_key.split(".") if self.filter_key != "" else [] - if not filter_key or filter_key[0] == "dns": - from yunohost.dns import _get_registrar_config_section + # Optimize wether or not to load the DNS section, + # e.g. we don't want to trigger the whole _get_registary_config_section + # when just getting the current value from the feature section + if not any_filter or panel_id == "dns": + from yunohost.dns import _get_registrar_config_section - toml["dns"]["registrar"] = _get_registrar_config_section(self.entity) + raw_config["dns"]["registrar"] = _get_registrar_config_section( + self.entity + ) - # FIXME: Ugly hack to save the registar id/value and reinject it in _get_raw_settings ... - self.registar_id = toml["dns"]["registrar"]["registrar"]["value"] - del toml["dns"]["registrar"]["registrar"]["value"] + # Cert stuff + if not any_filter or panel_id == "cert": + from yunohost.certificate import certificate_status - # Cert stuff - if not filter_key or filter_key[0] == "cert": - from yunohost.certificate import certificate_status + status = certificate_status([self.entity], full=True)["certificates"][ + self.entity + ] - status = certificate_status([self.entity], full=True)["certificates"][ - self.entity - ] + raw_config["cert"]["cert_"]["cert_summary"]["style"] = status["style"] - toml["cert"]["cert"]["cert_summary"]["style"] = status["style"] + # i18n: domain_config_cert_summary_expired + # i18n: domain_config_cert_summary_selfsigned + # i18n: domain_config_cert_summary_abouttoexpire + # i18n: domain_config_cert_summary_ok + # i18n: domain_config_cert_summary_letsencrypt + raw_config["cert"]["cert_"]["cert_summary"]["ask"] = m18n.n( + f"domain_config_cert_summary_{status['summary']}" + ) - # i18n: domain_config_cert_summary_expired - # i18n: domain_config_cert_summary_selfsigned - # i18n: domain_config_cert_summary_abouttoexpire - # i18n: domain_config_cert_summary_ok - # i18n: domain_config_cert_summary_letsencrypt - toml["cert"]["cert"]["cert_summary"]["ask"] = m18n.n( - f"domain_config_cert_summary_{status['summary']}" - ) + for option_id, status_key in [ + ("cert_validity", "validity"), + ("cert_issuer", "CA_type"), + ("acme_eligible", "ACME_eligible"), + # FIXME not sure why "summary" was injected in settings values + # ("summary", "summary") + ]: + raw_config["cert"]["cert_"][option_id]["default"] = status[ + status_key + ] + + # Other specific strings used in config panels + # i18n: domain_config_cert_renew_help + + return raw_config + + def _get_raw_settings(self) -> "RawSettings": + raw_settings = super()._get_raw_settings() + + custom_css = Path(f"/usr/share/yunohost/portal/customassets/{self.entity}.custom.css") + if custom_css.exists(): + raw_settings["custom_css"] = read_file(str(custom_css)) + + return raw_settings + + def _apply( + self, + form: "FormModel", + config: "ConfigPanelModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ) -> None: + next_settings = { + k: v for k, v in form.dict().items() if previous_settings.get(k) != v + } + + if "default_app" in next_settings: + from yunohost.app import app_map + + if "/" in app_map(raw=True).get(self.entity, {}): + raise YunohostValidationError( + "app_make_default_location_already_used", + app=next_settings["default_app"], + domain=self.entity, + other_app=app_map(raw=True)[self.entity]["/"]["id"], + ) - # FIXME: Ugly hack to save the cert status and reinject it in _get_raw_settings ... - self.cert_status = status - - return toml - - def _get_raw_settings(self): - # TODO add mechanism to share some settings with other domains on the same zone - super()._get_raw_settings() - - # FIXME: Ugly hack to save the registar id/value and reinject it in _get_raw_settings ... - filter_key = self.filter_key.split(".") if self.filter_key != "" else [] - if not filter_key or filter_key[0] == "dns": - self.values["registrar"] = self.registar_id - - # FIXME: Ugly hack to save the cert status and reinject it in _get_raw_settings ... - if not filter_key or filter_key[0] == "cert": - self.values["cert_validity"] = self.cert_status["validity"] - self.values["cert_issuer"] = self.cert_status["CA_type"] - self.values["acme_eligible"] = self.cert_status["ACME_eligible"] - self.values["summary"] = self.cert_status["summary"] - - def _apply(self): - if ( - "default_app" in self.future_values - and self.future_values["default_app"] != self.values["default_app"] - ): - from yunohost.app import app_ssowatconf, app_map - - if "/" in app_map(raw=True).get(self.entity, {}): - raise YunohostValidationError( - "app_make_default_location_already_used", - app=self.future_values["default_app"], - domain=self.entity, - other_app=app_map(raw=True)[self.entity]["/"]["id"], + if next_settings.get("recovery_password", None): + domain_dyndns_set_recovery_password( + self.entity, next_settings["recovery_password"] ) - if ( - "recovery_password" in self.new_values - and self.new_values["recovery_password"] - ): - domain_dyndns_set_recovery_password( - self.entity, self.new_values["recovery_password"] + + # NB: this is subtlely different from just checking `next_settings.get("use_auto_dns") since we want to find the exact situation where the admin *disables* the autodns` + remove_auto_dns_feature = "use_auto_dns" in next_settings and not next_settings["use_auto_dns"] + if remove_auto_dns_feature: + # disable auto dns by reseting every registrar form values + options = [ + option + for option in config.get_section("registrar").options + if not option.readonly + and option.id != "use_auto_dns" + and hasattr(form, option.id) + ] + for option in options: + setattr(form, option.id, option.default) + + if "custom_css" in next_settings: + write_to_file( + f"/usr/share/yunohost/portal/customassets/{self.entity}.custom.css", + next_settings.pop("custom_css", "").strip(), + ) + # Make sure the value doesnt get written in the yml + if hasattr(form, "custom_css"): + form.custom_css = "" + + portal_options = [ + "enable_public_apps_page", + "show_other_domains_apps", + "portal_title", + "portal_logo", + "portal_theme", + "portal_tile_theme", + "search_engine", + "search_engine_name", + "portal_user_intro", + "portal_public_intro", + ] + + if _get_parent_domain_of(self.entity, topest=True) is None and any( + option in next_settings for option in portal_options + ): + from yunohost.portal import PORTAL_SETTINGS_DIR + + # Portal options are also saved in a `domain.portal.yml` file + # that can be read by the portal API. + # FIXME remove those from the config panel saved values? + + portal_values = form.dict(include=set(portal_options)) + # Remove logo from values else filename will replace b64 content + if "portal_logo" in portal_values: + portal_values.pop("portal_logo") + + if "portal_logo" in next_settings: + if previous_settings.get("portal_logo"): + try: + os.remove(previous_settings["portal_logo"]) + except FileNotFoundError: + logger.warning( + f"Coulnd't remove previous logo file, maybe the file was already deleted, path: {previous_settings['portal_logo']}" + ) + finally: + portal_values["portal_logo"] = "" + + if next_settings["portal_logo"]: + portal_values["portal_logo"] = Path(next_settings["portal_logo"]).name + + portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{self.entity}.json") + portal_settings: dict[str, Any] = {"apps": {}} + + if portal_settings_path.exists(): + portal_settings.update(read_json(str(portal_settings_path))) + + # Merge settings since this config file is shared with `app_ssowatconf()` which populate the `apps` key. + portal_settings.update(portal_values) + write_to_json( + str(portal_settings_path), portal_settings, sort_keys=True, indent=4 + ) + + super()._apply( + form, config, previous_settings, exclude={"recovery_password"} ) - # Do not save password in yaml settings - if "recovery_password" in self.values: - del self.values["recovery_password"] - if "recovery_password" in self.new_values: - del self.new_values["recovery_password"] - assert "recovery_password" not in self.future_values - - super()._apply() - - # Reload ssowat if default app changed - if ( - "default_app" in self.future_values - and self.future_values["default_app"] != self.values["default_app"] - ): - app_ssowatconf() - stuff_to_regen_conf = [] - if ( - "xmpp" in self.future_values - and self.future_values["xmpp"] != self.values["xmpp"] - ): - stuff_to_regen_conf.append("nginx") - stuff_to_regen_conf.append("metronome") - - if ( - "mail_in" in self.future_values - and self.future_values["mail_in"] != self.values["mail_in"] - ) or ( - "mail_out" in self.future_values - and self.future_values["mail_out"] != self.values["mail_out"] - ): - if "nginx" not in stuff_to_regen_conf: - stuff_to_regen_conf.append("nginx") - stuff_to_regen_conf.append("postfix") - stuff_to_regen_conf.append("dovecot") - stuff_to_regen_conf.append("rspamd") - - if stuff_to_regen_conf: - regen_conf(names=stuff_to_regen_conf) + # Also remove `managed_dns_records_hashes` in settings which are not handled by the config panel + if remove_auto_dns_feature: + _set_managed_dns_records_hashes(self.entity, []) + + # Reload ssowat if default app changed + if "default_app" in next_settings or "enable_public_apps_page" in next_settings: + from yunohost.app import app_ssowatconf + + app_ssowatconf() + + stuff_to_regen_conf = set() + if "mail_in" in next_settings or "mail_out" in next_settings: + stuff_to_regen_conf.update({"nginx", "postfix", "dovecot"}) + + if stuff_to_regen_conf: + regen_conf(names=list(stuff_to_regen_conf)) + + return DomainConfigPanel def domain_action_run(domain, action, args=None): import urllib.parse - if action == "cert.cert.cert_install": + if action == "cert.cert_.cert_install": from yunohost.certificate import certificate_install as action_func - elif action == "cert.cert.cert_renew": + elif action == "cert.cert_.cert_renew": from yunohost.certificate import certificate_renew as action_func args = dict(urllib.parse.parse_qsl(args or "", keep_blank_values=True)) @@ -819,10 +966,6 @@ def domain_cert_renew(domain_list, force=False, no_checks=False, email=False): return certificate_renew(domain_list, force, no_checks, email) -def domain_dns_conf(domain): - return domain_dns_suggest(domain) - - def domain_dns_suggest(domain): from yunohost.dns import domain_dns_suggest diff --git a/src/dyndns.py b/src/dyndns.py index e48850f725..7c538a4477 100644 --- a/src/dyndns.py +++ b/src/dyndns.py @@ -22,10 +22,10 @@ import base64 import subprocess import hashlib +from logging import getLogger from moulinette import Moulinette, m18n from moulinette.core import MoulinetteError -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import write_to_file, rm, chown, chmod from yunohost.utils.error import YunohostError, YunohostValidationError @@ -35,7 +35,7 @@ from yunohost.log import is_unit_operation from yunohost.regenconf import regen_conf -logger = getActionLogger("yunohost.dyndns") +logger = getLogger("yunohost.dyndns") DYNDNS_PROVIDER = "dyndns.yunohost.org" DYNDNS_DNS_AUTH = ["ns0.yunohost.org", "ns1.yunohost.org"] @@ -471,7 +471,7 @@ def resolve_domain(domain, rdtype): # Delete custom DNS records, we don't support them (have to explicitly # authorize them on dynette) for category in dns_conf.keys(): - if category not in ["basic", "mail", "xmpp", "extra"]: + if category not in ["basic", "mail", "extra"]: del dns_conf[category] # Delete the old records for all domain/subdomains diff --git a/src/firewall.py b/src/firewall.py index 3ed33da61a..4e7337a5dd 100644 --- a/src/firewall.py +++ b/src/firewall.py @@ -19,16 +19,16 @@ import os import yaml import miniupnpc +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils import process -from moulinette.utils.log import getActionLogger FIREWALL_FILE = "/etc/yunohost/firewall.yml" UPNP_CRON_JOB = "/etc/cron.d/yunohost-firewall-upnp" -logger = getActionLogger("yunohost.firewall") +logger = getLogger("yunohost.firewall") def firewall_allow( @@ -402,7 +402,13 @@ def firewall_upnp(action="status", no_refresh=False): # Discover UPnP device(s) logger.debug("discovering UPnP devices...") - nb_dev = upnpc.discover() + try: + nb_dev = upnpc.discover() + except Exception: + logger.warning("Failed to find any UPnP device on the network") + nb_dev = -1 + enabled = False + logger.debug("found %d UPnP device(s)", int(nb_dev)) if nb_dev < 1: logger.error(m18n.n("upnp_dev_not_found")) diff --git a/src/hook.py b/src/hook.py index 49c2f4cc3a..acba650beb 100644 --- a/src/hook.py +++ b/src/hook.py @@ -23,16 +23,16 @@ import mimetypes from glob import iglob from importlib import import_module +from logging import getLogger from moulinette import m18n, Moulinette from yunohost.utils.error import YunohostError, YunohostValidationError -from moulinette.utils import log from moulinette.utils.filesystem import read_yaml, cp HOOK_FOLDER = "/usr/share/yunohost/hooks/" CUSTOM_HOOK_FOLDER = "/etc/yunohost/hooks.d/" -logger = log.getActionLogger("yunohost.hook") +logger = getLogger("yunohost.hook") def hook_add(app, file): @@ -359,6 +359,7 @@ def is_relevant_warning(msg): r"Removing obsolete dictionary files", r"Creating new PostgreSQL cluster", r"/usr/lib/postgresql/13/bin/initdb", + r"/usr/lib/postgresql/15/bin/initdb", r"The files belonging to this database system will be owned by user", r"This user must also own the server process.", r"The database cluster will be initialized with locale", @@ -366,6 +367,7 @@ def is_relevant_warning(msg): r"The default text search configuration will be set to", r"Data page checksums are disabled.", r"fixing permissions on existing directory /var/lib/postgresql/13/main ... ok", + r"fixing permissions on existing directory /var/lib/postgresql/15/main ... ok", r"creating subdirectories \.\.\. ok", r"selecting dynamic .* \.\.\. ", r"selecting default .* \.\.\. ", diff --git a/src/log.py b/src/log.py index d676e13ed1..f60977c14e 100755 --- a/src/log.py +++ b/src/log.py @@ -1,4 +1,3 @@ -# # Copyright (c) 2024 YunoHost Contributors # # This file is part of YunoHost (see https://yunohost.org) @@ -33,13 +32,11 @@ from moulinette.core import MoulinetteError from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.system import get_ynh_package_version -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import read_file, read_yaml -logger = getActionLogger("yunohost.log") +logger = getLogger("yunohost.log") -CATEGORIES_PATH = "/var/log/yunohost/categories/" -OPERATIONS_PATH = "/var/log/yunohost/categories/operation/" +OPERATIONS_PATH = "/var/log/yunohost/operations/" METADATA_FILE_EXT = ".yml" LOG_FILE_EXT = ".log" @@ -288,7 +285,7 @@ def _filter(lines): infos = {} # If it's a unit operation, display the name and the description - if base_path.startswith(CATEGORIES_PATH): + if base_path.startswith(OPERATIONS_PATH): infos["description"] = _get_description_from_name(base_filename) infos["name"] = base_filename @@ -392,11 +389,18 @@ def log_share(path): return log_show(path, share=True) +from typing import TypeVar, Callable, Concatenate, ParamSpec + +# FuncT = TypeVar("FuncT", bound=Callable[..., Any]) +Param = ParamSpec("Param") +RetType = TypeVar("RetType") + + def is_unit_operation( entities=["app", "domain", "group", "service", "user"], exclude=["password"], - operation_key=None, -): +) -> Callable[[Callable[Concatenate["OperationLogger", Param], RetType]], Callable[Param, RetType]]: + """ Configure quickly a unit operation @@ -413,17 +417,10 @@ def is_unit_operation( called 'password' are removed. If an argument is an object, you need to exclude it or create manually the unit operation without this decorator. - operation_key A key to describe the unit operation log used to create the - filename and search a translation. Please ensure that this key prefixed by - 'log_' is present in locales/en.json otherwise it won't be translatable. - """ - def decorate(func): + def decorate(func: Callable[Concatenate["OperationLogger", Param], RetType]) -> Callable[Param, RetType]: def func_wrapper(*args, **kwargs): - op_key = operation_key - if op_key is None: - op_key = func.__name__ # If the function is called directly from an other part of the code # and not by the moulinette framework, we need to complete kwargs @@ -474,7 +471,7 @@ def func_wrapper(*args, **kwargs): context[field] = value.name except Exception: context[field] = "IOBase" - operation_logger = OperationLogger(op_key, related_to, args=context) + operation_logger = OperationLogger(func.__name__, related_to, args=context) try: # Start the actual function, and give the unit operation @@ -542,7 +539,7 @@ class OperationLogger: This class record logs and metadata like context or start time/end time. """ - _instances: List[object] = [] + _instances: List["OperationLogger"] = [] def __init__(self, operation, related_to=None, **kwargs): # TODO add a way to not save password on app installation @@ -818,7 +815,7 @@ def dump_script_log_extract_for_debugging(self): # 2019-10-19 16:10:27,611: DEBUG - + mysql -u piwigo --password=********** -B piwigo # And we just want the part starting by "DEBUG - " lines = [line for line in lines if ":" in line.strip()] - lines = [line.strip().split(": ", 1)[1] for line in lines] + lines = [line.strip().split(": ", 1)[-1] for line in lines] # And we ignore boring/irrelevant lines # Annnnnnd we also ignore lines matching [number] + such as # 72971 DEBUG 29739 + ynh_exit_properly diff --git a/src/migrations/0021_migrate_to_bullseye.py b/src/migrations/0021_migrate_to_bullseye.py deleted file mode 100644 index d9924f97e7..0000000000 --- a/src/migrations/0021_migrate_to_bullseye.py +++ /dev/null @@ -1,582 +0,0 @@ -import glob -import os - -from moulinette import m18n -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger -from moulinette.utils.process import check_output, call_async_output -from moulinette.utils.filesystem import read_file, rm, write_to_file - -from yunohost.tools import ( - Migration, - tools_update, - tools_upgrade, - _apt_log_line_is_relevant, -) -from yunohost.app import unstable_apps -from yunohost.regenconf import manually_modified_files, _force_clear_hashes -from yunohost.utils.system import ( - free_space_in_directory, - get_ynh_package_version, - _list_upgradable_apt_packages, -) -from yunohost.service import _get_services, _save_services - -logger = getActionLogger("yunohost.migration") - -N_CURRENT_DEBIAN = 10 -N_CURRENT_YUNOHOST = 4 - -VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt" - - -def _get_all_venvs(dir, level=0, maxlevel=3): - """ - Returns the list of all python virtual env directories recursively - - Arguments: - dir - the directory to scan in - maxlevel - the depth of the recursion - level - do not edit this, used as an iterator - """ - if not os.path.exists(dir): - return [] - - result = [] - # Using os functions instead of glob, because glob doesn't support hidden folders, and we need recursion with a fixed depth - for file in os.listdir(dir): - path = os.path.join(dir, file) - if os.path.isdir(path): - activatepath = os.path.join(path, "bin", "activate") - if os.path.isfile(activatepath): - content = read_file(activatepath) - if ("VIRTUAL_ENV" in content) and ("PYTHONHOME" in content): - result.append(path) - continue - if level < maxlevel: - result += _get_all_venvs(path, level=level + 1) - return result - - -def _backup_pip_freeze_for_python_app_venvs(): - """ - Generate a requirements file for all python virtual env located inside /opt/ and /var/www/ - """ - - venvs = _get_all_venvs("/opt/") + _get_all_venvs("/var/www/") - for venv in venvs: - # Generate a requirements file from venv - os.system( - f"{venv}/bin/pip freeze > {venv}{VENV_REQUIREMENTS_SUFFIX} 2>/dev/null" - ) - - -class MyMigration(Migration): - "Upgrade the system to Debian Bullseye and Yunohost 11.x" - - mode = "manual" - - def run(self): - self.check_assertions() - - logger.info(m18n.n("migration_0021_start")) - - # - # Add new apt .deb signing key - # - - new_apt_key = "https://forge.yunohost.org/yunohost_bullseye.asc" - check_output(f"wget -O- {new_apt_key} -q | apt-key add -qq -") - - # - # Patch sources.list - # - logger.info(m18n.n("migration_0021_patching_sources_list")) - self.patch_apt_sources_list() - - # Stupid OVH has some repo configured which dont work with bullseye and break apt ... - os.system("sudo rm -f /etc/apt/sources.list.d/ovh-*.list") - - # Force add sury if it's not there yet - # This is to solve some weird issue with php-common breaking php7.3-common, - # hence breaking many php7.3-deps - # hence triggering some dependency conflict (or foobar-ynh-deps uninstall) - # Adding it there shouldnt be a big deal - Yunohost 11.x does add it - # through its regen conf anyway. - if not os.path.exists("/etc/apt/sources.list.d/extra_php_version.list"): - open("/etc/apt/sources.list.d/extra_php_version.list", "w").write( - "deb https://packages.sury.org/php/ bullseye main" - ) - - # Add Sury key even if extra_php_version.list was already there, - # because some old system may be using an outdated key not valid for Bullseye - # and that'll block the migration - os.system( - 'wget --timeout 900 --quiet "https://packages.sury.org/php/apt.gpg" --output-document=- | gpg --dearmor >"/etc/apt/trusted.gpg.d/extra_php_version.gpg"' - ) - - # Remove legacy, duplicated sury entry if it exists - if os.path.exists("/etc/apt/sources.list.d/sury.list"): - os.system("rm -rf /etc/apt/sources.list.d/sury.list") - - # - # Get requirements of the different venvs from python apps - # - - _backup_pip_freeze_for_python_app_venvs() - - # - # Run apt update - # - - tools_update(target="system") - - # Tell libc6 it's okay to restart system stuff during the upgrade - os.system( - "echo 'libc6 libraries/restart-without-asking boolean true' | debconf-set-selections" - ) - - # Do not restart nginx during the upgrade of nginx-common and nginx-extras ... - # c.f. https://manpages.debian.org/bullseye/init-system-helpers/deb-systemd-invoke.1p.en.html - # and zcat /usr/share/doc/init-system-helpers/README.policy-rc.d.gz - # and the code inside /usr/bin/deb-systemd-invoke to see how it calls /usr/sbin/policy-rc.d ... - # and also invoke-rc.d ... - write_to_file( - "/usr/sbin/policy-rc.d", - '#!/bin/bash\n[[ "$1" =~ "nginx" ]] && [[ "$2" == "restart" ]] && exit 101 || exit 0', - ) - os.system("chmod +x /usr/sbin/policy-rc.d") - - # Don't send an email to root about the postgresql migration. It should be handled automatically after. - os.system( - "echo 'postgresql-common postgresql-common/obsolete-major seen true' | debconf-set-selections" - ) - - # - # Patch yunohost conflicts - # - logger.info(m18n.n("migration_0021_patch_yunohost_conflicts")) - - self.patch_yunohost_conflicts() - - # - # Specific tweaking to get rid of custom my.cnf and use debian's default one - # (my.cnf is actually a symlink to mariadb.cnf) - # - - _force_clear_hashes(["/etc/mysql/my.cnf"]) - rm("/etc/mysql/mariadb.cnf", force=True) - rm("/etc/mysql/my.cnf", force=True) - ret = self.apt_install( - "mariadb-common --reinstall -o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - raise YunohostError("Failed to reinstall mariadb-common ?", raw_msg=True) - - # - # /usr/share/yunohost/yunohost-config/ssl/yunoCA -> /usr/share/yunohost/ssl - # - if os.path.exists("/usr/share/yunohost/yunohost-config/ssl/yunoCA"): - os.system( - "mv /usr/share/yunohost/yunohost-config/ssl/yunoCA /usr/share/yunohost/ssl" - ) - rm("/usr/share/yunohost/yunohost-config", recursive=True, force=True) - - # - # /home/yunohost.conf -> /var/cache/yunohost/regenconf - # - if os.path.exists("/home/yunohost.conf"): - os.system("mv /home/yunohost.conf /var/cache/yunohost/regenconf") - rm("/home/yunohost.conf", recursive=True, force=True) - - # Remove legacy postgresql service record added by helpers, - # will now be dynamically handled by the core in bullseye - services = _get_services() - if "postgresql" in services: - del services["postgresql"] - _save_services(services) - - # - # Critical fix for RPI otherwise network is down after rebooting - # https://forum.yunohost.org/t/20652 - # - if os.system("systemctl | grep -q dhcpcd") == 0: - logger.info("Applying fix for DHCPCD ...") - os.system("mkdir -p /etc/systemd/system/dhcpcd.service.d") - write_to_file( - "/etc/systemd/system/dhcpcd.service.d/wait.conf", - "[Service]\nExecStart=\nExecStart=/usr/sbin/dhcpcd -w", - ) - - # - # Another boring fix for the super annoying libc6-dev: Breaks libgcc-8-dev - # https://forum.yunohost.org/t/20617 - # - if ( - os.system("dpkg --list | grep '^ii' | grep -q ' libgcc-8-dev'") == 0 - and os.system( - "LC_ALL=C apt policy libgcc-8-dev | grep Candidate | grep -q rpi" - ) - == 0 - ): - logger.info( - "Attempting to fix the build-essential / libc6-dev / libgcc-8-dev hell ..." - ) - os.system("cp /var/lib/dpkg/status /root/dpkg_status.bkp") - # This removes the dependency to build-essential from $app-ynh-deps - os.system( - "perl -i~ -0777 -pe 's/(Package: .*-ynh-deps\\n(.+:.+\\n)+Depends:.*)(build-essential, ?)(.*)/$1$4/g' /var/lib/dpkg/status" - ) - self.apt_install( - "build-essential-" - ) # Note the '-' suffix to mean that we actually want to remove the packages - os.system( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt autoremove --assume-yes" - ) - self.apt_install( - "gcc-8- libgcc-8-dev- equivs" - ) # Note the '-' suffix to mean that we actually want to remove the packages .. we also explicitly add 'equivs' to the list because sometimes apt is dumb and will derp about it - - # - # Main upgrade - # - logger.info(m18n.n("migration_0021_main_upgrade")) - - apps_packages = self.get_apps_equivs_packages() - self.hold(apps_packages) - tools_upgrade(target="system", allow_yunohost_upgrade=False) - - if self.debian_major_version() == N_CURRENT_DEBIAN: - raise YunohostError("migration_0021_still_on_buster_after_main_upgrade") - - # Force explicit install of php7.4-fpm and other old 'default' dependencies - # that are now only in Recommends - # - # Also, we need to install php7.4 equivalents of other php7.3 dependencies. - # For example, Nextcloud may depend on php7.3-zip, and after the php pool migration - # to autoupgrade Nextcloud to 7.4, it will need the php7.4-zip to work. - # The following list is based on an ad-hoc analysis of php deps found in the - # app ecosystem, with a known equivalent on php7.4. - # - # This is kinda a dirty hack as it doesnt properly update the *-ynh-deps virtual packages - # with the proper list of dependencies, and the dependencies install this way - # will get flagged as 'manually installed'. - # - # We'll probably want to do something during the Bullseye->Bookworm migration to re-flag - # these as 'auto' so they get autoremoved if not needed anymore. - # Also hopefully by then we'll have manifestv2 (maybe) and will be able to use - # the apt resource mecanism to regenerate the *-ynh-deps virtual packages ;) - - php73packages_suffixes = [ - "apcu", - "bcmath", - "bz2", - "dom", - "gmp", - "igbinary", - "imagick", - "imap", - "mbstring", - "memcached", - "mysqli", - "mysqlnd", - "pgsql", - "redis", - "simplexml", - "soap", - "sqlite3", - "ssh2", - "tidy", - "xml", - "xmlrpc", - "xsl", - "zip", - ] - - cmd = ( - "apt show '*-ynh-deps' 2>/dev/null" - " | grep Depends" - f" | grep -o -E \"php7.3-({'|'.join(php73packages_suffixes)})\"" - " | sort | uniq" - " | sed 's/php7.3/php7.4/g'" - " || true" - ) - - basephp74packages_to_install = [ - "php7.4-fpm", - "php7.4-common", - "php7.4-ldap", - "php7.4-intl", - "php7.4-mysql", - "php7.4-gd", - "php7.4-curl", - "php-php-gettext", - ] - - php74packages_to_install = basephp74packages_to_install + [ - f.strip() for f in check_output(cmd).split("\n") if f.strip() - ] - - ret = self.apt_install( - f"{' '.join(php74packages_to_install)} " - "$(dpkg --list | grep ynh-deps | awk '{print $2}') " - "-o Dpkg::Options::='--force-confmiss'" - ) - if ret != 0: - raise YunohostError( - "Failed to force the install of php dependencies ?", raw_msg=True - ) - - # Clean the mess - logger.info(m18n.n("migration_0021_cleaning_up")) - os.system( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt autoremove --assume-yes" - ) - os.system("apt clean --assume-yes") - - # - # Stupid hack for stupid dnsmasq not picking up its new init.d script then breaking everything ... - # https://forum.yunohost.org/t/20676 - # - if os.path.exists("/etc/init.d/dnsmasq.dpkg-dist"): - logger.info("Copying new version for /etc/init.d/dnsmasq ...") - os.system("cp /etc/init.d/dnsmasq.dpkg-dist /etc/init.d/dnsmasq") - - # - # Yunohost upgrade - # - logger.info(m18n.n("migration_0021_yunohost_upgrade")) - - self.unhold(apps_packages) - - cmd = "LC_ALL=C" - cmd += " DEBIAN_FRONTEND=noninteractive" - cmd += " APT_LISTCHANGES_FRONTEND=none" - cmd += " apt dist-upgrade " - cmd += " --quiet -o=Dpkg::Use-Pty=0 --fix-broken --dry-run" - cmd += " | grep -q 'ynh-deps'" - - logger.info("Simulating upgrade...") - if os.system(cmd) == 0: - raise YunohostError( - "The upgrade cannot be completed, because some app dependencies would need to be removed?", - raw_msg=True, - ) - - postupgradecmds = f"apt-mark auto {' '.join(basephp74packages_to_install)}\n" - postupgradecmds += "rm -f /usr/sbin/policy-rc.d\n" - postupgradecmds += "echo 'Restarting nginx...' >&2\n" - postupgradecmds += "systemctl restart nginx\n" - - tools_upgrade(target="system", postupgradecmds=postupgradecmds) - - def debian_major_version(self): - # The python module "platform" and lsb_release are not reliable because - # on some setup, they may still return Release=9 even after upgrading to - # buster ... (Apparently this is related to OVH overriding some stuff - # with /etc/lsb-release for instance -_-) - # Instead, we rely on /etc/os-release which should be the raw info from - # the distribution... - return int( - check_output( - "grep VERSION_ID /etc/os-release | head -n 1 | tr '\"' ' ' | cut -d ' ' -f2" - ) - ) - - def yunohost_major_version(self): - return int(get_ynh_package_version("yunohost")["version"].split(".")[0]) - - def check_assertions(self): - # Be on buster (10.x) and yunohost 4.x - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be > 9.x but yunohost package - # would still be in 3.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - try: - # Here we try to find the previous migration log, which should be somewhat recent and be at least 10k (we keep the biggest one) - maybe_previous_migration_log_id = check_output( - "cd /var/log/yunohost/categories/operation && find -name '*migrate*.log' -size +10k -mtime -100 -exec ls -s {} \\; | sort -n | tr './' ' ' | awk '{print $2}' | tail -n 1" - ) - if maybe_previous_migration_log_id: - logger.info( - f"NB: the previous migration log id seems to be {maybe_previous_migration_log_id}. You can share it with the support team with : sudo yunohost log share {maybe_previous_migration_log_id}" - ) - except Exception: - # Yeah it's not that important ... it's to simplify support ... - pass - - raise YunohostError("migration_0021_not_buster2") - - # Have > 1 Go free space on /var/ ? - if free_space_in_directory("/var/") / (1024**3) < 1.0: - raise YunohostError("migration_0021_not_enough_free_space") - - # Have > 70 MB free space on /var/ ? - if free_space_in_directory("/boot/") / (1024**2) < 70.0: - raise YunohostError( - "/boot/ has less than 70MB available. This will probably trigger a crash during the upgrade because a new kernel needs to be installed. Please look for advice on the forum on how to remove old, unused kernels to free up some space in /boot/.", - raw_msg=True, - ) - - # Check system is up to date - # (but we don't if 'bullseye' is already in the sources.list ... - # which means maybe a previous upgrade crashed and we're re-running it) - if os.path.exists("/etc/apt/sources.list") and " bullseye " not in read_file( - "/etc/apt/sources.list" - ): - tools_update(target="system") - upgradable_system_packages = list(_list_upgradable_apt_packages()) - upgradable_system_packages = [ - package["name"] for package in upgradable_system_packages - ] - upgradable_system_packages = set(upgradable_system_packages) - # Lime2 have hold packages to avoid ethernet instability - # See https://github.com/YunoHost/arm-images/commit/b4ef8c99554fd1a122a306db7abacc4e2f2942df - lime2_hold_packages = set( - [ - "armbian-firmware", - "armbian-bsp-cli-lime2", - "linux-dtb-current-sunxi", - "linux-image-current-sunxi", - "linux-u-boot-lime2-current", - "linux-image-next-sunxi", - ] - ) - if upgradable_system_packages - lime2_hold_packages: - raise YunohostError("migration_0021_system_not_fully_up_to_date") - - @property - def disclaimer(self): - # Avoid having a super long disclaimer + uncessary check if we ain't - # on buster / yunohost 4.x anymore - # NB : we do both check to cover situations where the upgrade crashed - # in the middle and debian version could be >= 10.x but yunohost package - # would still be in 4.x... - if ( - not self.debian_major_version() == N_CURRENT_DEBIAN - and not self.yunohost_major_version() == N_CURRENT_YUNOHOST - ): - return None - - # Get list of problematic apps ? I.e. not official or community+working - problematic_apps = unstable_apps() - problematic_apps = "".join(["\n - " + app for app in problematic_apps]) - - # Manually modified files ? (c.f. yunohost service regen-conf) - modified_files = manually_modified_files() - modified_files = "".join(["\n - " + f for f in modified_files]) - - message = m18n.n("migration_0021_general_warning") - - message = ( - "N.B.: This migration has been tested by the community over the last few months but has only been declared stable recently. If your server hosts critical services and if you are not too confident with debugging possible issues, we recommend you to wait a little bit more while we gather more feedback and polish things up. If on the other hand you are relatively confident with debugging small issues that may arise, you are encouraged to run this migration ;)! You can read about remaining known issues and feedback from the community here: https://forum.yunohost.org/t/20590\n\n" - + message - ) - - if problematic_apps: - message += "\n\n" + m18n.n( - "migration_0021_problematic_apps_warning", - problematic_apps=problematic_apps, - ) - - if modified_files: - message += "\n\n" + m18n.n( - "migration_0021_modified_files", manually_modified_files=modified_files - ) - - return message - - def patch_apt_sources_list(self): - sources_list = glob.glob("/etc/apt/sources.list.d/*.list") - if os.path.exists("/etc/apt/sources.list"): - sources_list.append("/etc/apt/sources.list") - - # This : - # - replace single 'buster' occurence by 'bulleye' - # - comments lines containing "backports" - # - replace 'buster/updates' by 'bullseye/updates' (or same with -) - # Special note about the security suite: - # https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#security-archive - for f in sources_list: - command = ( - f"sed -i {f} " - "-e 's@ buster @ bullseye @g' " - "-e '/backports/ s@^#*@#@' " - "-e 's@ buster/updates @ bullseye-security @g' " - "-e 's@ buster-@ bullseye-@g' " - ) - os.system(command) - - def get_apps_equivs_packages(self): - command = ( - "dpkg --get-selections" - " | grep -v deinstall" - " | awk '{print $1}'" - " | { grep 'ynh-deps$' || true; }" - ) - - output = check_output(command) - - return output.split("\n") if output else [] - - def hold(self, packages): - for package in packages: - os.system(f"apt-mark hold {package}") - - def unhold(self, packages): - for package in packages: - os.system(f"apt-mark unhold {package}") - - def apt_install(self, cmd): - def is_relevant(line): - return "Reading database ..." not in line.rstrip() - - callbacks = ( - lambda l: ( - logger.info("+ " + l.rstrip() + "\r") - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip() + "\r") - ), - lambda l: ( - logger.warning(l.rstrip()) - if _apt_log_line_is_relevant(l) - else logger.debug(l.rstrip()) - ), - ) - - cmd = ( - "LC_ALL=C DEBIAN_FRONTEND=noninteractive APT_LISTCHANGES_FRONTEND=none apt install --quiet -o=Dpkg::Use-Pty=0 --fix-broken --assume-yes " - + cmd - ) - - logger.debug("Running: %s" % cmd) - - return call_async_output(cmd, callbacks, shell=True) - - def patch_yunohost_conflicts(self): - # - # This is a super dirty hack to remove the conflicts from yunohost's debian/control file - # Those conflicts are there to prevent mistakenly upgrading critical packages - # such as dovecot, postfix, nginx, openssl, etc... usually related to mistakenly - # using backports etc. - # - # The hack consists in savagely removing the conflicts directly in /var/lib/dpkg/status - # - - # We only patch the conflict if we're on yunohost 4.x - if self.yunohost_major_version() != N_CURRENT_YUNOHOST: - return - - conflicts = check_output("dpkg-query -s yunohost | grep '^Conflicts:'").strip() - if conflicts: - # We want to keep conflicting with apache/bind9 tho - new_conflicts = "Conflicts: apache2, bind9" - - command = ( - f"sed -i /var/lib/dpkg/status -e 's@{conflicts}@{new_conflicts}@g'" - ) - logger.debug(f"Running: {command}") - os.system(command) diff --git a/src/migrations/0022_php73_to_php74_pools.py b/src/migrations/0022_php73_to_php74_pools.py deleted file mode 100644 index dc428e5042..0000000000 --- a/src/migrations/0022_php73_to_php74_pools.py +++ /dev/null @@ -1,94 +0,0 @@ -import os -import glob -from shutil import copy2 - -from moulinette.utils.log import getActionLogger - -from yunohost.app import _is_installed -from yunohost.utils.legacy import _patch_legacy_php_versions_in_settings -from yunohost.tools import Migration -from yunohost.service import _run_service_command - -logger = getActionLogger("yunohost.migration") - -OLDPHP_POOLS = "/etc/php/7.3/fpm/pool.d" -NEWPHP_POOLS = "/etc/php/7.4/fpm/pool.d" - -OLDPHP_SOCKETS_PREFIX = "/run/php/php7.3-fpm" -NEWPHP_SOCKETS_PREFIX = "/run/php/php7.4-fpm" - -# Because of synapse é_è -OLDPHP_SOCKETS_PREFIX2 = "/run/php7.3-fpm" -NEWPHP_SOCKETS_PREFIX2 = "/run/php7.4-fpm" - -MIGRATION_COMMENT = ( - "; YunoHost note : this file was automatically moved from {}".format(OLDPHP_POOLS) -) - - -class MyMigration(Migration): - "Migrate php7.3-fpm 'pool' conf files to php7.4" - - dependencies = ["migrate_to_bullseye"] - - def run(self): - # Get list of php7.3 pool files - oldphp_pool_files = glob.glob("{}/*.conf".format(OLDPHP_POOLS)) - - # Keep only basenames - oldphp_pool_files = [os.path.basename(f) for f in oldphp_pool_files] - - # Ignore the "www.conf" (default stuff, probably don't want to touch it ?) - oldphp_pool_files = [f for f in oldphp_pool_files if f != "www.conf"] - - for pf in oldphp_pool_files: - # Copy the files to the php7.3 pool - src = "{}/{}".format(OLDPHP_POOLS, pf) - dest = "{}/{}".format(NEWPHP_POOLS, pf) - copy2(src, dest) - - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format( - OLDPHP_SOCKETS_PREFIX, NEWPHP_SOCKETS_PREFIX, dest - ) - os.system(c) - c = "sed -i -e 's@{}@{}@g' {}".format( - OLDPHP_SOCKETS_PREFIX2, NEWPHP_SOCKETS_PREFIX2, dest - ) - os.system(c) - - # Also add a comment that it was automatically moved from php7.3 - # (for human traceability and backward migration) - c = "sed -i '1i {}' {}".format(MIGRATION_COMMENT, dest) - os.system(c) - - app_id = os.path.basename(pf)[: -len(".conf")] - if _is_installed(app_id): - _patch_legacy_php_versions_in_settings( - "/etc/yunohost/apps/%s/" % app_id - ) - - nginx_conf_files = glob.glob("/etc/nginx/conf.d/*.d/%s.conf" % app_id) - for nf in nginx_conf_files: - # Replace the socket prefix if it's found - c = "sed -i -e 's@{}@{}@g' {}".format( - OLDPHP_SOCKETS_PREFIX, NEWPHP_SOCKETS_PREFIX, nf - ) - os.system(c) - c = "sed -i -e 's@{}@{}@g' {}".format( - OLDPHP_SOCKETS_PREFIX2, NEWPHP_SOCKETS_PREFIX2, nf - ) - os.system(c) - - os.system( - "rm /etc/logrotate.d/php7.3-fpm" - ) # We remove this otherwise the logrotate cron will be unhappy - - # Reload/restart the php pools - os.system("systemctl stop php7.3-fpm") - os.system("systemctl disable php7.3-fpm") - _run_service_command("restart", "php7.4-fpm") - _run_service_command("enable", "php7.4-fpm") - - # Reload nginx - _run_service_command("reload", "nginx") diff --git a/src/migrations/0025_global_settings_to_configpanel.py b/src/migrations/0025_global_settings_to_configpanel.py deleted file mode 100644 index 3a88184617..0000000000 --- a/src/migrations/0025_global_settings_to_configpanel.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -from yunohost.utils.error import YunohostError -from moulinette.utils.log import getActionLogger -from moulinette.utils.filesystem import read_json, write_to_yaml - -from yunohost.tools import Migration -from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings - -logger = getActionLogger("yunohost.migration") - -SETTINGS_PATH = "/etc/yunohost/settings.yml" -OLD_SETTINGS_PATH = "/etc/yunohost/settings.json" - - -class MyMigration(Migration): - "Migrate old global settings to the new ConfigPanel global settings" - - dependencies = ["migrate_to_bullseye"] - - def run(self): - if not os.path.exists(OLD_SETTINGS_PATH): - return - - try: - old_settings = read_json(OLD_SETTINGS_PATH) - except Exception as e: - raise YunohostError(f"Can't open setting file : {e}", raw_msg=True) - - settings = { - translate_legacy_settings_to_configpanel_settings(k).split(".")[-1]: v[ - "value" - ] - for k, v in old_settings.items() - } - - if settings.get("smtp_relay_host"): - settings["smtp_relay_enabled"] = True - - # Here we don't use settings_set() from settings.py to prevent - # Questions to be asked when one run the migration from CLI. - write_to_yaml(SETTINGS_PATH, settings) diff --git a/src/migrations/0026_new_admins_group.py b/src/migrations/0026_new_admins_group.py deleted file mode 100644 index 43f10a7b66..0000000000 --- a/src/migrations/0026_new_admins_group.py +++ /dev/null @@ -1,153 +0,0 @@ -from moulinette.utils.log import getActionLogger - -from yunohost.tools import Migration - -logger = getActionLogger("yunohost.migration") - -################################################### -# Tools used also for restoration -################################################### - - -class MyMigration(Migration): - """ - Add new permissions around SSH/SFTP features - """ - - introduced_in_version = "11.1" # FIXME? - dependencies = [] - - ldap_migration_started = False - - @Migration.ldap_migration - def run(self, *args): - from yunohost.user import ( - user_list, - user_info, - user_group_update, - user_update, - user_group_add_mailalias, - ADMIN_ALIASES, - ) - from yunohost.utils.ldap import _get_ldap_interface - from yunohost.permission import permission_sync_to_user - from yunohost.domain import _get_maindomain - - main_domain = _get_maindomain() - ldap = _get_ldap_interface() - - all_users = user_list()["users"].keys() - new_admin_user = None - for user in all_users: - if any( - alias.startswith("root@") - for alias in user_info(user).get("mail-aliases", []) - ): - new_admin_user = user - break - - # For some reason some system have no user with root@ alias, - # but the user does has admin / postmaster / ... alias - # ... try to find it instead otherwise this creashes the migration - # later because the admin@, postmaster@, .. aliases will already exist - if not new_admin_user: - for user in all_users: - aliases = user_info(user).get("mail-aliases", []) - if any( - alias.startswith(f"admin@{main_domain}") for alias in aliases - ) or any( - alias.startswith(f"postmaster@{main_domain}") for alias in aliases - ): - new_admin_user = user - break - - self.ldap_migration_started = True - - if new_admin_user: - aliases = user_info(new_admin_user).get("mail-aliases", []) - old_admin_aliases_to_remove = [ - alias - for alias in aliases - if any( - alias.startswith(a) - for a in [ - "root@", - "admin@", - "admins@", - "webmaster@", - "postmaster@", - "abuse@", - ] - ) - ] - - user_update(new_admin_user, remove_mailalias=old_admin_aliases_to_remove) - - admin_hashs = ldap.search("cn=admin", attrs={"userPassword"})[0]["userPassword"] - - stuff_to_delete = [ - "cn=admin,ou=sudo", - "cn=admin", - "cn=admins,ou=groups", - ] - - for stuff in stuff_to_delete: - if ldap.search(stuff): - ldap.remove(stuff) - - ldap.add( - "cn=admins,ou=sudo", - { - "cn": ["admins"], - "objectClass": ["top", "sudoRole"], - "sudoCommand": ["ALL"], - "sudoUser": ["%admins"], - "sudoHost": ["ALL"], - }, - ) - - ldap.add( - "cn=admins,ou=groups", - { - "cn": ["admins"], - "objectClass": ["top", "posixGroup", "groupOfNamesYnh"], - "gidNumber": ["4001"], - }, - ) - - user_group_add_mailalias( - "admins", [f"{alias}@{main_domain}" for alias in ADMIN_ALIASES] - ) - - permission_sync_to_user() - - if new_admin_user: - user_group_update(groupname="admins", add=new_admin_user, sync_perm=True) - - # Re-add admin as a regular user - attr_dict = { - "objectClass": [ - "mailAccount", - "inetOrgPerson", - "posixAccount", - "userPermissionYnh", - ], - "givenName": ["Admin"], - "sn": ["Admin"], - "displayName": ["Admin"], - "cn": ["Admin"], - "uid": ["admin"], - "mail": "admin_legacy", - "maildrop": ["admin"], - "mailuserquota": ["0"], - "userPassword": admin_hashs, - "gidNumber": ["1007"], - "uidNumber": ["1007"], - "homeDirectory": ["/home/admin"], - "loginShell": ["/bin/bash"], - } - ldap.add("uid=admin,ou=users", attr_dict) - user_group_update(groupname="admins", add="admin", sync_perm=True) - - def run_after_system_restore(self): - self.run() diff --git a/src/migrations/0028_delete_legacy_xmpp_permission.py b/src/migrations/0028_delete_legacy_xmpp_permission.py new file mode 100644 index 0000000000..de5d2b9832 --- /dev/null +++ b/src/migrations/0028_delete_legacy_xmpp_permission.py @@ -0,0 +1,32 @@ +from logging import getLogger + +from yunohost.tools import Migration + +logger = getLogger("yunohost.migration") + +################################################### +# Tools used also for restoration +################################################### + + +class MyMigration(Migration): + """ + Delete legacy XMPP permission + """ + + introduced_in_version = "12.0" + dependencies = [] + + ldap_migration_started = False + + @Migration.ldap_migration + def run(self, *args): + from yunohost.permission import user_permission_list, permission_delete + + self.ldap_migration_started = True + + if "xmpp.main" in user_permission_list()["permissions"]: + permission_delete("xmpp.main", force=True) + + def run_after_system_restore(self): + self.run() diff --git a/src/migrations/0023_postgresql_11_to_13.py b/src/migrations/0029_postgresql_13_to_15.py similarity index 70% rename from src/migrations/0023_postgresql_11_to_13.py rename to src/migrations/0029_postgresql_13_to_15.py index 6d37ffa740..f74d33a761 100644 --- a/src/migrations/0023_postgresql_11_to_13.py +++ b/src/migrations/0029_postgresql_13_to_15.py @@ -1,21 +1,21 @@ import subprocess import time import os +from logging import getLogger from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError -from moulinette.utils.log import getActionLogger from yunohost.tools import Migration from yunohost.utils.system import free_space_in_directory, space_used_by_directory -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") class MyMigration(Migration): - "Migrate DBs from Postgresql 11 to 13 after migrating to Bullseye" + "Migrate DBs from Postgresql 13 to 15 after migrating to Bookworm" - dependencies = ["migrate_to_bullseye"] + dependencies = ["migrate_to_bookworm"] def run(self): if ( @@ -27,37 +27,37 @@ def run(self): logger.info("No YunoHost app seem to require postgresql... Skipping!") return - if not self.package_is_installed("postgresql-11"): - logger.warning(m18n.n("migration_0023_postgresql_11_not_installed")) + if not self.package_is_installed("postgresql-13"): + logger.warning(m18n.n("migration_0029_postgresql_13_not_installed")) return - if not self.package_is_installed("postgresql-13"): - raise YunohostValidationError("migration_0023_postgresql_13_not_installed") + if not self.package_is_installed("postgresql-15"): + raise YunohostValidationError("migration_0029_postgresql_15_not_installed") - # Make sure there's a 11 cluster + # Make sure there's a 13 cluster try: - self.runcmd("pg_lsclusters | grep -q '^11 '") + self.runcmd("pg_lsclusters | grep -q '^13 '") except Exception: logger.warning( - "It looks like there's not active 11 cluster, so probably don't need to run this migration" + "It looks like there's not active 13 cluster, so probably don't need to run this migration" ) return if not space_used_by_directory( - "/var/lib/postgresql/11" + "/var/lib/postgresql/13" ) > free_space_in_directory("/var/lib/postgresql"): raise YunohostValidationError( - "migration_0023_not_enough_space", path="/var/lib/postgresql/" + "migration_0029_not_enough_space", path="/var/lib/postgresql/" ) self.runcmd("systemctl stop postgresql") time.sleep(3) self.runcmd( - "LC_ALL=C pg_dropcluster --stop 13 main || true" - ) # We do not trigger an exception if the command fails because that probably means cluster 13 doesn't exists, which is fine because it's created during the pg_upgradecluster) + "LC_ALL=C pg_dropcluster --stop 15 main || true" + ) # We do not trigger an exception if the command fails because that probably means cluster 15 doesn't exists, which is fine because it's created during the pg_upgradecluster) time.sleep(3) - self.runcmd("LC_ALL=C pg_upgradecluster -m upgrade 11 main") - self.runcmd("LC_ALL=C pg_dropcluster --stop 11 main") + self.runcmd("LC_ALL=C pg_upgradecluster -m upgrade 13 main -v 15") + self.runcmd("LC_ALL=C pg_dropcluster --stop 13 main") self.runcmd("systemctl start postgresql") def package_is_installed(self, package_name): diff --git a/src/migrations/0024_rebuild_python_venv.py b/src/migrations/0030_rebuild_python_venv_in_bookworm.py similarity index 74% rename from src/migrations/0024_rebuild_python_venv.py rename to src/migrations/0030_rebuild_python_venv_in_bookworm.py index 01a229b878..944d72e0fd 100644 --- a/src/migrations/0024_rebuild_python_venv.py +++ b/src/migrations/0030_rebuild_python_venv_in_bookworm.py @@ -1,16 +1,16 @@ import os +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import call_async_output from yunohost.tools import Migration, tools_migrations_state from moulinette.utils.filesystem import rm -logger = getActionLogger("yunohost.migration") +logger = getLogger("yunohost.migration") -VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bullseye_upgrade.txt" +VENV_REQUIREMENTS_SUFFIX = ".requirements_backup_for_bookworm_upgrade.txt" def extract_app_from_venv_path(venv_path): @@ -56,28 +56,28 @@ class MyMigration(Migration): """ ignored_python_apps = [ - "calibreweb", - "django-for-runners", - "ffsync", - "jupiterlab", - "librephotos", - "mautrix", - "mediadrop", - "mopidy", - "pgadmin", - "tracim", - "synapse", - "matrix-synapse", - "weblate", + "diacamma", # Does an ugly sed in the sites-packages/django_auth_ldap3_ad + "kresus", # uses virtualenv instead of venv, with --system-site-packages (?) + "librephotos", # runs a setup.py ? not sure pip freeze / pip install -r requirements.txt is gonna be equivalent .. + "mautrix", # install stuff from a .tar.gz + "microblogpub", # uses poetry ? x_x + "mopidy", # applies a custom patch? + "motioneye", # install stuff from a .tar.gz + "pgadmin", # bunch of manual patches + "searxng", # uses --system-site-packages ? + "synapse", # specific stuff for ARM to prevent local compiling etc + "matrix-synapse", # synapse is actually installed in /opt/yunohost/matrix-synapse because ... yeah ... + "tracim", # pip install -e . + "weblate", # weblate settings are .. inside the venv T_T ] - dependencies = ["migrate_to_bullseye"] + dependencies = ["migrate_to_bookworm"] state = None def is_pending(self): if not self.state: self.state = tools_migrations_state()["migrations"].get( - "0024_rebuild_python_venv", "pending" + "0030_rebuild_python_venv_in_bookworm", "pending" ) return self.state == "pending" @@ -121,15 +121,15 @@ def disclaimer(self): else: rebuild_apps.append(app_corresponding_to_venv) - msg = m18n.n("migration_0024_rebuild_python_venv_disclaimer_base") + msg = m18n.n("migration_0030_rebuild_python_venv_in_bookworm_disclaimer_base") if rebuild_apps: msg += "\n\n" + m18n.n( - "migration_0024_rebuild_python_venv_disclaimer_rebuild", + "migration_0030_rebuild_python_venv_in_bookworm_disclaimer_rebuild", rebuild_apps="\n - " + "\n - ".join(rebuild_apps), ) if ignored_apps: msg += "\n\n" + m18n.n( - "migration_0024_rebuild_python_venv_disclaimer_ignored", + "migration_0030_rebuild_python_venv_in_bookworm_disclaimer_ignored", ignored_apps="\n - " + "\n - ".join(ignored_apps), ) @@ -151,7 +151,7 @@ def run(self): rm(venv + VENV_REQUIREMENTS_SUFFIX) logger.info( m18n.n( - "migration_0024_rebuild_python_venv_broken_app", + "migration_0030_rebuild_python_venv_in_bookworm_broken_app", app=app_corresponding_to_venv, ) ) @@ -159,7 +159,7 @@ def run(self): logger.info( m18n.n( - "migration_0024_rebuild_python_venv_in_progress", + "migration_0030_rebuild_python_venv_in_bookworm_in_progress", app=app_corresponding_to_venv, ) ) @@ -178,7 +178,7 @@ def run(self): if status != 0: logger.error( m18n.n( - "migration_0024_rebuild_python_venv_failed", + "migration_0030_rebuild_python_venv_in_bookworm_failed", app=app_corresponding_to_venv, ) ) diff --git a/src/migrations/0031_terms_of_services.py b/src/migrations/0031_terms_of_services.py new file mode 100644 index 0000000000..94738d3a7d --- /dev/null +++ b/src/migrations/0031_terms_of_services.py @@ -0,0 +1,18 @@ +from moulinette import m18n +from yunohost.tools import Migration + +import logging +logger = logging.getLogger("yunohost.migration") + + +class MyMigration(Migration): + "Display new terms of services to admins" + + mode = "manual" + + def run(self): + pass + + @property + def disclaimer(self): + return m18n.n("migration_0031_terms_of_services") + "\n\n" + m18n.n("tos_postinstall_acknowledgement") diff --git a/src/permission.py b/src/permission.py index b416b05aba..8fb7d3499f 100644 --- a/src/permission.py +++ b/src/permission.py @@ -20,15 +20,15 @@ import copy import grp import random +from logging import getLogger from moulinette import m18n -from moulinette.utils.log import getActionLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.log import is_unit_operation -logger = getActionLogger("yunohost.user") +logger = getLogger("yunohost.user") -SYSTEM_PERMS = ["mail", "xmpp", "sftp", "ssh"] +SYSTEM_PERMS = ["mail", "sftp", "ssh"] # # @@ -170,7 +170,7 @@ def user_permission_update( existing_permission = user_permission_info(permission) - # Refuse to add "visitors" to mail, xmpp ... they require an account to make sense. + # Refuse to add "visitors" to mail ... they require an account to make sense. if add and "visitors" in add and permission.split(".")[0] in SYSTEM_PERMS: raise YunohostValidationError( "permission_require_account", permission=permission diff --git a/src/portal.py b/src/portal.py new file mode 100644 index 0000000000..1a6ef08b99 --- /dev/null +++ b/src/portal.py @@ -0,0 +1,284 @@ +# -*- coding: utf-8 -*- + +""" License + + Copyright (C) 2021 YUNOHOST.ORG + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program; if not, see http://www.gnu.org/licenses + +""" +import logging +from pathlib import Path +from typing import Any, Union + +import ldap +from moulinette.utils.filesystem import read_json +from yunohost.authenticators.ldap_ynhuser import Authenticator as Auth, user_is_allowed_on_domain +from yunohost.utils.error import YunohostError, YunohostValidationError +from yunohost.utils.ldap import _get_ldap_interface, _ldap_path_extract, LDAPInterface +from yunohost.utils.password import ( + assert_password_is_compatible, + assert_password_is_strong_enough, + _hash_user_password, +) + +logger = logging.getLogger("portal") + +PORTAL_SETTINGS_DIR = "/etc/yunohost/portal" +ADMIN_ALIASES = ["root", "admin", "admins", "webmaster", "postmaster", "abuse"] + + +def _get_user_infos( + user_attrs: list[str], +) -> tuple[str, str, dict[str, Any]]: + auth = Auth().get_session_cookie() + username = auth["user"] + result = _get_ldap_interface().search("ou=users", f"uid={username}", user_attrs) + if not result: + raise YunohostValidationError("user_unknown", user=username) + + return username, auth["host"], result[0] + + +def _get_portal_settings( + domain: Union[str, None] = None, username: Union[str, None] = None +): + """ + Returns domain's portal settings which are a combo of domain's portal config panel options + and the list of apps availables on this domain computed by `app.app_ssowatconf()`. + """ + + if not domain: + from bottle import request + + domain = request.get_header("host") + + assert domain and "/" not in domain + + settings: dict[str, Any] = { + "apps": {}, + "public": False, + "portal_logo": "", + "portal_theme": "system", + "portal_tile_theme": "simple", + "portal_title": "YunoHost", + "show_other_domains_apps": False, + "domain": domain, + } + + portal_settings_path = Path(f"{PORTAL_SETTINGS_DIR}/{domain}.json") + + if portal_settings_path.exists(): + settings.update(read_json(str(portal_settings_path))) + # Portal may be public (no login required) + settings["public"] = bool(settings.pop("enable_public_apps_page", False)) + + # First clear apps since it may contains private apps + apps: dict[str, Any] = settings.pop("apps", {}) + settings["apps"] = {} + + if settings["show_other_domains_apps"]: + # Enhanced apps with all other domain's apps + import glob + + for path in glob.glob(f"{PORTAL_SETTINGS_DIR}/*.json"): + if path != str(portal_settings_path): + apps.update(read_json(path)["apps"]) + + if username: + # Add user allowed or public apps + settings["apps"] = { + name: app + for name, app in apps.items() + if username in app["users"] or app["public"] + } + elif settings["public"]: + # Add public apps (e.g. with "visitors" in group permission) + settings["apps"] = {name: app for name, app in apps.items() if app["public"]} + + return settings + + +def portal_public(): + """Get public settings + If the portal is set as public, it will include the list of public apps + """ + + portal_settings = _get_portal_settings() + + try: + Auth().get_session_cookie() + except Exception: + if "portal_user_intro" in portal_settings: + del portal_settings["portal_user_intro"] + + # Prevent leaking the list of users + for infos in portal_settings["apps"].values(): + del infos["users"] + + return portal_settings + + +def portal_me(): + """ + Get user informations + """ + username, domain, user = _get_user_infos( + ["cn", "mail", "maildrop", "mailuserquota", "memberOf", "permission"] + ) + + groups = [_ldap_path_extract(g, "cn") for g in user["memberOf"]] + groups = [g for g in groups if g not in [username, "all_users"]] + # Get user allowed apps + apps = _get_portal_settings(domain, username)["apps"] + + # Prevent leaking the list of users + for infos in apps.values(): + del infos["users"] + + result_dict = { + "username": username, + "fullname": user["cn"][0], + "mail": user["mail"][0], + "mailalias": user["mail"][1:], + "mailforward": user["maildrop"][1:], + "groups": groups, + "apps": apps, + } + + # FIXME / TODO : add mail quota status ? + # result_dict["mailbox-quota"] = { + # "limit": userquota if is_limited else m18n.n("unlimit"), + # "use": storage_use, + # } + # Could use : doveadm -c /dev/null -f flow quota recalc -u johndoe + # But this requires to be in the mail group ... + + return result_dict + + +def portal_update( + fullname: Union[str, None] = None, + mailforward: Union[list[str], None] = None, + mailalias: Union[list[str], None] = None, + currentpassword: Union[str, None] = None, + newpassword: Union[str, None] = None, +): + from yunohost.domain import domain_list + + domains = domain_list()["domains"] + username, domain, current_user = _get_user_infos( + ["givenName", "sn", "cn", "mail", "maildrop", "memberOf"] + ) + new_attr_dict = {} + + if fullname is not None and fullname != current_user["cn"]: + fullname = fullname.strip() + firstname = fullname.split()[0] + lastname = ( + " ".join(fullname.split()[1:]) or " " + ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + new_attr_dict["givenName"] = [firstname] # TODO: Validate + new_attr_dict["sn"] = [lastname] # TODO: Validate + new_attr_dict["cn"] = new_attr_dict["displayName"] = [ + (firstname + " " + lastname).strip() + ] + + if mailalias is not None: + mailalias = [mail.strip() for mail in mailalias if mail and mail.strip()] + # keep first current mail unaltered + mails = [current_user["mail"][0]] + + for index, mail in enumerate(mailalias): + if mail in current_user["mail"]: + if mail != current_user["mail"][0] and mail not in mails: + mails.append(mail) + continue # already in mails, skip validation + + local_part, domain = mail.split("@") + if local_part in ADMIN_ALIASES: + raise YunohostValidationError( + "mail_unavailable", path=f"mailalias[{index}]" + ) + + try: + _get_ldap_interface().validate_uniqueness({"mail": mail}) + except YunohostError: + raise YunohostValidationError( + "mail_already_exists", mail=mail, path=f"mailalias[{index}]" + ) + + if domain not in domains or not user_is_allowed_on_domain(username, domain): + raise YunohostValidationError( + "mail_alias_unauthorized", domain=domain + ) + + mails.append(mail) + + new_attr_dict["mail"] = mails + + if mailforward is not None: + new_attr_dict["maildrop"] = [current_user["maildrop"][0]] + [ + mail.strip() + for mail in mailforward + if mail and mail.strip() and mail != current_user["maildrop"][0] + ] + + if newpassword: + # Ensure compatibility and sufficiently complex password + try: + assert_password_is_compatible(newpassword) + is_admin = ( + "cn=admins,ou=groups,dc=yunohost,dc=org" in current_user["memberOf"] + ) + assert_password_is_strong_enough( + "admin" if is_admin else "user", newpassword + ) + except YunohostValidationError as e: + raise YunohostValidationError(e.key, path="newpassword") + + new_attr_dict["userPassword"] = [_hash_user_password(newpassword)] + + # Check that current password is valid + # To be able to edit the user info, an authenticated ldap session is needed + if newpassword: + # When setting the password, check the user provided the valid current password + try: + ldap_interface = LDAPInterface(username, currentpassword) + except ldap.INVALID_CREDENTIALS: + raise YunohostValidationError("invalid_password", path="currentpassword") + else: + # Otherwise we use the encrypted password stored in the cookie + ldap_interface = LDAPInterface(username, Auth().get_session_cookie(decrypt_pwd=True)["pwd"]) + + try: + ldap_interface.update(f"uid={username},ou=users", new_attr_dict) + except Exception as e: + raise YunohostError("user_update_failed", user=username, error=e) + finally: + del ldap_interface + + if "userPassword" in new_attr_dict: + Auth.invalidate_all_sessions_for_user(username) + + # FIXME: Here we could want to trigger "post_user_update" hook but hooks has to + # be run as root + if all(field is not None for field in (fullname, mailalias, mailforward)): + return { + "fullname": new_attr_dict["cn"][0], + "mailalias": new_attr_dict["mail"][1:], + "mailforward": new_attr_dict["maildrop"][1:], + } + else: + return {} diff --git a/src/regenconf.py b/src/regenconf.py index 03017acf27..05a6edf815 100644 --- a/src/regenconf.py +++ b/src/regenconf.py @@ -20,12 +20,13 @@ import yaml import shutil import hashlib - +import json +from logging import getLogger from difflib import unified_diff from datetime import datetime from moulinette import m18n -from moulinette.utils import log, filesystem +from moulinette.utils.filesystem import mkdir from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError @@ -37,7 +38,7 @@ PENDING_CONF_DIR = os.path.join(BASE_CONF_PATH, "pending") REGEN_CONF_FILE = "/etc/yunohost/regenconf.yml" -logger = log.getActionLogger("yunohost.regenconf") +logger = getLogger("yunohost.regenconf") # FIXME : those ain't just services anymore ... what are we supposed to do with this ... @@ -63,6 +64,8 @@ def regen_conf( """ + from yunohost.settings import settings_get + if names is None: names = [] @@ -102,7 +105,7 @@ def regen_conf( for name in names: shutil.rmtree(os.path.join(PENDING_CONF_DIR, name), ignore_errors=True) else: - filesystem.mkdir(PENDING_CONF_DIR, 0o755, True) + mkdir(PENDING_CONF_DIR, 0o755, True) # Execute hooks for pre-regen # element 2 and 3 with empty string is because of legacy... @@ -111,7 +114,7 @@ def regen_conf( def _pre_call(name, priority, path, args): # create the pending conf directory for the category category_pending_path = os.path.join(PENDING_CONF_DIR, name) - filesystem.mkdir(category_pending_path, 0o755, True, uid="root") + mkdir(category_pending_path, 0o755, True, uid="root") # return the arguments to pass to the script return pre_args + [ @@ -140,6 +143,10 @@ def _pre_call(name, priority, path, args): domain_list(exclude_subdomains=True)["domains"] ) env["YNH_CONTEXT"] = "regenconf" + # perf: Export all global settings as a environment variable + # so that scripts dont have to call 'yunohost settings get' manually + # which is painful performance-wise + env["YNH_SETTINGS"] = json.dumps(settings_get("", export=True)) pre_result = hook_callback("conf_regen", names, pre_callback=_pre_call, env=env) @@ -432,7 +439,13 @@ def _get_regenconf_infos(): """ try: with open(REGEN_CONF_FILE, "r") as f: - return yaml.safe_load(f) + data = yaml.safe_load(f) + # Cleanup legacy + if "metronome" in data: + del data["metronome"] + if "rspamd" in data: + del data["rspamd"] + return data except Exception: return {} @@ -622,7 +635,7 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): backup_dir = os.path.dirname(backup_path) if not os.path.isdir(backup_dir): - filesystem.mkdir(backup_dir, 0o755, True) + mkdir(backup_dir, 0o755, True) shutil.copy2(system_conf, backup_path) logger.debug( @@ -637,7 +650,7 @@ def _process_regen_conf(system_conf, new_conf=None, save=True): system_dir = os.path.dirname(system_conf) if not os.path.isdir(system_dir): - filesystem.mkdir(system_dir, 0o755, True) + mkdir(system_dir, 0o755, True) shutil.copyfile(new_conf, system_conf) logger.debug(m18n.n("regenconf_file_updated", conf=system_conf)) diff --git a/src/service.py b/src/service.py index 97a5f4d657..d53d011088 100644 --- a/src/service.py +++ b/src/service.py @@ -21,7 +21,7 @@ import time import yaml import subprocess - +from logging import getLogger from glob import glob from datetime import datetime @@ -29,7 +29,6 @@ from yunohost.diagnosis import diagnosis_ignore, diagnosis_unignore from yunohost.utils.error import YunohostError, YunohostValidationError from moulinette.utils.process import check_output -from moulinette.utils.log import getActionLogger from moulinette.utils.filesystem import ( read_file, append_to_file, @@ -43,7 +42,7 @@ SERVICES_CONF = "/etc/yunohost/services.yml" SERVICES_CONF_BASE = "/usr/share/yunohost/conf/yunohost/services.yml" -logger = getActionLogger("yunohost.service") +logger = getLogger("yunohost.service") def service_add( @@ -716,10 +715,6 @@ def _get_services(): "category": "web", } - # Ignore metronome entirely if XMPP was disabled on all domains - if "metronome" in services and not glob("/etc/metronome/conf.d/*.cfg.lua"): - del services["metronome"] - # Remove legacy /var/log/daemon.log and /var/log/syslog from log entries # because they are too general. Instead, now the journalctl log is # returned by default which is more relevant. diff --git a/src/settings.py b/src/settings.py index 4d2c365b7e..1027a55a0b 100644 --- a/src/settings.py +++ b/src/settings.py @@ -18,18 +18,34 @@ # import os import subprocess +from logging import getLogger +from typing import TYPE_CHECKING, Any, Union from moulinette import m18n from yunohost.utils.error import YunohostError, YunohostValidationError -from yunohost.utils.configpanel import ConfigPanel +from yunohost.utils.configpanel import ConfigPanel, parse_filter_key from yunohost.utils.form import BaseOption -from moulinette.utils.log import getActionLogger from yunohost.regenconf import regen_conf from yunohost.firewall import firewall_reload from yunohost.log import is_unit_operation -from yunohost.utils.legacy import translate_legacy_settings_to_configpanel_settings -logger = getActionLogger("yunohost.settings") +if TYPE_CHECKING: + from typing import cast + + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + + from moulinette.utils.log import MoulinetteLogger + from yunohost.log import OperationLogger + from yunohost.utils.configpanel import ( + ConfigPanelGetMode, + ConfigPanelModel, + RawSettings, + ) + from yunohost.utils.form import FormModel + + logger = cast(MoulinetteLogger, getLogger("yunohost.settings")) +else: + logger = getLogger("yunohost.settings") SETTINGS_PATH = "/etc/yunohost/settings.yml" @@ -55,7 +71,6 @@ def settings_get(key="", full=False, export=False): mode = "classic" settings = SettingsConfigPanel() - key = translate_legacy_settings_to_configpanel_settings(key) return settings.get(key, mode) @@ -84,7 +99,6 @@ def settings_set(operation_logger, key=None, value=None, args=None, args_file=No """ BaseOption.operation_logger = operation_logger settings = SettingsConfigPanel() - key = translate_legacy_settings_to_configpanel_settings(key) return settings.set(key, value, args, args_file, operation_logger=operation_logger) @@ -99,7 +113,6 @@ def settings_reset(operation_logger, key): """ settings = SettingsConfigPanel() - key = translate_legacy_settings_to_configpanel_settings(key) return settings.reset(key, operation_logger=operation_logger) @@ -120,47 +133,49 @@ class SettingsConfigPanel(ConfigPanel): entity_type = "global" save_path_tpl = SETTINGS_PATH save_mode = "diff" - virtual_settings = ["root_password", "root_password_confirm", "passwordless_sudo"] + virtual_settings = {"root_password", "root_password_confirm", "passwordless_sudo"} - def __init__(self, config_path=None, save_path=None, creation=False): + def __init__(self, config_path=None, save_path=None, creation=False) -> None: super().__init__("settings") - def get(self, key="", mode="classic"): + def get( + self, key: Union[str, None] = None, mode: "ConfigPanelGetMode" = "classic" + ) -> Any: result = super().get(key=key, mode=mode) - if mode == "full": - for panel, section, option in self._iterate(): - if m18n.key_exists(self.config["i18n"] + "_" + option["id"] + "_help"): - option["help"] = m18n.n( - self.config["i18n"] + "_" + option["id"] + "_help" - ) - return self.config - # Dirty hack to let settings_get() to work from a python script if isinstance(result, str) and result in ["True", "False"]: result = bool(result == "True") return result - def reset(self, key="", operation_logger=None): - self.filter_key = key + def reset( + self, + key: Union[str, None] = None, + operation_logger: Union["OperationLogger", None] = None, + ) -> None: + self.filter_key = parse_filter_key(key) # Read config panel toml - self._get_config_panel() + self.config, self.form = self._get_config_panel(prevalidate=True) - if not self.config: - raise YunohostValidationError("config_no_panel") + # FIXME find a better way to exclude previous settings + previous_settings = self.form.dict() - # Replace all values with default values - self.values = self._get_default_values() + for option in self.config.options: + if not option.readonly and ( + option.optional or option.default not in {None, ""} + ): + # FIXME Mypy complains about option.default not being a valid type for normalize but this should be ok + self.form[option.id] = option.normalize(option.default, option) # type: ignore - BaseOption.operation_logger = operation_logger + # FIXME Not sure if this is need (redact call to operation logger does it on all the instances) + # BaseOption.operation_logger = operation_logger if operation_logger: operation_logger.start() - try: - self._apply() + self._apply(self.form, self.config, previous_settings) except YunohostError: raise # Script got manually interrupted ... @@ -178,53 +193,41 @@ def reset(self, key="", operation_logger=None): raise logger.success(m18n.n("global_settings_reset_success")) - operation_logger.success() - - def _get_raw_config(self): - toml = super()._get_raw_config() - - # Dynamic choice list for portal themes - THEMEDIR = "/usr/share/ssowat/portal/assets/themes/" - try: - themes = [d for d in os.listdir(THEMEDIR) if os.path.isdir(THEMEDIR + d)] - except Exception: - themes = ["unsplash", "vapor", "light", "default", "clouds"] - toml["misc"]["portal"]["portal_theme"]["choices"] = themes - return toml + if operation_logger: + operation_logger.success() - def _get_raw_settings(self): - super()._get_raw_settings() + def _get_raw_settings(self) -> "RawSettings": + raw_settings = super()._get_raw_settings() # Specific logic for those settings who are "virtual" settings # and only meant to have a custom setter mapped to tools_rootpw - self.values["root_password"] = "" - self.values["root_password_confirm"] = "" + raw_settings["root_password"] = "" + raw_settings["root_password_confirm"] = "" # Specific logic for virtual setting "passwordless_sudo" try: from yunohost.utils.ldap import _get_ldap_interface ldap = _get_ldap_interface() - self.values["passwordless_sudo"] = "!authenticate" in ldap.search( + raw_settings["passwordless_sudo"] = "!authenticate" in ldap.search( "ou=sudo", "cn=admins", ["sudoOption"] )[0].get("sudoOption", []) except Exception: - self.values["passwordless_sudo"] = False + raw_settings["passwordless_sudo"] = False - def _apply(self): - root_password = self.new_values.pop("root_password", None) - root_password_confirm = self.new_values.pop("root_password_confirm", None) - passwordless_sudo = self.new_values.pop("passwordless_sudo", None) - - self.values = { - k: v for k, v in self.values.items() if k not in self.virtual_settings - } - self.new_values = { - k: v for k, v in self.new_values.items() if k not in self.virtual_settings - } + return raw_settings - assert all(v not in self.future_values for v in self.virtual_settings) + def _apply( + self, + form: "FormModel", + config: "ConfigPanelModel", + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ) -> None: + root_password = form.get("root_password", None) + root_password_confirm = form.get("root_password_confirm", None) + passwordless_sudo = form.get("passwordless_sudo", None) if root_password and root_password.strip(): if root_password != root_password_confirm: @@ -243,15 +246,20 @@ def _apply(self): {"sudoOption": ["!authenticate"] if passwordless_sudo else []}, ) - super()._apply() - - settings = { - k: v for k, v in self.future_values.items() if self.values.get(k) != v + # First save settings except virtual + default ones + super()._apply(form, config, previous_settings, exclude=self.virtual_settings) + next_settings = { + k: v + for k, v in form.dict(exclude=self.virtual_settings).items() + if previous_settings.get(k) != v } - for setting_name, value in settings.items(): + + for setting_name, value in next_settings.items(): try: + # FIXME not sure to understand why we need the previous value if + # updated_settings has already been filtered trigger_post_change_hook( - setting_name, self.values.get(setting_name), value + setting_name, previous_settings.get(setting_name), value ) except Exception as e: logger.error(f"Post-change hook for setting failed : {e}") @@ -302,7 +310,6 @@ def regen_ssowatconf(setting_name, old_value, new_value): @post_change_hook("tls_passthrough_enabled") @post_change_hook("tls_passthrough_list") -@post_change_hook("ssowat_panel_overlay_enabled") @post_change_hook("nginx_redirect_to_https") @post_change_hook("nginx_compatibility") @post_change_hook("webadmin_allowlist_enabled") @@ -347,12 +354,12 @@ def reconfigure_postfix(setting_name, old_value, new_value): @post_change_hook("pop3_enabled") def reconfigure_dovecot(setting_name, old_value, new_value): - dovecot_package = "dovecot-pop3d" environment = os.environ.copy() environment.update({"DEBIAN_FRONTEND": "noninteractive"}) - if new_value is True: + # Depending on how consistent the config panel is, it may spit 1 or True or ..? ... + if new_value: command = [ "apt-get", "-y", @@ -360,7 +367,7 @@ def reconfigure_dovecot(setting_name, old_value, new_value): "-o Dpkg::Options::=--force-confdef", "-o Dpkg::Options::=--force-confold", "install", - dovecot_package, + "dovecot-pop3d", ] subprocess.call(command, env=environment) if old_value != new_value: @@ -368,5 +375,5 @@ def reconfigure_dovecot(setting_name, old_value, new_value): else: if old_value != new_value: regen_conf(names=["dovecot"]) - command = ["apt-get", "-y", "remove", dovecot_package] + command = ["apt-get", "-y", "remove", "dovecot-pop3d"] subprocess.call(command, env=environment) diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 393c335644..082b518433 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -1,5 +1,6 @@ import os import pytest +from unittest.mock import Mock import moulinette from moulinette import m18n, Moulinette @@ -12,7 +13,7 @@ def clone_test_app(request): cwd = os.path.split(os.path.realpath(__file__))[0] if not os.path.exists(cwd + "/apps"): - os.system("git clone https://github.com/YunoHost/test_apps %s/apps" % cwd) + os.system(f"git clone https://github.com/YunoHost/test_apps {cwd}/apps --depth 1") else: os.system("cd %s/apps && git pull > /dev/null 2>&1" % cwd) @@ -23,11 +24,15 @@ def get_test_apps_dir(): @contextmanager -def message(mocker, key, **kwargs): - mocker.spy(m18n, "n") +def message(key, **kwargs): + m = Mock(wraps=m18n.n) + old_m18n = m18n.n + m18n.n = m yield - m18n.n.assert_any_call(key, **kwargs) - + try: + m.assert_any_call(key, **kwargs) + finally: + m18n.n = old_m18n @contextmanager def raiseYunohostError(mocker, key, **kwargs): @@ -38,10 +43,6 @@ def raiseYunohostError(mocker, key, **kwargs): assert e_info._excinfo[1].kwargs == kwargs -def pytest_addoption(parser): - parser.addoption("--yunodebug", action="store_true", default=False) - - # # Tweak translator to raise exceptions if string keys are not defined # # @@ -67,11 +68,14 @@ def new_translate(self, key, *args, **kwargs): def pytest_cmdline_main(config): import sys + from pathlib import Path - sys.path.insert(0, "/usr/lib/moulinette/") - import yunohost + # Tweak python path such that "import yunohost" imports "this" code and not the one from /usr/lib/python3/dist-packages + code_root = str(Path(__file__).parent.parent.parent) + sys.path.insert(0, code_root) - yunohost.init(debug=config.option.yunodebug) + import yunohost + yunohost.init() class DummyInterface: type = "cli" diff --git a/src/tests/test_app_catalog.py b/src/tests/test_app_catalog.py index f7363dabef..f436dd8774 100644 --- a/src/tests/test_app_catalog.py +++ b/src/tests/test_app_catalog.py @@ -5,12 +5,13 @@ import glob import shutil +from .conftest import message + from moulinette import m18n from moulinette.utils.filesystem import read_json, write_to_json, write_to_yaml from yunohost.utils.error import YunohostError from yunohost.app_catalog import ( - _initialize_apps_catalog_system, _read_apps_catalog_list, _update_apps_catalog, _actual_apps_catalog_api_url, @@ -64,44 +65,17 @@ def teardown_function(function): # -def test_apps_catalog_init(mocker): - # Cache is empty - assert not glob.glob(APPS_CATALOG_CACHE + "/*") - # Conf doesn't exist yet - assert not os.path.exists(APPS_CATALOG_CONF) - - # Initialize ... - mocker.spy(m18n, "n") - _initialize_apps_catalog_system() - m18n.n.assert_any_call("apps_catalog_init_success") - - # And a conf with at least one list - assert os.path.exists(APPS_CATALOG_CONF) - apps_catalog_list = _read_apps_catalog_list() - assert len(apps_catalog_list) - - # Cache is expected to still be empty though - # (if we did update the apps_catalog during init, - # we couldn't differentiate easily exceptions - # related to lack of network connectivity) - assert not glob.glob(APPS_CATALOG_CACHE + "/*") - - def test_apps_catalog_emptylist(): - # Initialize ... - _initialize_apps_catalog_system() # Let's imagine somebody removed the default apps catalog because uh idk they dont want to use our default apps catalog os.system("rm %s" % APPS_CATALOG_CONF) os.system("touch %s" % APPS_CATALOG_CONF) apps_catalog_list = _read_apps_catalog_list() - assert not len(apps_catalog_list) + assert len(apps_catalog_list) == 0 def test_apps_catalog_update_nominal(mocker): - # Initialize ... - _initialize_apps_catalog_system() # Cache is empty assert not glob.glob(APPS_CATALOG_CACHE + "/*") @@ -133,8 +107,6 @@ def test_apps_catalog_update_nominal(mocker): def test_apps_catalog_update_404(mocker): - # Initialize ... - _initialize_apps_catalog_system() with requests_mock.Mocker() as m: # 404 error @@ -147,8 +119,6 @@ def test_apps_catalog_update_404(mocker): def test_apps_catalog_update_timeout(mocker): - # Initialize ... - _initialize_apps_catalog_system() with requests_mock.Mocker() as m: # Timeout @@ -163,8 +133,6 @@ def test_apps_catalog_update_timeout(mocker): def test_apps_catalog_update_sslerror(mocker): - # Initialize ... - _initialize_apps_catalog_system() with requests_mock.Mocker() as m: # SSL error @@ -179,8 +147,6 @@ def test_apps_catalog_update_sslerror(mocker): def test_apps_catalog_update_corrupted(mocker): - # Initialize ... - _initialize_apps_catalog_system() with requests_mock.Mocker() as m: # Corrupted json @@ -195,8 +161,6 @@ def test_apps_catalog_update_corrupted(mocker): def test_apps_catalog_load_with_empty_cache(mocker): - # Initialize ... - _initialize_apps_catalog_system() # Cache is empty assert not glob.glob(APPS_CATALOG_CACHE + "/*") @@ -221,8 +185,6 @@ def test_apps_catalog_load_with_empty_cache(mocker): def test_apps_catalog_load_with_conflicts_between_lists(mocker): - # Initialize ... - _initialize_apps_catalog_system() conf = [ {"id": "default", "url": APPS_CATALOG_DEFAULT_URL}, @@ -258,13 +220,10 @@ def test_apps_catalog_load_with_conflicts_between_lists(mocker): assert "bar" in app_dict.keys() -def test_apps_catalog_load_with_oudated_api_version(mocker): - # Initialize ... - _initialize_apps_catalog_system() +def test_apps_catalog_load_with_outdated_api_version(): # Update with requests_mock.Mocker() as m: - mocker.spy(m18n, "n") m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) _update_apps_catalog() @@ -282,10 +241,8 @@ def test_apps_catalog_load_with_oudated_api_version(mocker): with requests_mock.Mocker() as m: # Mock the server response with a dummy apps catalog m.register_uri("GET", APPS_CATALOG_DEFAULT_URL_FULL, text=DUMMY_APP_CATALOG) - - mocker.spy(m18n, "n") - app_dict = _load_apps_catalog()["apps"] - m18n.n.assert_any_call("apps_catalog_update_success") + with message("apps_catalog_update_success"): + app_dict = _load_apps_catalog()["apps"] assert "foo" in app_dict.keys() assert "bar" in app_dict.keys() diff --git a/src/tests/test_app_config.py b/src/tests/test_app_config.py index 4a74cbc0de..24abdc5dc1 100644 --- a/src/tests/test_app_config.py +++ b/src/tests/test_app_config.py @@ -125,9 +125,9 @@ def test_app_config_get_nonexistentstuff(config_app): with pytest.raises(YunohostValidationError): app_config_get(config_app, "main.components.nonexistent") - app_setting(config_app, "boolean", delete=True) + app_setting(config_app, "number", delete=True) with pytest.raises(YunohostError): - app_config_get(config_app, "main.components.boolean") + app_config_get(config_app, "main.components.number") def test_app_config_regular_setting(config_app): diff --git a/src/tests/test_app_resources.py b/src/tests/test_app_resources.py index 1375735095..d39c920d7a 100644 --- a/src/tests/test_app_resources.py +++ b/src/tests/test_app_resources.py @@ -44,6 +44,7 @@ def setup_function(function): os.system("echo 'id: testapp' > /etc/yunohost/apps/testapp/settings.yml") os.system("echo 'packaging_format = 2' > /etc/yunohost/apps/testapp/manifest.toml") os.system("echo 'id = \"testapp\"' >> /etc/yunohost/apps/testapp/manifest.toml") + os.system("echo 'description.en = \"A dummy app to test app resources\"' >> /etc/yunohost/apps/testapp/manifest.toml") def teardown_function(function): @@ -55,7 +56,7 @@ def clean(): os.system("rm -rf /etc/yunohost/apps/testapp") os.system("rm -rf /var/www/testapp") os.system("rm -rf /home/yunohost.app/testapp") - os.system("apt remove lolcat sl nyancat yarn >/dev/null 2>/dev/null") + os.system("apt remove lolcat sl nyancat influxdb2 >/dev/null 2>/dev/null") os.system("userdel testapp 2>/dev/null") for p in user_permission_list()["permissions"]: @@ -294,17 +295,17 @@ def test_resource_apt(): conf = { "packages": "nyancat, sl", "extras": { - "yarn": { - "repo": "deb https://dl.yarnpkg.com/debian/ stable main", - "key": "https://dl.yarnpkg.com/debian/pubkey.gpg", - "packages": "yarn", + "influxdb": { + "repo": "deb https://repos.influxdata.com/debian stable main", + "key": "https://repos.influxdata.com/influxdata-archive_compat.key", + "packages": "influxdb2", } }, } assert os.system("dpkg --list | grep -q 'ii *nyancat '") != 0 assert os.system("dpkg --list | grep -q 'ii *sl '") != 0 - assert os.system("dpkg --list | grep -q 'ii *yarn '") != 0 + assert os.system("dpkg --list | grep -q 'ii *influxdb2 '") != 0 assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0 assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") != 0 @@ -312,7 +313,7 @@ def test_resource_apt(): assert os.system("dpkg --list | grep -q 'ii *nyancat '") == 0 assert os.system("dpkg --list | grep -q 'ii *sl '") == 0 - assert os.system("dpkg --list | grep -q 'ii *yarn '") == 0 + assert os.system("dpkg --list | grep -q 'ii *influxdb2 '") == 0 assert ( os.system("dpkg --list | grep -q 'ii *lolcat '") != 0 ) # Lolcat shouldnt be installed yet @@ -323,7 +324,7 @@ def test_resource_apt(): assert os.system("dpkg --list | grep -q 'ii *nyancat '") == 0 assert os.system("dpkg --list | grep -q 'ii *sl '") == 0 - assert os.system("dpkg --list | grep -q 'ii *yarn '") == 0 + assert os.system("dpkg --list | grep -q 'ii *influxdb2 '") == 0 assert os.system("dpkg --list | grep -q 'ii *lolcat '") == 0 assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") == 0 @@ -331,7 +332,7 @@ def test_resource_apt(): assert os.system("dpkg --list | grep -q 'ii *nyancat '") != 0 assert os.system("dpkg --list | grep -q 'ii *sl '") != 0 - assert os.system("dpkg --list | grep -q 'ii *yarn '") != 0 + assert os.system("dpkg --list | grep -q 'ii *influxdb2 '") != 0 assert os.system("dpkg --list | grep -q 'ii *lolcat '") != 0 assert os.system("dpkg --list | grep -q 'ii *testapp-ynh-deps '") != 0 diff --git a/src/tests/test_apps.py b/src/tests/test_apps.py index d7a591a362..ede986b100 100644 --- a/src/tests/test_apps.py +++ b/src/tests/test_apps.py @@ -394,9 +394,9 @@ def test_legacy_app_install_private(secondary_domain): assert app_is_not_installed(secondary_domain, "legacy_app") -def test_legacy_app_install_unknown_domain(mocker): +def test_legacy_app_install_unknown_domain(): with pytest.raises(YunohostError): - with message(mocker, "app_argument_invalid"): + with message("app_argument_invalid"): install_legacy_app("whatever.nope", "/legacy") assert app_is_not_installed("whatever.nope", "legacy_app") @@ -423,12 +423,12 @@ def test_legacy_app_install_multiple_instances(secondary_domain): assert app_is_not_installed(secondary_domain, "legacy_app__2") -def test_legacy_app_install_path_unavailable(mocker, secondary_domain): +def test_legacy_app_install_path_unavailable(secondary_domain): # These will be removed in teardown install_legacy_app(secondary_domain, "/legacy") with pytest.raises(YunohostError): - with message(mocker, "app_location_unavailable"): + with message("app_location_unavailable"): install_legacy_app(secondary_domain, "/") assert app_is_installed(secondary_domain, "legacy_app") @@ -444,19 +444,19 @@ def test_legacy_app_install_with_nginx_down(mocker, secondary_domain): install_legacy_app(secondary_domain, "/legacy") -def test_legacy_app_failed_install(mocker, secondary_domain): +def test_legacy_app_failed_install(secondary_domain): # This will conflict with the folder that the app # attempts to create, making the install fail mkdir("/var/www/legacy_app/", 0o750) with pytest.raises(YunohostError): - with message(mocker, "app_install_script_failed"): + with message("app_install_script_failed"): install_legacy_app(secondary_domain, "/legacy") assert app_is_not_installed(secondary_domain, "legacy_app") -def test_legacy_app_failed_remove(mocker, secondary_domain): +def test_legacy_app_failed_remove(secondary_domain): install_legacy_app(secondary_domain, "/legacy") # The remove script runs with set -eu and attempt to remove this @@ -488,52 +488,52 @@ def test_full_domain_app_with_conflicts(mocker, secondary_domain): install_full_domain_app(secondary_domain) -def test_systemfuckedup_during_app_install(mocker, secondary_domain): +def test_systemfuckedup_during_app_install(secondary_domain): with pytest.raises(YunohostError): - with message(mocker, "app_install_failed"): - with message(mocker, "app_action_broke_system"): + with message("app_install_failed"): + with message("app_action_broke_system"): install_break_yo_system(secondary_domain, breakwhat="install") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_remove(mocker, secondary_domain): +def test_systemfuckedup_during_app_remove(secondary_domain): install_break_yo_system(secondary_domain, breakwhat="remove") with pytest.raises(YunohostError): - with message(mocker, "app_action_broke_system"): - with message(mocker, "app_removed"): + with message("app_action_broke_system"): + with message("app_removed"): app_remove("break_yo_system") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_install_and_remove(mocker, secondary_domain): +def test_systemfuckedup_during_app_install_and_remove(secondary_domain): with pytest.raises(YunohostError): - with message(mocker, "app_install_failed"): - with message(mocker, "app_action_broke_system"): + with message("app_install_failed"): + with message("app_action_broke_system"): install_break_yo_system(secondary_domain, breakwhat="everything") assert app_is_not_installed(secondary_domain, "break_yo_system") -def test_systemfuckedup_during_app_upgrade(mocker, secondary_domain): +def test_systemfuckedup_during_app_upgrade(secondary_domain): install_break_yo_system(secondary_domain, breakwhat="upgrade") with pytest.raises(YunohostError): - with message(mocker, "app_action_broke_system"): + with message("app_action_broke_system"): app_upgrade( "break_yo_system", file=os.path.join(get_test_apps_dir(), "break_yo_system_ynh"), ) -def test_failed_multiple_app_upgrade(mocker, secondary_domain): +def test_failed_multiple_app_upgrade(secondary_domain): install_legacy_app(secondary_domain, "/legacy") install_break_yo_system(secondary_domain, breakwhat="upgrade") with pytest.raises(YunohostError): - with message(mocker, "app_not_upgraded"): + with message("app_not_upgraded"): app_upgrade( ["break_yo_system", "legacy_app"], file={ diff --git a/src/tests/test_appurl.py b/src/tests/test_appurl.py index d0c55f7328..8d1c43f15f 100644 --- a/src/tests/test_appurl.py +++ b/src/tests/test_appurl.py @@ -217,10 +217,6 @@ def test_normalize_permission_path_with_bad_regex(): ) # Full Regex - with pytest.raises(YunohostError): - _validate_and_sanitize_permission_url( - "re:" + maindomain + "/yolo?+/", maindomain + "/path", "test_permission" - ) with pytest.raises(YunohostError): _validate_and_sanitize_permission_url( "re:" + maindomain + "/yolo[1-9]**/", diff --git a/src/tests/test_backuprestore.py b/src/tests/test_backuprestore.py index bca1b29a5c..3f30a54925 100644 --- a/src/tests/test_backuprestore.py +++ b/src/tests/test_backuprestore.py @@ -49,8 +49,8 @@ def setup_function(function): for m in function.__dict__.get("pytestmark", []) } - if "with_wordpress_archive_from_4p2" in markers: - add_archive_wordpress_from_4p2() + if "with_wordpress_archive_from_11p2" in markers: + add_archive_wordpress_from_11p2() assert len(backup_list()["archives"]) == 1 if "with_legacy_app_installed" in markers: @@ -72,8 +72,8 @@ def setup_function(function): ) assert app_is_installed("backup_recommended_app") - if "with_system_archive_from_4p2" in markers: - add_archive_system_from_4p2() + if "with_system_archive_from_11p2" in markers: + add_archive_system_from_11p2() assert len(backup_list()["archives"]) == 1 if "with_permission_app_installed" in markers: @@ -148,7 +148,7 @@ def app_is_installed(app): def backup_test_dependencies_are_met(): # Dummy test apps (or backup archives) assert os.path.exists( - os.path.join(get_test_apps_dir(), "backup_wordpress_from_4p2") + os.path.join(get_test_apps_dir(), "backup_wordpress_from_11p2") ) assert os.path.exists(os.path.join(get_test_apps_dir(), "legacy_app_ynh")) assert os.path.exists( @@ -211,23 +211,23 @@ def install_app(app, path, additionnal_args=""): ) -def add_archive_wordpress_from_4p2(): +def add_archive_wordpress_from_11p2(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join(get_test_apps_dir(), "backup_wordpress_from_4p2/backup.tar") - + " /home/yunohost.backup/archives/backup_wordpress_from_4p2.tar" + + os.path.join(get_test_apps_dir(), "backup_wordpress_from_11p2/backup.tar") + + " /home/yunohost.backup/archives/backup_wordpress_from_11p2.tar" ) -def add_archive_system_from_4p2(): +def add_archive_system_from_11p2(): os.system("mkdir -p /home/yunohost.backup/archives") os.system( "cp " - + os.path.join(get_test_apps_dir(), "backup_system_from_4p2/backup.tar") - + " /home/yunohost.backup/archives/backup_system_from_4p2.tar" + + os.path.join(get_test_apps_dir(), "backup_system_from_11p2/backup.tar") + + " /home/yunohost.backup/archives/backup_system_from_11p2.tar" ) @@ -236,10 +236,10 @@ def add_archive_system_from_4p2(): # -def test_backup_only_ldap(mocker): +def test_backup_only_ldap(): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=["conf_ldap"], apps=None) archives = backup_list()["archives"] @@ -253,7 +253,7 @@ def test_backup_only_ldap(mocker): def test_backup_system_part_that_does_not_exists(mocker): # Create the backup - with message(mocker, "backup_hook_unknown", hook="doesnt_exist"): + with message("backup_hook_unknown", hook="doesnt_exist"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=["doesnt_exist"], apps=None) @@ -263,10 +263,10 @@ def test_backup_system_part_that_does_not_exists(mocker): # -def test_backup_and_restore_all_sys(mocker): +def test_backup_and_restore_all_sys(): name = random_ascii(8) # Create the backup - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] @@ -284,7 +284,7 @@ def test_backup_and_restore_all_sys(mocker): assert not os.path.exists("/etc/ssowat/conf.json") # Restore the backup - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore(name=archives[0], force=True, system=[], apps=None) # Check ssowat conf is back @@ -292,22 +292,22 @@ def test_backup_and_restore_all_sys(mocker): # -# System restore from 3.8 # +# System restore from 11.2 # # -@pytest.mark.with_system_archive_from_4p2 -def test_restore_system_from_Ynh4p2(monkeypatch, mocker): +@pytest.mark.with_system_archive_from_11p2 +def test_restore_system_from_Ynh11p2(monkeypatch): name = random_ascii(8) # Backup current system - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] assert len(archives) == 2 - # Restore system archive from 3.8 + # Restore system archive from 11.2 try: - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( name=backup_list()["archives"][1], system=[], apps=None, force=True ) @@ -336,7 +336,7 @@ def custom_hook_exec(name, *args, **kwargs): # with the expected error message key monkeypatch.setattr("yunohost.backup.hook_exec", custom_hook_exec) - with message(mocker, "backup_app_failed", app="backup_recommended_app"): + with message("backup_app_failed", app="backup_recommended_app"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["backup_recommended_app"]) @@ -363,7 +363,7 @@ def custom_free_space_in_directory(dirpath): def test_backup_app_not_installed(mocker): assert not _is_installed("wordpress") - with message(mocker, "unbackup_app", app="wordpress"): + with message("unbackup_app", app="wordpress"): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["wordpress"]) @@ -375,14 +375,14 @@ def test_backup_app_with_no_backup_script(mocker): assert not os.path.exists(backup_script) with message( - mocker, "backup_with_no_backup_script_for_app", app="backup_recommended_app" + "backup_with_no_backup_script_for_app", app="backup_recommended_app" ): with raiseYunohostError(mocker, "backup_nothings_done"): backup_create(system=None, apps=["backup_recommended_app"]) @pytest.mark.with_backup_recommended_app_installed -def test_backup_app_with_no_restore_script(mocker): +def test_backup_app_with_no_restore_script(): restore_script = "/etc/yunohost/apps/backup_recommended_app/scripts/restore" os.system("rm %s" % restore_script) assert not os.path.exists(restore_script) @@ -391,16 +391,16 @@ def test_backup_app_with_no_restore_script(mocker): # user... with message( - mocker, "backup_with_no_restore_script_for_app", app="backup_recommended_app" + "backup_with_no_restore_script_for_app", app="backup_recommended_app" ): backup_create(system=None, apps=["backup_recommended_app"]) @pytest.mark.clean_opt_dir -def test_backup_with_different_output_directory(mocker): +def test_backup_with_different_output_directory(): name = random_ascii(8) # Create the backup - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create( system=["conf_ynh_settings"], apps=None, @@ -420,10 +420,10 @@ def test_backup_with_different_output_directory(mocker): @pytest.mark.clean_opt_dir -def test_backup_using_copy_method(mocker): +def test_backup_using_copy_method(): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create( system=["conf_ynh_settings"], apps=None, @@ -439,29 +439,30 @@ def test_backup_using_copy_method(mocker): # App restore # # - -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") -def test_restore_app_wordpress_from_Ynh4p2(mocker): - with message(mocker, "restore_complete"): +def test_restore_app_wordpress_from_Ynh11p2(): + with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_script_failure_handling(monkeypatch, mocker): def custom_hook_exec(name, *args, **kwargs): if os.path.basename(name).startswith("restore"): monkeypatch.undo() return (1, None) + else: + return (0, {}) monkeypatch.setattr("yunohost.hook.hook_exec", custom_hook_exec) assert not _is_installed("wordpress") - with message(mocker, "app_restore_script_failed"): + with message("app_restore_script_failed"): with raiseYunohostError(mocker, "restore_nothings_done"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] @@ -470,7 +471,7 @@ def custom_hook_exec(name, *args, **kwargs): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_app_not_enough_free_space(monkeypatch, mocker): def custom_free_space_in_directory(dirpath): return 0 @@ -489,12 +490,12 @@ def custom_free_space_in_directory(dirpath): assert not _is_installed("wordpress") -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_app_not_in_backup(mocker): assert not _is_installed("wordpress") assert not _is_installed("yoloswag") - with message(mocker, "backup_archive_app_not_found", app="yoloswag"): + with message("backup_archive_app_not_found", app="yoloswag"): with raiseYunohostError(mocker, "restore_nothings_done"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["yoloswag"] @@ -504,12 +505,12 @@ def test_restore_app_not_in_backup(mocker): assert not _is_installed("yoloswag") -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 @pytest.mark.with_custom_domain("yolo.test") def test_restore_app_already_installed(mocker): assert not _is_installed("wordpress") - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( system=None, name=backup_list()["archives"][0], apps=["wordpress"] ) @@ -525,22 +526,22 @@ def test_restore_app_already_installed(mocker): @pytest.mark.with_legacy_app_installed -def test_backup_and_restore_legacy_app(mocker): - _test_backup_and_restore_app(mocker, "legacy_app") +def test_backup_and_restore_legacy_app(): + _test_backup_and_restore_app("legacy_app") @pytest.mark.with_backup_recommended_app_installed -def test_backup_and_restore_recommended_app(mocker): - _test_backup_and_restore_app(mocker, "backup_recommended_app") +def test_backup_and_restore_recommended_app(): + _test_backup_and_restore_app("backup_recommended_app") @pytest.mark.with_backup_recommended_app_installed_with_ynh_restore -def test_backup_and_restore_with_ynh_restore(mocker): - _test_backup_and_restore_app(mocker, "backup_recommended_app") +def test_backup_and_restore_with_ynh_restore(): + _test_backup_and_restore_app("backup_recommended_app") @pytest.mark.with_permission_app_installed -def test_backup_and_restore_permission_app(mocker): +def test_backup_and_restore_permission_app(): res = user_permission_list(full=True)["permissions"] assert "permissions_app.main" in res assert "permissions_app.admin" in res @@ -554,7 +555,7 @@ def test_backup_and_restore_permission_app(mocker): assert res["permissions_app.admin"]["allowed"] == ["alice"] assert res["permissions_app.dev"]["allowed"] == [] - _test_backup_and_restore_app(mocker, "permissions_app") + _test_backup_and_restore_app("permissions_app") res = user_permission_list(full=True)["permissions"] assert "permissions_app.main" in res @@ -570,10 +571,10 @@ def test_backup_and_restore_permission_app(mocker): assert res["permissions_app.dev"]["allowed"] == [] -def _test_backup_and_restore_app(mocker, app): +def _test_backup_and_restore_app(app): # Create a backup of this app name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=None, apps=[app]) archives = backup_list()["archives"] @@ -590,7 +591,7 @@ def _test_backup_and_restore_app(mocker, app): assert app + ".main" not in user_permission_list()["permissions"] # Restore the app - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore(system=None, name=archives[0], apps=[app]) assert app_is_installed(app) @@ -616,34 +617,34 @@ def test_restore_archive_with_no_json(mocker): backup_restore(name="badbackup", force=True) -@pytest.mark.with_wordpress_archive_from_4p2 +@pytest.mark.with_wordpress_archive_from_11p2 def test_restore_archive_with_bad_archive(mocker): # Break the archive os.system( - "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_4p2.tar > /home/yunohost.backup/archives/backup_wordpress_from_4p2_bad.tar" + "head -n 1000 /home/yunohost.backup/archives/backup_wordpress_from_11p2.tar > /home/yunohost.backup/archives/backup_wordpress_from_11p2_bad.tar" ) - assert "backup_wordpress_from_4p2_bad" in backup_list()["archives"] + assert "backup_wordpress_from_11p2_bad" in backup_list()["archives"] with raiseYunohostError(mocker, "backup_archive_corrupted"): - backup_restore(name="backup_wordpress_from_4p2_bad", force=True) + backup_restore(name="backup_wordpress_from_11p2_bad", force=True) clean_tmp_backup_directory() -def test_restore_archive_with_custom_hook(mocker): +def test_restore_archive_with_custom_hook(): custom_restore_hook_folder = os.path.join(CUSTOM_HOOK_FOLDER, "restore") os.system("touch %s/99-yolo" % custom_restore_hook_folder) # Backup with custom hook system name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[], apps=None) archives = backup_list()["archives"] assert len(archives) == 1 # Restore system with custom hook - with message(mocker, "restore_complete"): + with message("restore_complete"): backup_restore( name=backup_list()["archives"][0], system=[], apps=None, force=True ) @@ -651,7 +652,7 @@ def test_restore_archive_with_custom_hook(mocker): os.system("rm %s/99-yolo" % custom_restore_hook_folder) -def test_backup_binds_are_readonly(mocker, monkeypatch): +def test_backup_binds_are_readonly(monkeypatch): def custom_mount_and_backup(self): self._organize_files() @@ -676,5 +677,5 @@ def custom_mount_and_backup(self): # Create the backup name = random_ascii(8) - with message(mocker, "backup_created", name=name): + with message("backup_created", name=name): backup_create(name=name, system=[]) diff --git a/src/tests/test_changeurl.py b/src/tests/test_changeurl.py index 04cb4a1a9e..b8ca20355f 100644 --- a/src/tests/test_changeurl.py +++ b/src/tests/test_changeurl.py @@ -39,7 +39,7 @@ def check_changeurl_app(path): assert appmap[maindomain][path]["id"] == "change_url_app" r = requests.get( - "https://127.0.0.1%s/" % path, headers={"domain": maindomain}, verify=False + "https://127.0.0.1%s/" % path, headers={"Host": maindomain}, verify=False ) assert r.status_code == 200 diff --git a/src/tests/test_dns.py b/src/tests/test_dns.py index e896d9c9f7..744e3e7899 100644 --- a/src/tests/test_dns.py +++ b/src/tests/test_dns.py @@ -49,19 +49,19 @@ def test_registrar_list_integrity(): def test_magic_guess_registrar_weird_domain(): - assert _get_registrar_config_section("yolo.tld")["registrar"]["value"] is None + assert _get_registrar_config_section("yolo.tld")["registrar"]["default"] is None def test_magic_guess_registrar_ovh(): assert ( - _get_registrar_config_section("yolo.yunohost.org")["registrar"]["value"] + _get_registrar_config_section("yolo.yunohost.org")["registrar"]["default"] == "ovh" ) def test_magic_guess_registrar_yunodyndns(): assert ( - _get_registrar_config_section("yolo.nohost.me")["registrar"]["value"] + _get_registrar_config_section("yolo.nohost.me")["registrar"]["default"] == "yunohost" ) diff --git a/src/tests/test_domains.py b/src/tests/test_domains.py index 13a9f63b87..273b8689aa 100644 --- a/src/tests/test_domains.py +++ b/src/tests/test_domains.py @@ -147,21 +147,19 @@ def test_change_main_domain(): # Domain settings testing def test_domain_config_get_default(): - assert domain_config_get(TEST_DOMAINS[0], "feature.xmpp.xmpp") == 1 - assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 0 + assert domain_config_get(TEST_DOMAINS[0], "feature.mail.mail_out") == 1 def test_domain_config_get_export(): - assert domain_config_get(TEST_DOMAINS[0], export=True)["xmpp"] == 1 - assert domain_config_get(TEST_DOMAINS[1], export=True)["xmpp"] == 0 + assert domain_config_get(TEST_DOMAINS[0], export=True)["mail_out"] == 1 def test_domain_config_set(): - assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 0 - domain_config_set(TEST_DOMAINS[1], "feature.xmpp.xmpp", "yes") - assert domain_config_get(TEST_DOMAINS[1], "feature.xmpp.xmpp") == 1 + assert domain_config_get(TEST_DOMAINS[1], "feature.mail.mail_out") == 1 + domain_config_set(TEST_DOMAINS[1], "feature.mail.mail_out", "no") + assert domain_config_get(TEST_DOMAINS[1], "feature.mail.mail_out") == 0 def test_domain_configs_unknown(): with pytest.raises(YunohostError): - domain_config_get(TEST_DOMAINS[2], "feature.xmpp.xmpp.xmpp") + domain_config_get(TEST_DOMAINS[2], "feature.foo.bar.baz") diff --git a/src/tests/test_ldapauth.py b/src/tests/test_ldapauth.py index 9e3ae36cc3..73cb09d274 100644 --- a/src/tests/test_ldapauth.py +++ b/src/tests/test_ldapauth.py @@ -59,7 +59,7 @@ def test_authenticate_with_wrong_password(): assert expected_msg in str(exception) -def test_authenticate_server_down(mocker): +def test_authenticate_server_down(): os.system("systemctl stop slapd && sleep 5") LDAPAuth().authenticate_credentials(credentials="alice:Yunohost") diff --git a/src/tests/test_permission.py b/src/tests/test_permission.py index 10bd018d23..f52825574f 100644 --- a/src/tests/test_permission.py +++ b/src/tests/test_permission.py @@ -158,7 +158,7 @@ def new_getaddrinfo(*args): socket.getaddrinfo = new_getaddrinfo - user_create("alice", maindomain, dummy_password, fullname="Alice White") + user_create("alice", maindomain, dummy_password, fullname="Alice White", admin=True) user_create("bob", maindomain, dummy_password, fullname="Bob Snow") _permission_create_with_dummy_app( permission="wiki.main", @@ -338,7 +338,7 @@ def check_LDAP_db_integrity(): def check_permission_for_apps(): # We check that the for each installed apps we have at last the "main" permission # and we don't have any permission linked to no apps. The only exception who is not liked to an app - # is mail, xmpp, and sftp + # is mail, and sftp app_perms = user_permission_list(ignore_system_perms=True)["permissions"].keys() @@ -355,7 +355,7 @@ def check_permission_for_apps(): def can_access_webpage(webpath, logged_as=None): webpath = webpath.rstrip("/") - sso_url = "https://" + maindomain + "/yunohost/sso/" + login_endpoint = f"https://{maindomain}/yunohost/portalapi/login" # Anonymous access if not logged_as: @@ -363,12 +363,11 @@ def can_access_webpage(webpath, logged_as=None): # Login as a user using dummy password else: with requests.Session() as session: - session.post( - sso_url, - data={"user": logged_as, "password": dummy_password}, + r = session.post( + login_endpoint, + data={"credentials": f"{logged_as}:{dummy_password}"}, headers={ - "Referer": sso_url, - "Content-Type": "application/x-www-form-urlencoded", + "X-Requested-With": "", }, verify=False, ) @@ -377,6 +376,15 @@ def can_access_webpage(webpath, logged_as=None): r = session.get(webpath, verify=False) # If we can't access it, we got redirected to the SSO + # with `r=` for anonymous access because they're encouraged to log-in, + # and `msg=access_denied` if we are logged but not allowed for this url + # with `r= + sso_url = f"https://{maindomain}/yunohost/sso/" + if not logged_as: + sso_url += "?r=" + else: + sso_url += "?msg=access_denied" + return not r.url.startswith(sso_url) @@ -389,7 +397,6 @@ def test_permission_list(): res = user_permission_list(full=True)["permissions"] assert "mail.main" in res - assert "xmpp.main" in res assert "wiki.main" in res assert "blog.main" in res @@ -435,8 +442,8 @@ def test_permission_list(): # -def test_permission_create_main(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_main(): + with message("permission_created", permission="site.main"): permission_create("site.main", allowed=["all_users"], protected=False) res = user_permission_list(full=True)["permissions"] @@ -446,8 +453,8 @@ def test_permission_create_main(mocker): assert res["site.main"]["protected"] is False -def test_permission_create_extra(mocker): - with message(mocker, "permission_created", permission="site.test"): +def test_permission_create_extra(): + with message("permission_created", permission="site.test"): permission_create("site.test") res = user_permission_list(full=True)["permissions"] @@ -466,8 +473,8 @@ def test_permission_create_with_specific_user(): assert res["site.test"]["allowed"] == ["alice"] -def test_permission_create_with_tile_management(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_tile_management(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -483,8 +490,8 @@ def test_permission_create_with_tile_management(mocker): assert res["site.main"]["show_tile"] is False -def test_permission_create_with_tile_management_with_main_default_value(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_tile_management_with_main_default_value(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -500,8 +507,8 @@ def test_permission_create_with_tile_management_with_main_default_value(mocker): assert res["site.main"]["show_tile"] is True -def test_permission_create_with_tile_management_with_not_main_default_value(mocker): - with message(mocker, "permission_created", permission="wiki.api"): +def test_permission_create_with_tile_management_with_not_main_default_value(): + with message("permission_created", permission="wiki.api"): _permission_create_with_dummy_app( "wiki.api", allowed=["all_users"], @@ -517,8 +524,8 @@ def test_permission_create_with_tile_management_with_not_main_default_value(mock assert res["wiki.api"]["show_tile"] is True -def test_permission_create_with_urls_management_without_url(mocker): - with message(mocker, "permission_created", permission="wiki.api"): +def test_permission_create_with_urls_management_without_url(): + with message("permission_created", permission="wiki.api"): _permission_create_with_dummy_app( "wiki.api", allowed=["all_users"], domain=maindomain, path="/site" ) @@ -530,8 +537,8 @@ def test_permission_create_with_urls_management_without_url(mocker): assert res["wiki.api"]["auth_header"] is True -def test_permission_create_with_urls_management_simple_domain(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_urls_management_simple_domain(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -553,8 +560,8 @@ def test_permission_create_with_urls_management_simple_domain(mocker): @pytest.mark.other_domains(number=2) -def test_permission_create_with_urls_management_multiple_domain(mocker): - with message(mocker, "permission_created", permission="site.main"): +def test_permission_create_with_urls_management_multiple_domain(): + with message("permission_created", permission="site.main"): _permission_create_with_dummy_app( "site.main", allowed=["all_users"], @@ -575,14 +582,14 @@ def test_permission_create_with_urls_management_multiple_domain(mocker): assert res["site.main"]["auth_header"] is True -def test_permission_delete(mocker): - with message(mocker, "permission_deleted", permission="wiki.main"): +def test_permission_delete(): + with message("permission_deleted", permission="wiki.main"): permission_delete("wiki.main", force=True) res = user_permission_list()["permissions"] assert "wiki.main" not in res - with message(mocker, "permission_deleted", permission="blog.api"): + with message("permission_deleted", permission="blog.api"): permission_delete("blog.api", force=False) res = user_permission_list()["permissions"] @@ -607,7 +614,6 @@ def test_permission_delete_doesnt_existing(mocker): assert "wiki.main" in res assert "blog.main" in res assert "mail.main" in res - assert "xmpp.main" in res def test_permission_delete_main_without_force(mocker): @@ -625,8 +631,8 @@ def test_permission_delete_main_without_force(mocker): # user side functions -def test_permission_add_group(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_add_group(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", add="alice") res = user_permission_list(full=True)["permissions"] @@ -634,8 +640,8 @@ def test_permission_add_group(mocker): assert set(res["wiki.main"]["corresponding_users"]) == {"alice", "bob"} -def test_permission_remove_group(mocker): - with message(mocker, "permission_updated", permission="blog.main"): +def test_permission_remove_group(): + with message("permission_updated", permission="blog.main"): user_permission_update("blog.main", remove="alice") res = user_permission_list(full=True)["permissions"] @@ -643,8 +649,8 @@ def test_permission_remove_group(mocker): assert res["blog.main"]["corresponding_users"] == [] -def test_permission_add_and_remove_group(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_add_and_remove_group(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", add="alice", remove="all_users") res = user_permission_list(full=True)["permissions"] @@ -652,9 +658,9 @@ def test_permission_add_and_remove_group(mocker): assert res["wiki.main"]["corresponding_users"] == ["alice"] -def test_permission_add_group_already_allowed(mocker): +def test_permission_add_group_already_allowed(): with message( - mocker, "permission_already_allowed", permission="blog.main", group="alice" + "permission_already_allowed", permission="blog.main", group="alice" ): user_permission_update("blog.main", add="alice") @@ -663,9 +669,9 @@ def test_permission_add_group_already_allowed(mocker): assert res["blog.main"]["corresponding_users"] == ["alice"] -def test_permission_remove_group_already_not_allowed(mocker): +def test_permission_remove_group_already_not_allowed(): with message( - mocker, "permission_already_disallowed", permission="blog.main", group="bob" + "permission_already_disallowed", permission="blog.main", group="bob" ): user_permission_update("blog.main", remove="bob") @@ -674,8 +680,8 @@ def test_permission_remove_group_already_not_allowed(mocker): assert res["blog.main"]["corresponding_users"] == ["alice"] -def test_permission_reset(mocker): - with message(mocker, "permission_updated", permission="blog.main"): +def test_permission_reset(): + with message("permission_updated", permission="blog.main"): user_permission_reset("blog.main") res = user_permission_list(full=True)["permissions"] @@ -693,42 +699,42 @@ def test_permission_reset_idempotency(): assert set(res["blog.main"]["corresponding_users"]) == {"alice", "bob"} -def test_permission_change_label(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_change_label(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", label="New Wiki") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["label"] == "New Wiki" -def test_permission_change_label_with_same_value(mocker): - with message(mocker, "permission_updated", permission="wiki.main"): +def test_permission_change_label_with_same_value(): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", label="Wiki") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["label"] == "Wiki" -def test_permission_switch_show_tile(mocker): +def test_permission_switch_show_tile(): # Note that from the actionmap the value is passed as string, not as bool # Try with lowercase - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="false") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["show_tile"] is False # Try with uppercase - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="TRUE") res = user_permission_list(full=True)["permissions"] assert res["wiki.main"]["show_tile"] is True -def test_permission_switch_show_tile_with_same_value(mocker): +def test_permission_switch_show_tile_with_same_value(): # Note that from the actionmap the value is passed as string, not as bool - with message(mocker, "permission_updated", permission="wiki.main"): + with message("permission_updated", permission="wiki.main"): user_permission_update("wiki.main", show_tile="True") res = user_permission_list(full=True)["permissions"] @@ -806,7 +812,7 @@ def test_permission_main_url_regex(): def test_permission_main_url_bad_regex(mocker): with raiseYunohostError(mocker, "invalid_regex"): - permission_url("blog.main", url="re:/[a-z]++reboy/.*") + permission_url("blog.main", url="re:/[a-z]+++reboy/.*") @pytest.mark.other_domains(number=1) @@ -837,7 +843,7 @@ def test_permission_add_additional_regex(): def test_permission_add_additional_bad_regex(mocker): with raiseYunohostError(mocker, "invalid_regex"): - permission_url("blog.main", add_url=["re:/[a-z]++reboy/.*"]) + permission_url("blog.main", add_url=["re:/[a-z]+++reboy/.*"]) def test_permission_remove_additional_url(): @@ -949,13 +955,7 @@ def test_ssowat_conf(): assert permissions["blog.main"]["public"] is False assert permissions["wiki.main"]["auth_header"] is False - assert permissions["blog.main"]["auth_header"] is True - - assert permissions["wiki.main"]["label"] == "Wiki" - assert permissions["blog.main"]["label"] == "Blog" - - assert permissions["wiki.main"]["show_tile"] is True - assert permissions["blog.main"]["show_tile"] is False + assert permissions["blog.main"]["auth_header"] == "basic-with-password" def test_show_tile_cant_be_enabled(): @@ -995,12 +995,11 @@ def test_show_tile_cant_be_enabled(): # -@pytest.mark.other_domains(number=1) def test_permission_app_install(): app_install( os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&domain_2=%s&path=%s&is_public=0&admin=%s" - % (maindomain, other_domains[0], "/urlpermissionapp", "alice"), + % (maindomain, maindomain, "/urlpermissionapp", "alice"), force=True, ) @@ -1028,12 +1027,11 @@ def test_permission_app_install(): assert maindomain + "/urlpermissionapp" in app_map(user="bob").keys() -@pytest.mark.other_domains(number=1) def test_permission_app_remove(): app_install( os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&domain_2=%s&path=%s&is_public=0&admin=%s" - % (maindomain, other_domains[0], "/urlpermissionapp", "alice"), + % (maindomain, maindomain, "/urlpermissionapp", "alice"), force=True, ) app_remove("permissions_app") @@ -1043,12 +1041,11 @@ def test_permission_app_remove(): assert not any(p.startswith("permissions_app.") for p in res.keys()) -@pytest.mark.other_domains(number=1) def test_permission_app_change_url(): app_install( os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s" - % (maindomain, other_domains[0], "/urlpermissionapp", "alice"), + % (maindomain, maindomain, "/urlpermissionapp", "alice"), force=True, ) @@ -1066,12 +1063,11 @@ def test_permission_app_change_url(): assert res["permissions_app.dev"]["url"] == "/dev" -@pytest.mark.other_domains(number=1) def test_permission_protection_management_by_helper(): app_install( os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s" - % (maindomain, other_domains[0], "/urlpermissionapp", "alice"), + % (maindomain, maindomain, "/urlpermissionapp", "alice"), force=True, ) @@ -1091,12 +1087,11 @@ def test_permission_protection_management_by_helper(): assert res["permissions_app.dev"]["protected"] is True -@pytest.mark.other_domains(number=1) def test_permission_app_propagation_on_ssowat(): app_install( os.path.join(get_test_apps_dir(), "permissions_app_ynh"), args="domain=%s&domain_2=%s&path=%s&is_public=1&admin=%s" - % (maindomain, other_domains[0], "/urlpermissionapp", "alice"), + % (maindomain, maindomain, "/urlpermissionapp", "alice"), force=True, ) @@ -1127,24 +1122,23 @@ def test_permission_app_propagation_on_ssowat(): assert not can_access_webpage(app_webroot + "/admin", logged_as="bob") -@pytest.mark.other_domains(number=1) def test_permission_legacy_app_propagation_on_ssowat(): app_install( os.path.join(get_test_apps_dir(), "legacy_app_ynh"), - args="domain=%s&domain_2=%s&path=%s&is_public=1" - % (maindomain, other_domains[0], "/legacy"), + args="domain=%s&domain_2=%s&path=%s&is_public=0" + % (maindomain, maindomain, "/legacy"), force=True, ) # App is configured as public by default using the legacy unprotected_uri mechanics # It should automatically be migrated during the install res = user_permission_list(full=True)["permissions"] - assert "visitors" in res["legacy_app.main"]["allowed"] + assert "visitors" not in res["legacy_app.main"]["allowed"] assert "all_users" in res["legacy_app.main"]["allowed"] app_webroot = "https://%s/legacy" % maindomain - assert can_access_webpage(app_webroot, logged_as=None) + assert not can_access_webpage(app_webroot, logged_as=None) assert can_access_webpage(app_webroot, logged_as="alice") # Try to update the permission and check that permissions are still consistent diff --git a/src/tests/test_questions.py b/src/tests/test_questions.py index ae116801fa..af8bc4469f 100644 --- a/src/tests/test_questions.py +++ b/src/tests/test_questions.py @@ -11,22 +11,23 @@ from _pytest.mark.structures import ParameterSet - from moulinette import Moulinette from yunohost import app, domain, user from yunohost.utils.form import ( OPTIONS, + FORBIDDEN_PASSWORD_CHARS, + READONLY_TYPES, ask_questions_and_parse_answers, BaseChoicesOption, BaseInputOption, BaseReadonlyOption, - PasswordOption, DomainOption, WebPathOption, BooleanOption, FileOption, evaluate_simple_js_expression, ) +from yunohost.utils import form from yunohost.utils.error import YunohostError, YunohostValidationError @@ -95,6 +96,12 @@ def patch_with_tty(): yield +@pytest.fixture +def patch_cli_retries(): + with patch.object(form, "MAX_RETRIES", 0): + yield + + # ╭───────────────────────────────────────────────────────╮ # │ ╭─╴╭─╴┌─╴╭╮╷╭─┐┌─╮╶┬╴╭─╮╭─╴ │ # │ ╰─╮│ ├─╴│││├─┤├┬╯ │ │ │╰─╮ │ @@ -382,8 +389,8 @@ def _fill_or_prompt_one_option(raw_option, intake): options = {id_: raw_option} answers = {id_: intake} if intake is not None else {} - option = ask_questions_and_parse_answers(options, answers)[0] - return (option, option.value if isinstance(option, BaseInputOption) else None) + options, form = ask_questions_and_parse_answers(options, answers) + return (options[0], form[id_] if isinstance(options[0], BaseInputOption) else None) def _test_value_is_expected_output(value, expected_output): @@ -410,6 +417,7 @@ def _test_intake_may_fail(raw_option, intake, expected_output): _test_intake(raw_option, intake, expected_output) +@pytest.mark.usefixtures("patch_cli_retries") # To avoid chain error logging class BaseTest: raw_option: dict[str, Any] = {} prefill: dict[Literal["raw_option", "prefill", "intake"], Any] @@ -440,6 +448,13 @@ def get_raw_option(cls, raw_option={}, **kwargs): @classmethod def _test_basic_attrs(self): raw_option = self.get_raw_option(optional=True) + + if raw_option["type"] in READONLY_TYPES: + del raw_option["optional"] + + if raw_option["type"] == "select": + raw_option["choices"] = ["one"] + id_ = raw_option["id"] option, value = _fill_or_prompt_one_option(raw_option, None) @@ -448,7 +463,7 @@ def _test_basic_attrs(self): assert isinstance(option, OPTIONS[raw_option["type"]]) assert option.type == raw_option["type"] assert option.id == id_ - assert option.ask == {"en": id_} + assert option.ask == id_ assert option.readonly is (True if is_special_readonly_option else False) assert option.visible is True # assert option.bind is None @@ -485,6 +500,7 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): base_raw_option = prefill_data["raw_option"] prefill = prefill_data["prefill"] + # FIXME could patch prompt with prefill if we switch to "do not apply default if value is None|''" with patch_prompt("") as prompt: raw_option = self.get_raw_option( raw_option=base_raw_option, @@ -493,7 +509,7 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): ) option, value = _fill_or_prompt_one_option(raw_option, None) - expected_message = option.ask["en"] + expected_message = option.ask choices = [] if isinstance(option, BaseChoicesOption): @@ -510,7 +526,7 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): prefill=prefill, is_multiline=option.type == "text", autocomplete=choices, - help=option.help["en"], + help=option.help, ) def test_scenarios(self, intake, expected_output, raw_option, data): @@ -555,10 +571,10 @@ def test_scenarios(self, intake, expected_output, raw_option, data): ask_questions_and_parse_answers({_id: raw_option}, answers) else: with patch.object(sys, "stdout", new_callable=StringIO) as stdout: - options = ask_questions_and_parse_answers( + options, form = ask_questions_and_parse_answers( {_id: raw_option}, answers ) - assert stdout.getvalue() == f"{options[0].ask['en']}\n" + assert stdout.getvalue() == f"{options[0].ask}\n" # ╭───────────────────────────────────────────────────────╮ @@ -587,9 +603,7 @@ class TestAlert(TestDisplayText): (None, None, {"ask": "Some text\na new line"}), (None, None, {"ask": {"en": "Some text\na new line", "fr": "Un peu de texte\nune nouvelle ligne"}}), *[(None, None, {"ask": "question", "style": style}) for style in ("success", "info", "warning", "danger")], - *xpass(scenarios=[ - (None, None, {"ask": "question", "style": "nimp"}), - ], reason="Should fail, wrong style"), + (None, YunohostError, {"ask": "question", "style": "nimp"}), ] # fmt: on @@ -608,10 +622,10 @@ def test_scenarios(self, intake, expected_output, raw_option, data): ) else: with patch.object(sys, "stdout", new_callable=StringIO) as stdout: - options = ask_questions_and_parse_answers( + options, form = ask_questions_and_parse_answers( {"display_text_id": raw_option}, answers ) - ask = options[0].ask["en"] + ask = options[0].ask if style in colors: color = colors[style] title = style.title() + (":" if style != "success" else "!") @@ -647,11 +661,15 @@ class TestString(BaseTest): scenarios = [ *nones(None, "", output=""), # basic typed values - *unchanged(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), # FIXME should output as str? + (False, "False"), + (True, "True"), + (0, "0"), + (1, "1"), + (-1, "-1"), + (1337, "1337"), + (13.37, "13.37"), + *all_fails([], ["one"], {}), *unchanged("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", raw_option={"optional": True}), - *xpass(scenarios=[ - ([], []), - ], reason="Should fail"), # test strip ("value", "value"), ("value\n", "value"), @@ -664,7 +682,7 @@ class TestString(BaseTest): (" ##value \n \tvalue\n ", "##value \n \tvalue"), ], reason=r"should fail or without `\n`?"), # readonly - ("overwrite", "expected value", {"readonly": True, "current_value": "expected value"}), + ("overwrite", "expected value", {"readonly": True, "default": "expected value"}), # FIXME do we want to fail instead? ] # fmt: on @@ -684,13 +702,19 @@ class TestText(BaseTest): scenarios = [ *nones(None, "", output=""), # basic typed values - *unchanged(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), # FIXME should fail or output as str? + (False, "False"), + (True, "True"), + (0, "0"), + (1, "1"), + (-1, "-1"), + (1337, "1337"), + (13.37, "13.37"), + *all_fails([], ["one"], {}), *unchanged("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", raw_option={"optional": True}), - *xpass(scenarios=[ - ([], []) - ], reason="Should fail"), ("value", "value"), ("value\n value", "value\n value"), + ("value", FAIL, {"pattern": {"regexp": r'^[A-F]\d\d$', "error": "Provide a room like F12 : one uppercase and 2 numbers"}}), + ("F12", "F12", {"pattern": {"regexp": r'^[A-F]\d\d$', "error": "Provide a room like F12 : one uppercase and 2 numbers"}}), # test no strip *xpass(scenarios=[ ("value\n", "value"), @@ -701,7 +725,7 @@ class TestText(BaseTest): (r" ##value \n \tvalue\n ", r"##value \n \tvalue\n"), ], reason="Should not be stripped"), # readonly - ("overwrite", "expected value", {"readonly": True, "current_value": "expected value"}), + ("overwrite", "expected value", {"readonly": True, "default": "expected value"}), ] # fmt: on @@ -719,11 +743,11 @@ class TestPassword(BaseTest): } # fmt: off scenarios = [ - *all_fails(False, True, 0, 1, -1, 1337, 13.37, raw_option={"optional": True}, error=TypeError), # FIXME those fails with TypeError + *all_fails(False, True, 0, 1, -1, 1337, 13.37, raw_option={"optional": True}), *all_fails([], ["one"], {}, raw_option={"optional": True}, error=AttributeError), # FIXME those fails with AttributeError *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), - ("s3cr3t!!", FAIL, {"default": "SUPAs3cr3t!!"}), # default is forbidden + ("s3cr3t!!", YunohostError, {"default": "SUPAs3cr3t!!"}), # default is forbidden *xpass(scenarios=[ ("s3cr3t!!", "s3cr3t!!", {"example": "SUPAs3cr3t!!"}), # example is forbidden ], reason="Should fail; example is forbidden"), @@ -733,9 +757,9 @@ class TestPassword(BaseTest): ], reason="Should output exactly the same"), ("s3cr3t!!", "s3cr3t!!"), ("secret", FAIL), - *[("supersecret" + char, FAIL) for char in PasswordOption.forbidden_chars], # FIXME maybe add ` \n` to the list? + *[("supersecret" + char, FAIL) for char in FORBIDDEN_PASSWORD_CHARS], # FIXME maybe add ` \n` to the list? # readonly - ("s3cr3t!!", YunohostError, {"readonly": True, "current_value": "isforbidden"}), # readonly is forbidden + ("s3cr3t!!", YunohostError, {"readonly": True}), # readonly is forbidden ] # fmt: on @@ -748,35 +772,31 @@ class TestPassword(BaseTest): class TestColor(BaseTest): raw_option = {"type": "color", "id": "color_id"} prefill = { - "raw_option": {"default": "#ff0000"}, - "prefill": "#ff0000", - # "intake": "#ff00ff", + "raw_option": {"default": "red"}, + "prefill": "red", } # fmt: off scenarios = [ *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), - *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "0", "1", "-1", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), # custom valid - ("#000000", "#000000"), + (" #fe1 ", "#fe1"), + ("#000000", "#000"), ("#000", "#000"), - ("#fe100", "#fe100"), - (" #fe100 ", "#fe100"), - ("#ABCDEF", "#ABCDEF"), + ("#ABCDEF", "#abcdef"), + ('1337', "#1337"), # rgba=(17, 51, 51, 0.47) + ("000000", "#000"), + ("#feaf", "#fea"), # `#feaf` is `#fea` with alpha at `f|100%` -> equivalent to `#fea` + # named + ("red", "#f00"), + ("yellow", "#ff0"), # custom fail - *xpass(scenarios=[ - ("#feaf", "#feaf"), - ], reason="Should fail; not a legal color value"), - ("000000", FAIL), ("#12", FAIL), ("#gggggg", FAIL), ("#01010101af", FAIL), - *xfail(scenarios=[ - ("red", "#ff0000"), - ("yellow", "#ffff00"), - ], reason="Should work with pydantic"), # readonly - ("#ffff00", "#fe100", {"readonly": True, "current_value": "#fe100"}), + ("#ffff00", "#000", {"readonly": True, "default": "#000"}), ] # fmt: on @@ -800,10 +820,8 @@ class TestNumber(BaseTest): *nones(None, "", output=None), *unchanged(0, 1, -1, 1337), - *xpass(scenarios=[(False, False)], reason="should fail or output as `0`"), - *xpass(scenarios=[(True, True)], reason="should fail or output as `1`"), - *all_as("0", 0, output=0), - *all_as("1", 1, output=1), + *all_as(False, "0", 0, output=0), # FIXME should `False` fail instead? + *all_as(True, "1", 1, output=1), # FIXME should `True` fail instead? *all_as("1337", 1337, output=1337), *xfail(scenarios=[ ("-1", -1) @@ -818,7 +836,7 @@ class TestNumber(BaseTest): (-10, -10, {"default": 10}), (-10, -10, {"default": 10, "optional": True}), # readonly - (1337, 10000, {"readonly": True, "current_value": 10000}), + (1337, 10000, {"readonly": True, "default": "10000"}), ] # fmt: on # FIXME should `step` be some kind of "multiple of"? @@ -843,14 +861,20 @@ class TestBoolean(BaseTest): *all_fails("none", "None"), # FIXME should output as `0` (default) like other none values when required? *all_as(None, "", output=0, raw_option={"optional": True}), # FIXME should output as `None`? *all_as("none", "None", output=None, raw_option={"optional": True}), - # FIXME even if default is explicity `None|""`, it ends up with class_default `0` - *all_as(None, "", output=0, raw_option={"default": None}), # FIXME this should fail, default is `None` - *all_as(None, "", output=0, raw_option={"optional": True, "default": None}), # FIXME even if default is explicity None, it ends up with class_default - *all_as(None, "", output=0, raw_option={"default": ""}), # FIXME this should fail, default is `""` - *all_as(None, "", output=0, raw_option={"optional": True, "default": ""}), # FIXME even if default is explicity None, it ends up with class_default - # With "none" behavior is ok - *all_fails(None, "", raw_option={"default": "none"}), - *all_as(None, "", output=None, raw_option={"optional": True, "default": "none"}), + { + "raw_options": [ + {"default": None}, + {"default": ""}, + {"default": "none"}, + {"default": "None"} + ], + "scenarios": [ + # All none values fails if default is overriden + *all_fails(None, "", "none", "None"), + # All none values ends up as None if default is overriden + *all_as(None, "", "none", "None", output=None, raw_option={"optional": True}), + ] + }, # Unhandled types should fail *all_fails(1337, "1337", "string", [], "[]", ",", "one,two"), *all_fails(1337, "1337", "string", [], "[]", ",", "one,two", {"optional": True}), @@ -883,7 +907,7 @@ class TestBoolean(BaseTest): "scenarios": all_fails("", "y", "n", error=AssertionError), }, # readonly - (1, 0, {"readonly": True, "current_value": 0}), + (1, 0, {"readonly": True, "default": 0}), ] @@ -900,8 +924,12 @@ class TestDate(BaseTest): } # fmt: off scenarios = [ - *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), - *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + # Those passes since False|True are parsed as 0|1 then int|float are considered a timestamp in seconds which ends up as default Unix date + *all_as(False, True, 0, 1, 1337, 13.37, "0", "1", "1337", "13.37", output="1970-01-01"), + # Those are negative one second timestamp ending up as Unix date - 1 sec (so day change) + *all_as(-1, "-1", output="1969-12-31"), + *all_fails([], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), # custom valid ("2070-12-31", "2070-12-31"), @@ -910,18 +938,16 @@ class TestDate(BaseTest): ("2025-06-15T13:45:30", "2025-06-15"), ("2025-06-15 13:45:30", "2025-06-15") ], reason="iso date repr should be valid and extra data striped"), - *xfail(scenarios=[ - (1749938400, "2025-06-15"), - (1749938400.0, "2025-06-15"), - ("1749938400", "2025-06-15"), - ("1749938400.0", "2025-06-15"), - ], reason="timestamp could be an accepted value"), + (1749938400, "2025-06-14"), + (1749938400.0, "2025-06-14"), + ("1749938400", "2025-06-14"), + ("1749938400.0", "2025-06-14"), # custom invalid ("29-12-2070", FAIL), ("12-01-10", FAIL), ("2022-02-29", FAIL), # readonly - ("2070-12-31", "2024-02-29", {"readonly": True, "current_value": "2024-02-29"}), + ("2070-12-31", "2024-02-29", {"readonly": True, "default": "2024-02-29"}), ] # fmt: on @@ -939,22 +965,26 @@ class TestTime(BaseTest): } # fmt: off scenarios = [ - *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), - *all_fails("none", "_none", "False", "True", "0", "1", "-1", "1337", "13.37", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), + # Those passes since False|True are parsed as 0|1 then int|float are considered a timestamp in seconds but we don't take seconds into account so -> 00:00 + *all_as(False, True, 0, 1, 13.37, "0", "1", "13.37", output="00:00"), + # 1337 seconds == 22 minutes + *all_as(1337, "1337", output="00:22"), + # Negative timestamp fails + *all_fails(-1, "-1", error=OverflowError), # FIXME should handle that as a validation error + # *all_fails(False, True, 0, 1, -1, 1337, 13.37, [], ["one"], {}, raw_option={"optional": True}), + *all_fails("none", "_none", "False", "True", "[]", ",", "['one']", "one,two", r"{}", "value", "value\n", raw_option={"optional": True}), *nones(None, "", output=""), # custom valid *unchanged("00:00", "08:00", "12:19", "20:59", "23:59"), - ("3:00", "3:00"), # FIXME should fail or output as `"03:00"`? - *xfail(scenarios=[ - ("22:35:05", "22:35"), - ("22:35:03.514", "22:35"), - ], reason="time as iso format could be valid"), + ("3:00", "03:00"), + ("23:1", "23:01"), + ("22:35:05", "22:35"), + ("22:35:03.514", "22:35"), # custom invalid ("24:00", FAIL), - ("23:1", FAIL), ("23:005", FAIL), # readonly - ("00:00", "08:00", {"readonly": True, "current_value": "08:00"}), + ("00:00", "08:00", {"readonly": True, "default": "08:00"}), ] # fmt: on @@ -977,72 +1007,75 @@ class TestEmail(BaseTest): *nones(None, "", output=""), ("\n Abc@example.tld ", "Abc@example.tld"), + *xfail(scenarios=[("admin@ynh.local", "admin@ynh.local")], reason="Should this pass?"), # readonly - ("Abc@example.tld", "admin@ynh.local", {"readonly": True, "current_value": "admin@ynh.local"}), + ("Abc@example.tld", "admin@ynh.org", {"readonly": True, "default": "admin@ynh.org"}), # Next examples are from https://github.com/JoshData/python-email-validator/blob/main/tests/test_syntax.py # valid email values - ("Abc@example.tld", "Abc@example.tld"), - ("Abc.123@test-example.com", "Abc.123@test-example.com"), - ("user+mailbox/department=shipping@example.tld", "user+mailbox/department=shipping@example.tld"), - ("伊昭傑@郵件.商務", "伊昭傑@郵件.商務"), - ("राम@मोहन.ईन्फो", "राम@मोहन.ईन्फो"), - ("юзер@екзампл.ком", "юзер@екзампл.ком"), - ("θσερ@εχαμπλε.ψομ", "θσερ@εχαμπλε.ψομ"), - ("葉士豪@臺網中心.tw", "葉士豪@臺網中心.tw"), - ("jeff@臺網中心.tw", "jeff@臺網中心.tw"), - ("葉士豪@臺網中心.台灣", "葉士豪@臺網中心.台灣"), - ("jeff葉@臺網中心.tw", "jeff葉@臺網中心.tw"), - ("ñoñó@example.tld", "ñoñó@example.tld"), - ("甲斐黒川日本@example.tld", "甲斐黒川日本@example.tld"), - ("чебурашкаящик-с-апельсинами.рф@example.tld", "чебурашкаящик-с-апельсинами.рф@example.tld"), - ("उदाहरण.परीक्ष@domain.with.idn.tld", "उदाहरण.परीक्ष@domain.with.idn.tld"), - ("ιωάννης@εεττ.gr", "ιωάννης@εεττ.gr"), + *unchanged( + "Abc@example.tld", + "Abc.123@test-example.com", + "user+mailbox/department=shipping@example.tld", + "伊昭傑@郵件.商務", + "राम@मोहन.ईन्फो", + "юзер@екзампл.ком", + "θσερ@εχαμπλε.ψομ", + "葉士豪@臺網中心.tw", + "jeff@臺網中心.tw", + "葉士豪@臺網中心.台灣", + "jeff葉@臺網中心.tw", + "ñoñó@example.tld", + "甲斐黒川日本@example.tld", + "чебурашкаящик-с-апельсинами.рф@example.tld", + "उदाहरण.परीक्ष@domain.with.idn.tld", + "ιωάννης@εεττ.gr", + ), # invalid email (Hiding because our current regex is very permissive) - # ("my@localhost", FAIL), - # ("my@.leadingdot.com", FAIL), - # ("my@．leadingfwdot.com", FAIL), - # ("my@twodots..com", FAIL), - # ("my@twofwdots．．.com", FAIL), - # ("my@trailingdot.com.", FAIL), - # ("my@trailingfwdot.com．", FAIL), - # ("me@-leadingdash", FAIL), - # ("me@－leadingdashfw", FAIL), - # ("me@trailingdash-", FAIL), - # ("me@trailingdashfw－", FAIL), - # ("my@baddash.-.com", FAIL), - # ("my@baddash.-a.com", FAIL), - # ("my@baddash.b-.com", FAIL), - # ("my@baddashfw.－.com", FAIL), - # ("my@baddashfw.－a.com", FAIL), - # ("my@baddashfw.b－.com", FAIL), - # ("my@example.com\n", FAIL), - # ("my@example\n.com", FAIL), - # ("me@x!", FAIL), - # ("me@x ", FAIL), - # (".leadingdot@domain.com", FAIL), - # ("twodots..here@domain.com", FAIL), - # ("trailingdot.@domain.email", FAIL), - # ("me@⒈wouldbeinvalid.com", FAIL), - ("@example.com", FAIL), - # ("\nmy@example.com", FAIL), - ("m\ny@example.com", FAIL), - ("my\n@example.com", FAIL), - # ("11111111112222222222333333333344444444445555555555666666666677777@example.com", FAIL), - # ("111111111122222222223333333333444444444455555555556666666666777777@example.com", FAIL), - # ("me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444444444455555555556.com", FAIL), - # ("me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", FAIL), - # ("me@中1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", FAIL), - # ("my.long.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333333344444.info", FAIL), - # ("my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333.info", FAIL), - # ("my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", FAIL), - # ("my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444.info", FAIL), - # ("my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", FAIL), - # ("me@bad-tld-1", FAIL), - # ("me@bad.tld-2", FAIL), - # ("me@xn--0.tld", FAIL), - # ("me@yy--0.tld", FAIL), - # ("me@yy－－0.tld", FAIL), + *all_fails( + "my@localhost", + "my@.leadingdot.com", + "my@．leadingfwdot.com", + "my@twodots..com", + "my@twofwdots．．.com", + "my@trailingdot.com.", + "my@trailingfwdot.com．", + "me@-leadingdash", + "me@－leadingdashfw", + "me@trailingdash-", + "me@trailingdashfw－", + "my@baddash.-.com", + "my@baddash.-a.com", + "my@baddash.b-.com", + "my@baddashfw.－.com", + "my@baddashfw.－a.com", + "my@baddashfw.b－.com", + "my@example\n.com", + "me@x!", + "me@x ", + ".leadingdot@domain.com", + "twodots..here@domain.com", + "trailingdot.@domain.email", + "me@⒈wouldbeinvalid.com", + "@example.com", + "m\ny@example.com", + "my\n@example.com", + "11111111112222222222333333333344444444445555555555666666666677777@example.com", + "111111111122222222223333333333444444444455555555556666666666777777@example.com", + "me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444444444455555555556.com", + "me@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", + "me@中1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555566.com", + "my.long.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333333344444.info", + "my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.11111111112222222222333333.info", + "my.long.address@λ111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", + "my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.111111111122222222223333333333444.info", + "my.λong.address@1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444444444555555555.6666666666777777777788888888889999999999000000000.1111111111222222222233333333334444.info", + "me@bad-tld-1", + "me@bad.tld-2", + "me@xn--0.tld", + "me@yy--0.tld", + "me@yy－－0.tld", + ) ] # fmt: on @@ -1091,7 +1124,7 @@ class TestWebPath(BaseTest): ("https://example.com/folder", "/https://example.com/folder") ], reason="Should fail or scheme+domain removed"), # readonly - ("/overwrite", "/value", {"readonly": True, "current_value": "/value"}), + ("/overwrite", "/value", {"readonly": True, "default": "/value"}), # FIXME should path have forbidden_chars? ] # fmt: on @@ -1115,21 +1148,17 @@ class TestUrl(BaseTest): *nones(None, "", output=""), ("http://some.org/folder/file.txt", "http://some.org/folder/file.txt"), + (' https://www.example.com \n', 'https://www.example.com'), # readonly - ("https://overwrite.org", "https://example.org", {"readonly": True, "current_value": "https://example.org"}), + ("https://overwrite.org", "https://example.org", {"readonly": True, "default": "https://example.org"}), # rest is taken from https://github.com/pydantic/pydantic/blob/main/tests/test_networks.py # valid *unchanged( # Those are valid but not sure how they will output with pydantic 'http://example.org', - 'http://test', - 'http://localhost', 'https://example.org/whatever/next/', 'https://example.org', - 'http://localhost', - 'http://localhost/', - 'http://localhost:8000', - 'http://localhost:8000/', + 'https://foo_bar.example.com/', 'http://example.co.jp', 'http://www.example.com/a%C2%B1b', @@ -1153,29 +1182,31 @@ class TestUrl(BaseTest): 'http://twitter.com/@handle/', 'http://11.11.11.11.example.com/action', 'http://abc.11.11.11.11.example.com/action', - 'http://example#', - 'http://example/#', - 'http://example/#fragment', - 'http://example/?#', 'http://example.org/path#', 'http://example.org/path#fragment', 'http://example.org/path?query#', 'http://example.org/path?query#fragment', + 'https://foo_bar.example.com/', + 'https://exam_ple.com/', + 'HTTP://EXAMPLE.ORG', + 'https://example.org', + 'https://example.org?a=1&b=2', + 'https://example.org#a=3;b=3', + 'https://example.xn--p1ai', + 'https://example.xn--vermgensberatung-pwb', + 'https://example.xn--zfr164b', ), - # Pydantic default parsing add a final `/` - ('https://foo_bar.example.com/', 'https://foo_bar.example.com/'), - ('https://exam_ple.com/', 'https://exam_ple.com/'), *xfail(scenarios=[ - (' https://www.example.com \n', 'https://www.example.com/'), - ('HTTP://EXAMPLE.ORG', 'http://example.org/'), - ('https://example.org', 'https://example.org/'), - ('https://example.org?a=1&b=2', 'https://example.org/?a=1&b=2'), - ('https://example.org#a=3;b=3', 'https://example.org/#a=3;b=3'), - ('https://example.xn--p1ai', 'https://example.xn--p1ai/'), - ('https://example.xn--vermgensberatung-pwb', 'https://example.xn--vermgensberatung-pwb/'), - ('https://example.xn--zfr164b', 'https://example.xn--zfr164b/'), - ], reason="pydantic default behavior would append a final `/`"), - + ('http://test', 'http://test'), + ('http://localhost', 'http://localhost'), + ('http://localhost/', 'http://localhost/'), + ('http://localhost:8000', 'http://localhost:8000'), + ('http://localhost:8000/', 'http://localhost:8000/'), + ('http://example#', 'http://example#'), + ('http://example/#', 'http://example/#'), + ('http://example/#fragment', 'http://example/#fragment'), + ('http://example/?#', 'http://example/?#'), + ], reason="Should this be valid?"), # invalid *all_fails( 'ftp://example.com/', @@ -1186,15 +1217,13 @@ class TestUrl(BaseTest): "/", "+http://example.com/", "ht*tp://example.com/", + "http:///", + "http://??", + "https://example.org more", + "http://2001:db8::ff00:42:8329", + "http://[192.168.1.1]:8329", + "http://example.com:99999", ), - *xpass(scenarios=[ - ("http:///", "http:///"), - ("http://??", "http://??"), - ("https://example.org more", "https://example.org more"), - ("http://2001:db8::ff00:42:8329", "http://2001:db8::ff00:42:8329"), - ("http://[192.168.1.1]:8329", "http://[192.168.1.1]:8329"), - ("http://example.com:99999", "http://example.com:99999"), - ], reason="Should fail"), ] # fmt: on @@ -1275,7 +1304,7 @@ class TestFile(BaseTest): def test_basic_attrs(self): raw_option, option, value = self._test_basic_attrs() - accept = raw_option.get("accept", "") # accept default + accept = raw_option.get("accept", None) # accept default assert option.accept == accept def test_options_prompted_with_ask_help(self): @@ -1365,7 +1394,6 @@ class TestSelect(BaseTest): # [-1, 0, 1] "raw_options": [ {"choices": [-1, 0, 1, 10]}, - {"choices": {-1: "verbose -one", 0: "verbose zero", 1: "verbose one", 10: "verbose ten"}}, ], "scenarios": [ *nones(None, "", output=""), @@ -1379,6 +1407,18 @@ class TestSelect(BaseTest): *all_fails("100", 100), ] }, + { + "raw_options": [ + {"choices": {-1: "verbose -one", 0: "verbose zero", 1: "verbose one", 10: "verbose ten"}}, + {"choices": {"-1": "verbose -one", "0": "verbose zero", "1": "verbose one", "10": "verbose ten"}}, + ], + "scenarios": [ + *nones(None, "", output=""), + *all_fails(-1, 0, 1, 10), # Should pass? converted to str? + *unchanged("-1", "0", "1", "10"), + *all_fails("100", 100), + ] + }, # [True, False, None] *unchanged(True, False, raw_option={"choices": [True, False, None]}), # FIXME we should probably forbid None in choices (None, FAIL, {"choices": [True, False, None]}), @@ -1406,7 +1446,7 @@ class TestSelect(BaseTest): ] }, # readonly - ("one", "two", {"readonly": True, "choices": ["one", "two"], "current_value": "two"}), + ("one", "two", {"readonly": True, "choices": ["one", "two"], "default": "two"}), ] # fmt: on @@ -1416,6 +1456,7 @@ class TestSelect(BaseTest): # ╰───────────────────────────────────────────────────────╯ +# [], ["one"], {} class TestTags(BaseTest): raw_option = {"type": "tags", "id": "tags_id"} prefill = { @@ -1424,12 +1465,7 @@ class TestTags(BaseTest): } # fmt: off scenarios = [ - *nones(None, [], "", output=""), - # FIXME `","` could be considered a none value which kinda already is since it fail when required - (",", FAIL), - *xpass(scenarios=[ - (",", ",", {"optional": True}) - ], reason="Should output as `''`? ie: None"), + *nones(None, [], "", ",", output=""), { "raw_options": [ {}, @@ -1449,12 +1485,12 @@ class TestTags(BaseTest): # basic types (not in a list) should fail *all_fails(True, False, -1, 0, 1, 1337, 13.37, {}), # Mixed choices should fail - ([False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}], FAIL, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), - ("False,True,-1,0,1,1337,13.37,[],['one'],{}", FAIL, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), - *all_fails(*([t] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), - *all_fails(*([str(t)] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + ([False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}], YunohostError, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + ("False,True,-1,0,1,1337,13.37,[],['one'],{}", YunohostError, {"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}), + *all_fails(*([t] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}, error=YunohostError), + *all_fails(*([str(t)] for t in [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]), raw_option={"choices": [False, True, -1, 0, 1, 1337, 13.37, [], ["one"], {}]}, error=YunohostError), # readonly - ("one", "one,two", {"readonly": True, "choices": ["one", "two"], "current_value": "one,two"}), + ("one", "one,two", {"readonly": True, "choices": ["one", "two"], "default": "one,two"}), ] # fmt: on @@ -1570,8 +1606,7 @@ class TestApp(BaseTest): ], "scenarios": [ # FIXME there are currently 3 different nones (`None`, `""` and `_none`), choose one? - *nones(None, output=None), # FIXME Should return chosen none? - *nones("", output=""), # FIXME Should return chosen none? + *nones(None, "", output=""), # FIXME Should return chosen none? *xpass(scenarios=[ ("_none", "_none"), ("_none", "_none", {"default": "_none"}), @@ -1594,7 +1629,7 @@ class TestApp(BaseTest): (installed_webapp["id"], installed_webapp["id"], {"filter": "is_webapp"}), (installed_webapp["id"], FAIL, {"filter": "is_webapp == false"}), (installed_webapp["id"], FAIL, {"filter": "id != 'my_webapp'"}), - (None, None, {"filter": "id == 'fake_app'", "optional": True}), + (None, "", {"filter": "id == 'fake_app'", "optional": True}), ] }, { @@ -1650,47 +1685,29 @@ def test_scenarios(self, intake, expected_output, raw_option, data): admin_username = "admin_user" admin_user = { - "ssh_allowed": False, "username": admin_username, - "mailbox-quota": "0", "mail": "a@ynh.local", - "mail-aliases": [f"root@{main_domain}"], # Faking "admin" "fullname": "john doe", - "group": [], + "groups": ["admins"], } regular_username = "normal_user" regular_user = { - "ssh_allowed": False, "username": regular_username, - "mailbox-quota": "0", "mail": "z@ynh.local", "fullname": "john doe", - "group": [], + "groups": [], } @contextmanager -def patch_users( - *, - users, - admin_username, - main_domain, -): +def patch_users(*, users): """ Data mocking for UserOption: - yunohost.user.user_list - yunohost.user.user_info - yunohost.domain._get_maindomain """ - admin_info = next( - (user for user in users.values() if user["username"] == admin_username), - {"mail-aliases": []}, - ) - with patch.object(user, "user_list", return_value={"users": users}), patch.object( - user, - "user_info", - return_value=admin_info, # Faking admin user - ), patch.object(domain, "_get_maindomain", return_value=main_domain): + with patch.object(user, "user_list", return_value={"users": users}): yield @@ -1701,8 +1718,8 @@ class TestUser(BaseTest): # No tests for empty users since it should not happens { "data": [ - {"users": {admin_username: admin_user}, "admin_username": admin_username, "main_domain": main_domain}, - {"users": {admin_username: admin_user, regular_username: regular_user}, "admin_username": admin_username, "main_domain": main_domain}, + {"users": {admin_username: admin_user}}, + {"users": {admin_username: admin_user, regular_username: regular_user}}, ], "scenarios": [ # FIXME User option is not really nullable, even if optional @@ -1713,7 +1730,7 @@ class TestUser(BaseTest): }, { "data": [ - {"users": {admin_username: admin_user, regular_username: regular_user}, "admin_username": admin_username, "main_domain": main_domain}, + {"users": {admin_username: admin_user, regular_username: regular_user}}, ], "scenarios": [ *xpass(scenarios=[ @@ -1728,18 +1745,12 @@ class TestUser(BaseTest): @pytest.mark.usefixtures("patch_no_tty") def test_basic_attrs(self): - with patch_users( - users={admin_username: admin_user}, - admin_username=admin_username, - main_domain=main_domain, - ): + with patch_users(users={admin_username: admin_user}): self._test_basic_attrs() def test_options_prompted_with_ask_help(self, prefill_data=None): with patch_users( - users={admin_username: admin_user, regular_username: regular_user}, - admin_username=admin_username, - main_domain=main_domain, + users={admin_username: admin_user, regular_username: regular_user} ): super().test_options_prompted_with_ask_help( prefill_data={"raw_option": {}, "prefill": admin_username} @@ -1761,8 +1772,8 @@ def test_scenarios(self, intake, expected_output, raw_option, data): # │ GROUP │ # ╰───────────────────────────────────────────────────────╯ -groups1 = ["all_users", "visitors", "admins"] -groups2 = ["all_users", "visitors", "admins", "custom_group"] +groups1 = {"all_users": {}, "visitors": {}, "admins": {}} +groups2 = {"all_users": {}, "visitors": {}, "admins": {}, "custom_group": {}} @contextmanager @@ -1800,9 +1811,7 @@ class TestGroup(BaseTest): "scenarios": [ ("custom_group", "custom_group"), *all_as("", None, output="visitors", raw_option={"default": "visitors"}), - *xpass(scenarios=[ - ("", "custom_group", {"default": "custom_group"}), - ], reason="Should throw 'default must be in (None, 'all_users', 'visitors', 'admins')"), + ("", YunohostError, {"default": "custom_group"}), # Not allowed to set a default which is not a default group # readonly ("admins", YunohostError, {"readonly": True}), # readonly is forbidden ] @@ -1821,13 +1830,6 @@ def test_options_prompted_with_ask_help(self, prefill_data=None): "prefill": "admins", } ) - # FIXME This should fail, not allowed to set a default which is not a default group - super().test_options_prompted_with_ask_help( - prefill_data={ - "raw_option": {"default": "custom_group"}, - "prefill": "custom_group", - } - ) def test_scenarios(self, intake, expected_output, raw_option, data): with patch_groups(**data): @@ -1845,8 +1847,6 @@ def patch_entities(): apps=[installed_webapp, installed_non_webapp] ), patch_users( users={admin_username: admin_user, regular_username: regular_user}, - admin_username=admin_username, - main_domain=main_domain, ), patch_groups( groups=groups2 ): @@ -1884,12 +1884,12 @@ def test_options_query_string(): "string_id": "string", "text_id": "text\ntext", "password_id": "sUpRSCRT", - "color_id": "#ffff00", + "color_id": "#ff0", "number_id": 10, "boolean_id": 1, "date_id": "2030-03-06", "time_id": "20:55", - "email_id": "coucou@ynh.local", + "email_id": "coucou@ynh.org", "path_id": "/ynh-dev", "url_id": "https://yunohost.org", "file_id": file_content1, @@ -1912,7 +1912,7 @@ def patch_query_string(file_repr): "&boolean_id=y" "&date_id=2030-03-06" "&time_id=20:55" - "&email_id=coucou@ynh.local" + "&email_id=coucou@ynh.org" "&path_id=ynh-dev/" "&url_id=https://yunohost.org" f"&file_id={file_repr}" @@ -1929,9 +1929,7 @@ def patch_query_string(file_repr): "&fake_id=fake_value" ) - def _assert_correct_values(options, raw_options): - form = {option.id: option.value for option in options} - + def _assert_correct_values(options, form, raw_options): for k, v in results.items(): if k == "file_id": assert os.path.exists(form["file_id"]) and os.path.isfile( @@ -1947,24 +1945,24 @@ def _assert_correct_values(options, raw_options): with patch_interface("api"), patch_file_api(file_content1) as b64content: with patch_query_string(b64content.decode("utf-8")) as query_string: - options = ask_questions_and_parse_answers(raw_options, query_string) - _assert_correct_values(options, raw_options) + options, form = ask_questions_and_parse_answers(raw_options, query_string) + _assert_correct_values(options, form, raw_options) with patch_interface("cli"), patch_file_cli(file_content1) as filepath: with patch_query_string(filepath) as query_string: - options = ask_questions_and_parse_answers(raw_options, query_string) - _assert_correct_values(options, raw_options) + options, form = ask_questions_and_parse_answers(raw_options, query_string) + _assert_correct_values(options, form, raw_options) def test_question_string_default_type(): questions = {"some_string": {}} answers = {"some_string": "some_value"} - out = ask_questions_and_parse_answers(questions, answers)[0] - - assert out.id == "some_string" - assert out.type == "string" - assert out.value == "some_value" + options, form = ask_questions_and_parse_answers(questions, answers) + option = options[0] + assert option.id == "some_string" + assert option.type == "string" + assert form[option.id] == "some_value" def test_option_default_type_with_choices_is_select(): @@ -1976,10 +1974,10 @@ def test_option_default_type_with_choices_is_select(): } answers = {"some_choices": "a", "some_legacy": "a"} - options = ask_questions_and_parse_answers(questions, answers) + options, form = ask_questions_and_parse_answers(questions, answers) for option in options: assert option.type == "select" - assert option.value == "a" + assert form[option.id] == "a" @pytest.mark.skip # we should do something with this example diff --git a/src/tests/test_regenconf.py b/src/tests/test_regenconf.py index 8dda1a7f25..3966ef2911 100644 --- a/src/tests/test_regenconf.py +++ b/src/tests/test_regenconf.py @@ -87,7 +87,7 @@ def test_ssh_conf_unmanaged(): assert SSHD_CONFIG in _get_conf_hashes("ssh") -def test_ssh_conf_unmanaged_and_manually_modified(mocker): +def test_ssh_conf_unmanaged_and_manually_modified(): _force_clear_hashes([SSHD_CONFIG]) os.system("echo ' ' >> %s" % SSHD_CONFIG) @@ -98,7 +98,7 @@ def test_ssh_conf_unmanaged_and_manually_modified(mocker): assert SSHD_CONFIG in _get_conf_hashes("ssh") assert SSHD_CONFIG in manually_modified_files() - with message(mocker, "regenconf_need_to_explicitly_specify_ssh"): + with message("regenconf_need_to_explicitly_specify_ssh"): regen_conf(force=True) assert SSHD_CONFIG in _get_conf_hashes("ssh") diff --git a/src/tests/test_sso_and_portalapi.py b/src/tests/test_sso_and_portalapi.py new file mode 100644 index 0000000000..17c19119d2 --- /dev/null +++ b/src/tests/test_sso_and_portalapi.py @@ -0,0 +1,362 @@ +import base64 +import time +import requests +from pathlib import Path +import os + +from .conftest import message, raiseYunohostError, get_test_apps_dir + +from yunohost.domain import _get_maindomain, domain_add, domain_remove, domain_list +from yunohost.user import user_create, user_list, user_delete, user_update +from yunohost.authenticators.ldap_ynhuser import Authenticator, SESSION_FOLDER, short_hash +from yunohost.app import app_install, app_remove, app_setting, app_ssowatconf, app_change_url +from yunohost.permission import user_permission_list, user_permission_update + + +# Get main domain +maindomain = open("/etc/yunohost/current_host").read().strip() +subdomain = f"sub.{maindomain}" +secondarydomain = "secondary.test" +dummy_password = "test123Ynh" + + +def setup_function(function): + Authenticator.invalidate_all_sessions_for_user("alice") + assert number_of_active_session_for_user("alice") == 0 + Authenticator.invalidate_all_sessions_for_user("bob") + assert number_of_active_session_for_user("bob") == 0 + + user_permission_update( + "hellopy.main", add=["visitors", "all_users"], remove=["alice", "bob"] + ) + + app_setting("hellopy", "auth_header", delete=True) + app_setting("hellopy", "protect_against_basic_auth_spoofing", delete=True) + app_ssowatconf() + + +def teardown_function(function): + pass + + +def setup_module(module): + + assert os.system("systemctl is-active yunohost-portal-api >/dev/null") == 0 + + if "alice" not in user_list()["users"]: + user_create("alice", maindomain, dummy_password, fullname="Alice White", admin=True) + if "bob" not in user_list()["users"]: + user_create("bob", maindomain, dummy_password, fullname="Bob Marley") + + app_install( + os.path.join(get_test_apps_dir(), "hellopy_ynh"), + args=f"domain={maindomain}&init_main_permission=visitors", + force=True, + ) + + +def teardown_module(module): + if "alice" in user_list()["users"]: + user_delete("alice") + if "bob" in user_list()["users"]: + user_delete("bob") + + app_remove("hellopy") + + if subdomain in domain_list()["domains"]: + domain_remove(subdomain) + if secondarydomain in domain_list()["domains"]: + domain_remove(secondarydomain) + + +def login(session, logged_as, logged_on=None): + + if not logged_on: + logged_on = maindomain + + login_endpoint = f"https://{logged_on}/yunohost/portalapi/login" + r = session.post( + login_endpoint, + data={"credentials": f"{logged_as}:{dummy_password}"}, + headers={ + "X-Requested-With": "", + }, + verify=False, + ) + + return r + + +def logout(session): + logout_endpoint = f"https://{maindomain}/yunohost/portalapi/logout" + r = session.get( + logout_endpoint, + headers={ + "X-Requested-With": "", + }, + verify=False, + ) + return r + + +def number_of_active_session_for_user(user): + return len(list(Path(SESSION_FOLDER).glob(f"{short_hash(user)}*"))) + + +def request(webpath, logged_as=None, session=None, inject_auth=None, logged_on=None): + webpath = webpath.rstrip("/") + + headers = {} + if inject_auth: + b64loginpassword = base64.b64encode((inject_auth[0] + ":" + inject_auth[1]).encode()).decode() + headers["Authorization"] = f"Basic {b64loginpassword}" + + # Anonymous access + if session: + r = session.get(webpath, verify=False, allow_redirects=False, headers=headers) + elif not logged_as: + r = requests.get(webpath, verify=False, allow_redirects=False, headers=headers) + # Login as a user using dummy password + else: + with requests.Session() as session: + r = login(session, logged_as, logged_on) + # We should have some cookies related to authentication now + assert session.cookies + r = session.get(webpath, verify=False, allow_redirects=False, headers=headers) + + return r + + +def test_api_public_as_anonymous(): + + # FIXME : should list apps only if the domain option is enabled + + r = request(f"https://{maindomain}/yunohost/portalapi/public") + assert r.status_code == 200 and "apps" in r.json() + + +def test_api_me_as_anonymous(): + + r = request(f"https://{maindomain}/yunohost/portalapi/me") + assert r.status_code == 401 + + +def test_api_login_and_logout(): + + with requests.Session() as session: + r = login(session, "alice") + + assert "yunohost.portal" in session.cookies + assert r.status_code == 200 + + assert number_of_active_session_for_user("alice") == 1 + + r = logout(session) + + assert number_of_active_session_for_user("alice") == 0 + + +def test_api_login_nonexistinguser(): + + with requests.Session() as session: + r = login(session, "nonexistent") + + assert r.status_code == 401 + + +def test_api_public_and_me_logged_in(): + + r = request(f"https://{maindomain}/yunohost/portalapi/public", logged_as="alice") + assert r.status_code == 200 and "apps" in r.json() + r = request(f"https://{maindomain}/yunohost/portalapi/me", logged_as="alice") + assert r.status_code == 200 and r.json()["username"] == "alice" + + assert number_of_active_session_for_user("alice") == 2 + + +def test_api_session_expired(): + + with requests.Session() as session: + r = login(session, "alice") + + assert "yunohost.portal" in session.cookies + assert r.status_code == 200 + + r = request(f"https://{maindomain}/yunohost/portalapi/me", session=session) + assert r.status_code == 200 and r.json()["username"] == "alice" + + for file in Path(SESSION_FOLDER).glob(f"{short_hash('alice')}*"): + os.utime(str(file), (0, 0)) + + r = request(f"https://{maindomain}/yunohost/portalapi/me", session=session) + assert number_of_active_session_for_user("alice") == 0 + assert r.status_code == 401 + + +def test_public_routes_not_blocked_by_ssowat(): + + r = request(f"https://{maindomain}/yunohost/api/whatever") + # Getting code 405, Method not allowed, which means the API does answer, + # meaning it's not blocked by ssowat + # Or : on the CI, the yunohost-api is likely to be down (to save resources) + assert r.status_code in [405, 502] + + os.system("mkdir -p /var/www/.well-known/acme-challenge-public") + Path("/var/www/.well-known/acme-challenge-public/toto").touch() + r = request(f"http://{maindomain}/.well-known/acme-challenge/toto") + assert r.status_code == 200 + + r = request(f"http://{maindomain}/.well-known/acme-challenge/nonexistent") + assert r.status_code == 404 + + +def test_permission_propagation_on_ssowat(): + + res = user_permission_list(full=True)["permissions"] + assert "visitors" in res["hellopy.main"]["allowed"] + assert "all_users" in res["hellopy.main"]["allowed"] + + r = request(f"https://{maindomain}/") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + r = request(f"https://{maindomain}/", logged_as="alice") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + r = request(f"https://{maindomain}/", logged_as="bob") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + user_permission_update( + "hellopy.main", remove=["visitors", "all_users"], add="alice" + ) + + # Visitors now get redirected to portal + r = request(f"https://{maindomain}/") + assert r.status_code == 302 + assert r.headers['Location'].startswith(f"https://{maindomain}/yunohost/sso?r=") + + # Alice can still access the app fine + r = request(f"https://{maindomain}/", logged_as="alice") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + +def test_login_right_depending_on_app_access_and_mail(): + + r = request(f"https://{maindomain}/", logged_as="bob") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + user_permission_update( + "hellopy.main", remove=["visitors", "all_users"], add="alice" + ) + + # Bob can still login even though he has no access to any apps, because its mail address is on the maindomain + with requests.Session() as session: + r = login(session, "bob") + assert session.cookies + + if secondarydomain not in domain_list()["domains"]: + domain_add(secondarydomain) + + user_update("bob", mail=f"bob@{secondarydomain}") + + # Now bob shouldn't be able to login anymore (on the main domain) + with requests.Session() as session: + r = login(session, "bob") + assert not session.cookies + + user_permission_update( + "hellopy.main", add="bob" + ) + + # Bob should be allowed to login again (even though its mail is on secondarydomain) + r = request(f"https://{maindomain}/", logged_as="bob") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + + +def test_sso_basic_auth_header(): + + r = request(f"https://{maindomain}/show-auth") + assert r.status_code == 200 and r.content.decode().strip() == "User: None\nPwd: None" + + r = request(f"https://{maindomain}/show-auth", logged_as="alice") + assert r.status_code == 200 and r.content.decode().strip() == f"User: alice\nPwd: {dummy_password}" + + app_setting("hellopy", "auth_header", value="basic-without-password") + app_ssowatconf() + + r = request(f"https://{maindomain}/show-auth", logged_as="alice") + assert r.status_code == 200 and r.content.decode().strip() == f"User: alice\nPwd: -" + + +def test_sso_basic_auth_header_spoofing(): + + r = request(f"https://{maindomain}/show-auth") + assert r.status_code == 200 and r.content.decode().strip() == "User: None\nPwd: None" + + r = request(f"https://{maindomain}/show-auth", inject_auth=("foo", "bar")) + assert r.status_code == 200 and r.content.decode().strip() == "User: None\nPwd: None" + + app_setting("hellopy", "protect_against_basic_auth_spoofing", value="false") + app_ssowatconf() + + r = request(f"https://{maindomain}/show-auth", inject_auth=("foo", "bar")) + assert r.status_code == 200 and r.content.decode().strip() == "User: foo\nPwd: bar" + + +def test_sso_on_subdomain(): + + if subdomain not in domain_list()["domains"]: + domain_add(subdomain) + + app_change_url("hellopy", domain=subdomain, path="/") + + r = request(f"https://{subdomain}/") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + r = request(f"https://{subdomain}/", logged_as="alice") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + r = request(f"https://{subdomain}/show-auth", logged_as="alice") + assert r.status_code == 200 and r.content.decode().strip().startswith("User: alice") + + +def test_sso_on_secondary_domain(): + + if secondarydomain not in domain_list()["domains"]: + domain_add(secondarydomain) + + app_change_url("hellopy", domain=secondarydomain, path="/") + + r = request(f"https://{secondarydomain}/") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + r = request(f"https://{secondarydomain}/", logged_as="alice") + assert r.status_code == 200 and r.content.decode().strip() == "Hello world!" + + r = request(f"https://{secondarydomain}/show-auth", logged_as="alice") + # Getting 'User: None despite being logged on the main domain + assert r.status_code == 200 and r.content.decode().strip().startswith("User: None") + + r = request(f"https://{secondarydomain}/show-auth", logged_as="alice", logged_on=secondarydomain) + assert r.status_code == 200 and r.content.decode().strip().startswith("User: alice") + + + + + +# accès à l'api portal + # -> test des routes + # apps publique (seulement si activé ?) + # /me + # /update + + +# accès aux trucs précédent meme avec une app installée sur la racine ? +# ou une app par défaut ? + +# accès à un deuxième "domain principal" + +# accès à un app sur un sous-domaine +# pas loggué -> redirect vers sso sur domaine principal +# se logger sur API sur domain principal, puis utilisation du cookie sur le sous-domaine + diff --git a/src/tests/test_user-group.py b/src/tests/test_user-group.py index eececb8279..f347fc9bcc 100644 --- a/src/tests/test_user-group.py +++ b/src/tests/test_user-group.py @@ -91,8 +91,8 @@ def test_list_groups(): # -def test_create_user(mocker): - with message(mocker, "user_created"): +def test_create_user(): + with message("user_created"): user_create("albert", maindomain, "test123Ynh", fullname="Albert Good") group_res = user_group_list()["groups"] @@ -102,8 +102,8 @@ def test_create_user(mocker): assert "albert" in group_res["all_users"]["members"] -def test_del_user(mocker): - with message(mocker, "user_deleted"): +def test_del_user(): + with message("user_deleted"): user_delete("alice") group_res = user_group_list()["groups"] @@ -112,7 +112,7 @@ def test_del_user(mocker): assert "alice" not in group_res["all_users"]["members"] -def test_import_user(mocker): +def test_import_user(): import csv from io import StringIO @@ -157,7 +157,7 @@ def test_import_user(mocker): } ) csv_io.seek(0) - with message(mocker, "user_import_success"): + with message("user_import_success"): user_import(csv_io, update=True, delete=True) group_res = user_group_list()["groups"] @@ -171,7 +171,7 @@ def test_import_user(mocker): assert "alice" not in group_res["dev"]["members"] -def test_export_user(mocker): +def test_export_user(): result = user_export() should_be = ( "username;firstname;lastname;password;mail;mail-alias;mail-forward;mailbox-quota;groups\r\n" @@ -182,8 +182,8 @@ def test_export_user(mocker): assert result == should_be -def test_create_group(mocker): - with message(mocker, "group_created", group="adminsys"): +def test_create_group(): + with message("group_created", group="adminsys"): user_group_create("adminsys") group_res = user_group_list()["groups"] @@ -192,8 +192,8 @@ def test_create_group(mocker): assert group_res["adminsys"]["members"] == [] -def test_del_group(mocker): - with message(mocker, "group_deleted", group="dev"): +def test_del_group(): + with message("group_deleted", group="dev"): user_group_delete("dev") group_res = user_group_list()["groups"] @@ -262,46 +262,40 @@ def test_del_group_that_does_not_exist(mocker): # -def test_update_user(mocker): - with message(mocker, "user_updated"): - user_update("alice", firstname="NewName", lastname="NewLast") - - info = user_info("alice") - assert info["fullname"] == "NewName NewLast" - - with message(mocker, "user_updated"): +def test_update_user(): + with message("user_updated"): user_update("alice", fullname="New2Name New2Last") info = user_info("alice") assert info["fullname"] == "New2Name New2Last" -def test_update_group_add_user(mocker): - with message(mocker, "group_updated", group="dev"): +def test_update_group_add_user(): + with message("group_updated", group="dev"): user_group_update("dev", add=["bob"]) group_res = user_group_list()["groups"] assert set(group_res["dev"]["members"]) == {"alice", "bob"} -def test_update_group_add_user_already_in(mocker): - with message(mocker, "group_user_already_in_group", user="bob", group="apps"): +def test_update_group_add_user_already_in(): + with message("group_user_already_in_group", user="bob", group="apps"): user_group_update("apps", add=["bob"]) group_res = user_group_list()["groups"] assert group_res["apps"]["members"] == ["bob"] -def test_update_group_remove_user(mocker): - with message(mocker, "group_updated", group="apps"): +def test_update_group_remove_user(): + with message("group_updated", group="apps"): user_group_update("apps", remove=["bob"]) group_res = user_group_list()["groups"] assert group_res["apps"]["members"] == [] -def test_update_group_remove_user_not_already_in(mocker): - with message(mocker, "group_user_not_in_group", user="jack", group="apps"): +def test_update_group_remove_user_not_already_in(): + with message("group_user_not_in_group", user="jack", group="apps"): user_group_update("apps", remove=["jack"]) group_res = user_group_list()["groups"] @@ -315,7 +309,7 @@ def test_update_group_remove_user_not_already_in(mocker): def test_update_user_that_doesnt_exist(mocker): with raiseYunohostError(mocker, "user_unknown"): - user_update("doesnt_exist", firstname="NewName", lastname="NewLast") + user_update("doesnt_exist", fullname="Foo Bar") def test_update_group_that_doesnt_exist(mocker): diff --git a/src/tools.py b/src/tools.py index 07a3d67ede..506ec592a4 100644 --- a/src/tools.py +++ b/src/tools.py @@ -24,24 +24,12 @@ from importlib import import_module from packaging import version from typing import List +from logging import getLogger from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import call_async_output from moulinette.utils.filesystem import read_yaml, write_to_yaml, cp, mkdir, rm, chown -from yunohost.app import ( - app_upgrade, - app_list, - _list_upgradable_apps, -) -from yunohost.app_catalog import ( - _initialize_apps_catalog_system, - _update_apps_catalog, -) -from yunohost.domain import domain_add -from yunohost.firewall import firewall_upnp -from yunohost.regenconf import regen_conf from yunohost.utils.system import ( _dump_sources_list, _list_upgradable_apt_packages, @@ -55,7 +43,7 @@ MIGRATIONS_STATE_PATH = "/etc/yunohost/migrations.yaml" -logger = getActionLogger("yunohost.tools") +logger = getLogger("yunohost.tools") def tools_versions(): @@ -63,10 +51,10 @@ def tools_versions(): def tools_rootpw(new_password, check_strength=True): - from yunohost.user import _hash_user_password from yunohost.utils.password import ( assert_password_is_strong_enough, assert_password_is_compatible, + _hash_user_password, ) import spwd @@ -155,6 +143,7 @@ def tools_postinstall( ignore_dyndns=False, force_diskspace=False, overwrite_root_password=True, + i_have_read_terms_of_services=False ): from yunohost.service import _run_service_command from yunohost.dyndns import _dyndns_available, dyndns_unsubscribe @@ -163,8 +152,12 @@ def tools_postinstall( assert_password_is_strong_enough, assert_password_is_compatible, ) - from yunohost.domain import domain_main_domain + from yunohost.domain import domain_main_domain, domain_add from yunohost.user import user_create, ADMIN_ALIASES + from yunohost.app import _ask_confirmation + from yunohost.app_catalog import _update_apps_catalog + from yunohost.firewall import firewall_upnp + import psutil # Do some checks at first @@ -177,6 +170,11 @@ def tools_postinstall( raw_msg=True, ) + if Moulinette.interface.type == "cli" and os.isatty(1): + Moulinette.display(m18n.n("tos_postinstall_acknowledgement"), style="warning") + if not i_have_read_terms_of_services: + _ask_confirmation("confirm_tos_acknowledgement", kind="soft") + # Crash early if the username is already a system user, which is # a common confusion. We don't want to crash later and end up in an half-configured state. all_existing_usernames = {x.pw_name for x in pwd.getpwall()} @@ -242,6 +240,7 @@ def tools_postinstall( domain, dyndns_recovery_password=dyndns_recovery_password, ignore_dyndns=ignore_dyndns, + skip_tos=True, # skip_tos is here to prevent re-asking about the ToS when adding a dyndns service, because the ToS are already displayed right before in postinstall ) domain_main_domain(domain) @@ -254,10 +253,7 @@ def tools_postinstall( # Enable UPnP silently and reload firewall firewall_upnp("enable", no_refresh=True) - # Initialize the apps catalog system - _initialize_apps_catalog_system() - - # Try to update the apps catalog ... + # Try to fetch the apps catalog ... # we don't fail miserably if this fails, # because that could be for example an offline installation... try: @@ -274,7 +270,7 @@ def tools_postinstall( _run_service_command("enable", "yunohost-firewall") _run_service_command("start", "yunohost-firewall") - regen_conf(names=["ssh"], force=True) + tools_regen_conf(names=["ssh"], force=True) # Restore original ssh conf, as chosen by the # admin during the initial install @@ -289,7 +285,7 @@ def tools_postinstall( if os.path.exists(original_sshd_conf): os.rename(original_sshd_conf, "/etc/ssh/sshd_config") - regen_conf(force=True) + tools_regen_conf(force=True) logger.success(m18n.n("yunohost_configured")) @@ -299,17 +295,8 @@ def tools_postinstall( def tools_regen_conf( names=[], with_diff=False, force=False, dry_run=False, list_pending=False ): - # Make sure the settings are migrated before running the migration, - # which may otherwise fuck things up such as the ssh config ... - # We do this here because the regen-conf is called before the migration in debian/postinst - if os.path.exists("/etc/yunohost/settings.json") and not os.path.exists( - "/etc/yunohost/settings.yml" - ): - try: - tools_migrations_run(["0025_global_settings_to_configpanel"]) - except Exception as e: - logger.error(e) + from yunohost.regenconf import regen_conf return regen_conf(names, with_diff, force, dry_run, list_pending) @@ -317,6 +304,8 @@ def tools_update(target=None): """ Update apps & system package cache """ + from yunohost.app_catalog import _update_apps_catalog + from yunohost.app import _list_upgradable_apps if not target: target = "all" @@ -425,6 +414,8 @@ def tools_upgrade(operation_logger, target=None): system -- True to upgrade system """ + from yunohost.app import app_upgrade, app_list + if dpkg_is_broken(): raise YunohostValidationError("dpkg_is_broken") @@ -688,6 +679,15 @@ def get_matching_migration(target): raise YunohostValidationError("migrations_no_such_migration", id=target) + # Dirty hack to mark the bullseye->bookworm as done ... + # it may still be marked as 'pending' if for some reason the migration crashed, + # but the admins ran 'apt full-upgrade' to manually finish the migration + # ... in which case it won't be magically flagged as 'done' until here + migrate_to_bookworm = get_matching_migration("migrate_to_bookworm") + if migrate_to_bookworm.state == "pending": + migrate_to_bookworm.state = "done" + _write_migration_state(migrate_to_bookworm.id, "done") + # auto, skip and force are exclusive options if auto + skip + force_rerun > 1: raise YunohostValidationError("migrations_exclusive_options") diff --git a/src/user.py b/src/user.py index 606e8abd54..e3fc5d9d43 100644 --- a/src/user.py +++ b/src/user.py @@ -20,14 +20,13 @@ import re import pwd import grp -import crypt import random -import string import subprocess import copy +from logging import getLogger +from typing import TYPE_CHECKING, Any, TextIO, Optional, Callable, Union, cast from moulinette import Moulinette, m18n -from moulinette.utils.log import getActionLogger from moulinette.utils.process import check_output from yunohost.utils.error import YunohostError, YunohostValidationError @@ -35,7 +34,14 @@ from yunohost.log import is_unit_operation from yunohost.utils.system import binary_to_human -logger = getActionLogger("yunohost.user") +if TYPE_CHECKING: + from yunohost.log import OperationLogger + from bottle import HTTPResponse as HTTPResponseType + from moulinette.utils.log import MoulinetteLogger + logger = cast(MoulinetteLogger, getLogger("yunohost.user")) +else: + logger = getLogger("yunohost.user") + FIELDS_FOR_IMPORT = { "username": r"^[a-z0-9_.]+$", @@ -52,7 +58,7 @@ ADMIN_ALIASES = ["root", "admin", "admins", "webmaster", "postmaster", "abuse"] -def user_list(fields=None): +def user_list(fields: Optional[list[str]] = None) -> dict[str, dict[str, Any]]: from yunohost.utils.ldap import _get_ldap_interface ldap_attrs = { @@ -73,7 +79,7 @@ def user_list(fields=None): def display_default(values, _): return values[0] if len(values) == 1 else values - display = { + display: dict[str, Callable[[list[str], dict], Any]] = { "password": lambda values, user: "", "mail": lambda values, user: display_default(values[:1], user), "mail-alias": lambda values, _: values[1:], @@ -110,15 +116,18 @@ def display_default(values, _): ) for user in result: - entry = {} + entry: dict[str, str] = {} for field in fields: values = [] if ldap_attrs[field] in user: values = user[ldap_attrs[field]] entry[field] = display.get(field, display_default)(values, user) - users[user["uid"][0]] = entry + username: str = user["uid"][0] + users[username] = entry + # Dict entry 0 has incompatible type "str": "dict[Any, dict[str, Any]]"; + # expected "str": "dict[str, str]" [dict-item] return {"users": users} @@ -136,44 +145,32 @@ def shellexists(shell): @is_unit_operation([("username", "user")]) def user_create( - operation_logger, - username, - domain, - password, - fullname=None, - firstname=None, - lastname=None, + operation_logger: "OperationLogger", + username: str, + domain: str, + password: str, + fullname: str, mailbox_quota="0", - admin=False, - from_import=False, + admin: bool = False, + from_import: bool = False, loginShell=None, -): - if firstname or lastname: - logger.warning( - "Options --firstname / --lastname of 'yunohost user create' are deprecated. We recommend using --fullname instead." +) -> dict[str, str]: + if not fullname.strip(): + raise YunohostValidationError( + "You should specify the fullname of the user using option -F" ) - - if not fullname or not fullname.strip(): - if not firstname.strip(): - raise YunohostValidationError( - "You should specify the fullname of the user using option -F" - ) - lastname = ( - lastname or " " - ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... - fullname = f"{firstname} {lastname}".strip() - else: - fullname = fullname.strip() - firstname = fullname.split()[0] - lastname = ( - " ".join(fullname.split()[1:]) or " " - ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + fullname = fullname.strip() + firstname = fullname.split()[0] + lastname = ( + " ".join(fullname.split()[1:]) or " " + ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... from yunohost.domain import domain_list, _get_maindomain, _assert_domain_exists from yunohost.hook import hook_callback from yunohost.utils.password import ( assert_password_is_strong_enough, assert_password_is_compatible, + _hash_user_password, ) from yunohost.utils.ldap import _get_ldap_interface @@ -181,7 +178,7 @@ def user_create( assert_password_is_compatible(password) assert_password_is_strong_enough("admin" if admin else "user", password) - # Validate domain used for email address/xmpp account + # Validate domain used for email address account if domain is None: if Moulinette.interface.type == "api": raise YunohostValidationError( @@ -238,7 +235,7 @@ def user_create( uid_guid_found = False while not uid_guid_found: # LXC uid number is limited to 65536 by default - uid = str(random.randint(1001, 65000)) + uid: str = str(random.randint(1001, 65000)) uid_guid_found = uid not in all_uid and uid not in all_gid if not loginShell: @@ -284,12 +281,12 @@ def user_create( except subprocess.CalledProcessError: home = f"/home/{username}" if not os.path.isdir(home): - logger.warning(m18n.n("user_home_creation_failed", home=home), exc_info=1) + logger.warning(m18n.n("user_home_creation_failed", home=home), exc_info=True) try: subprocess.check_call(["setfacl", "-m", "g:all_users:---", f"/home/{username}"]) except subprocess.CalledProcessError: - logger.warning(f"Failed to protect /home/{username}", exc_info=1) + logger.warning(f"Failed to protect /home/{username}", exc_info=True) # Create group for user and add to group 'all_users' user_group_create(groupname=username, gid=uid, primary_group=True, sync_perm=False) @@ -316,9 +313,11 @@ def user_create( @is_unit_operation([("username", "user")]) -def user_delete(operation_logger, username, purge=False, from_import=False): +def user_delete(operation_logger: "OperationLogger", username: str, purge: bool = False, from_import: bool = False): from yunohost.hook import hook_callback from yunohost.utils.ldap import _get_ldap_interface + from yunohost.authenticators.ldap_ynhuser import Authenticator as PortalAuth + from yunohost.authenticators.ldap_admin import Authenticator as AdminAuth if username not in user_list()["users"]: raise YunohostValidationError("user_unknown", user=username) @@ -347,6 +346,9 @@ def user_delete(operation_logger, username, purge=False, from_import=False): except Exception as e: raise YunohostError("user_deletion_failed", user=username, error=e) + PortalAuth.invalidate_all_sessions_for_user(username) + AdminAuth.invalidate_all_sessions_for_user(username) + # Invalidate passwd to take user deletion into account subprocess.call(["nscd", "-i", "passwd"]) @@ -362,38 +364,35 @@ def user_delete(operation_logger, username, purge=False, from_import=False): @is_unit_operation([("username", "user")], exclude=["change_password"]) def user_update( - operation_logger, - username, - firstname=None, - lastname=None, - mail=None, - change_password=None, - add_mailforward=None, - remove_mailforward=None, - add_mailalias=None, - remove_mailalias=None, - mailbox_quota=None, - from_import=False, - fullname=None, - loginShell=None, + operation_logger: "OperationLogger", + username: str, + mail: Optional[str] = None, + change_password: Optional[str] = None, + add_mailforward: None | str | list[str] = None, + remove_mailforward: None | str | list[str] = None, + add_mailalias: None | str | list[str] = None, + remove_mailalias: None | str | list[str] = None, + mailbox_quota: Optional[str] = None, + from_import: bool = False, + fullname: Optional[str] = None, + loginShell: Optional[str] = None, ): - if firstname or lastname: - logger.warning( - "Options --firstname / --lastname of 'yunohost user create' are deprecated. We recommend using --fullname instead." - ) - if fullname and fullname.strip(): fullname = fullname.strip() firstname = fullname.split()[0] lastname = ( " ".join(fullname.split()[1:]) or " " ) # Stupid hack because LDAP requires the sn/lastname attr, but it accepts a single whitespace... + else: + firstname = None + lastname = None from yunohost.domain import domain_list from yunohost.app import app_ssowatconf from yunohost.utils.password import ( assert_password_is_strong_enough, assert_password_is_compatible, + _hash_user_password, ) from yunohost.utils.ldap import _get_ldap_interface from yunohost.hook import hook_callback @@ -411,7 +410,7 @@ def user_update( if not result: raise YunohostValidationError("user_unknown", user=username) user = result[0] - env_dict = {"YNH_USER_USERNAME": username} + env_dict: dict[str, str] = {"YNH_USER_USERNAME": username} # Get modifications from arguments new_attr_dict = {} @@ -440,9 +439,9 @@ def user_update( # without a specified value, change_password will be set to the const 0. # In this case we prompt for the new password. if Moulinette.interface.type == "cli" and not change_password: - change_password = Moulinette.prompt( + change_password = cast(str, Moulinette.prompt( m18n.n("ask_password"), is_password=True, confirm=True - ) + )) # Ensure compatibility and sufficiently complex password assert_password_is_compatible(change_password) @@ -474,7 +473,7 @@ def user_update( new_attr_dict["mail"] = [mail] + user["mail"][1:] - if add_mailalias: + if add_mailalias is not None: if not isinstance(add_mailalias, list): add_mailalias = [add_mailalias] for mail in add_mailalias: @@ -549,6 +548,11 @@ def user_update( except Exception as e: raise YunohostError("user_update_failed", user=username, error=e) + if "userPassword" in new_attr_dict: + logger.info("Invalidating sessions") + from yunohost.authenticators.ldap_ynhuser import Authenticator as PortalAuth + PortalAuth.invalidate_all_sessions_for_user(username) + # Invalidate passwd and group to update the loginShell subprocess.call(["nscd", "-i", "passwd"]) subprocess.call(["nscd", "-i", "group"]) @@ -562,7 +566,7 @@ def user_update( return user_info(username) -def user_info(username): +def user_info(username: str) -> dict[str, str]: """ Get user informations @@ -631,8 +635,8 @@ def user_info(username): has_value = re.search(r"Value=(\d+)", cmd_result) if has_value: - storage_use = int(has_value.group(1)) * 1000 - storage_use = binary_to_human(storage_use) + storage_use_int = int(has_value.group(1)) * 1000 + storage_use = binary_to_human(storage_use_int) if is_limited: has_percent = re.search(r"%=(\d+)", cmd_result) @@ -649,7 +653,7 @@ def user_info(username): return result_dict -def user_export(): +def user_export() -> Union[str, "HTTPResponseType"]: """ Export users into CSV @@ -691,7 +695,7 @@ def user_export(): @is_unit_operation() -def user_import(operation_logger, csvfile, update=False, delete=False): +def user_import(operation_logger: "OperationLogger", csvfile: TextIO, update: bool = False, delete: bool = False) -> dict[str, int]: """ Import users from CSV @@ -707,7 +711,7 @@ def user_import(operation_logger, csvfile, update=False, delete=False): from yunohost.domain import domain_list # Pre-validate data and prepare what should be done - actions = {"created": [], "updated": [], "deleted": []} + actions: dict[str, list[dict[str, Any]]] = {"created": [], "updated": [], "deleted": []} is_well_formatted = True def to_list(str_list): @@ -720,10 +724,11 @@ def to_list(str_list): existing_domains = domain_list()["domains"] reader = csv.DictReader(csvfile, delimiter=";", quotechar='"') + reader_fields = cast(list[str], reader.fieldnames) users_in_csv = [] - missing_columns = [ - key for key in FIELDS_FOR_IMPORT.keys() if key not in reader.fieldnames + missing_columns: list[str] = [ + key for key in FIELDS_FOR_IMPORT.keys() if key not in reader_fields ] if missing_columns: raise YunohostValidationError( @@ -765,7 +770,7 @@ def to_list(str_list): for mail in user["mail-alias"] if mail.split("@", 1)[1] not in existing_domains ] - unknown_domains = set(unknown_domains) + unknown_domains = list(set(unknown_domains)) if unknown_domains: format_errors.append( @@ -799,7 +804,7 @@ def to_list(str_list): if delete: actions["deleted"] = [ - user for user in existing_users if user not in users_in_csv + {"username": user} for user in existing_users if user not in users_in_csv ] if delete and not users_in_csv: @@ -815,7 +820,7 @@ def to_list(str_list): if total == 0: logger.info(m18n.n("user_import_nothing_to_do")) - return + return {} # Apply creation, update and deletion operation result = {"created": 0, "updated": 0, "deleted": 0, "errors": 0} @@ -832,14 +837,14 @@ def progress(info=""): progress.old = bar logger.info(bar) - progress.nb = 0 - progress.old = "" + progress.nb = 0 # type: ignore[attr-defined] + progress.old = "" # type: ignore[attr-defined] - def on_failure(user, exception): + def _on_failure(user, exception): result["errors"] += 1 logger.error(user + ": " + str(exception)) - def update(new_infos, old_infos=False): + def _import_update(new_infos, old_infos=False): remove_alias = None remove_forward = None remove_groups = [] @@ -884,8 +889,7 @@ def update(new_infos, old_infos=False): user_update( new_infos["username"], - firstname=new_infos["firstname"], - lastname=new_infos["lastname"], + fullname=(new_infos["firstname"] + " " + new_infos["lastname"]).strip(), change_password=new_infos["password"], mailbox_quota=new_infos["mailbox-quota"], mail=new_infos["mail"], @@ -908,18 +912,18 @@ def update(new_infos, old_infos=False): # We do delete and update before to avoid mail uniqueness issues for user in actions["deleted"]: try: - user_delete(user, purge=True, from_import=True) + user_delete(user["username"], purge=True, from_import=True) result["deleted"] += 1 except YunohostError as e: - on_failure(user, e) + _on_failure(user, e) progress(f"Deleting {user}") for user in actions["updated"]: try: - update(user, users[user["username"]]) + _import_update(user, users[user["username"]]) result["updated"] += 1 except YunohostError as e: - on_failure(user["username"], e) + _on_failure(user["username"], e) progress(f"Updating {user['username']}") for user in actions["created"]: @@ -928,15 +932,14 @@ def update(new_infos, old_infos=False): user["username"], user["domain"], user["password"], - user["mailbox-quota"], + mailbox_quota=user["mailbox-quota"], from_import=True, - firstname=user["firstname"], - lastname=user["lastname"], + fullname=(user["firstname"] + " " + user["lastname"]).strip(), ) - update(user) + _import_update(user) result["created"] += 1 except YunohostError as e: - on_failure(user["username"], e) + _on_failure(user["username"], e) progress(f"Creating {user['username']}") permission_sync_to_user() @@ -957,12 +960,11 @@ def update(new_infos, old_infos=False): # # Group subcategory # -def user_group_list(short=False, full=False, include_primary_groups=True): +def user_group_list(full: bool = False, include_primary_groups: bool = True) -> dict[str, dict[str, dict]]: """ List users Keyword argument: - short -- Only list the name of the groups without any additional info full -- List all the info available for each groups include_primary_groups -- Include groups corresponding to users (which should always only contains this user) This option is set to false by default in the action map because we don't want to have @@ -984,7 +986,7 @@ def user_group_list(short=False, full=False, include_primary_groups=True): # Parse / organize information to be outputed users = user_list()["users"] - groups = {} + groups: dict[str, dict[str, Any]] = {} for infos in groups_infos: name = infos["cn"][0] @@ -1002,16 +1004,13 @@ def user_group_list(short=False, full=False, include_primary_groups=True): _ldap_path_extract(p, "cn") for p in infos.get("permission", []) ] - if short: - groups = list(groups.keys()) - return {"groups": groups} @is_unit_operation([("groupname", "group")]) def user_group_create( - operation_logger, groupname, gid=None, primary_group=False, sync_perm=True -): + operation_logger: "OperationLogger", groupname: str, gid: Optional[str] = None, primary_group: bool = False, sync_perm: bool = True +) -> dict[str, str]: """ Create group @@ -1046,7 +1045,7 @@ def user_group_create( if not gid: # Get random GID - all_gid = {x.gr_gid for x in grp.getgrall()} + all_gid = {str(x.gr_gid) for x in grp.getgrall()} uid_guid_found = False while not uid_guid_found: @@ -1056,7 +1055,7 @@ def user_group_create( attr_dict = { "objectClass": ["top", "groupOfNamesYnh", "posixGroup"], "cn": groupname, - "gidNumber": gid, + "gidNumber": [gid], } # Here we handle the creation of a primary group @@ -1083,7 +1082,7 @@ def user_group_create( @is_unit_operation([("groupname", "group")]) -def user_group_delete(operation_logger, groupname, force=False, sync_perm=True): +def user_group_delete(operation_logger: "OperationLogger", groupname: str, force: bool = False, sync_perm: bool = True) -> None: """ Delete user @@ -1125,16 +1124,16 @@ def user_group_delete(operation_logger, groupname, force=False, sync_perm=True): @is_unit_operation([("groupname", "group")]) def user_group_update( - operation_logger, - groupname, - add=None, - remove=None, - add_mailalias=None, - remove_mailalias=None, - force=False, - sync_perm=True, - from_import=False, -): + operation_logger: "OperationLogger", + groupname: str, + add: None | str | list[str] = None, + remove: None | str | list[str] = None, + add_mailalias: None | str | list[str] = None, + remove_mailalias: None | str | list[str] = None, + force: bool = False, + sync_perm: bool = True, + from_import: bool = False, +) -> None | dict[str, Any]: from yunohost.permission import permission_sync_to_user from yunohost.utils.ldap import _get_ldap_interface, _ldap_path_extract from yunohost.hook import hook_callback @@ -1175,7 +1174,7 @@ def user_group_update( _ldap_path_extract(p, "uid") for p in group.get("member", []) ] new_group_members = copy.copy(current_group_members) - new_attr_dict = {} + new_attr_dict: dict[str, list] = {} # Group permissions current_group_permissions = [ @@ -1220,8 +1219,8 @@ def user_group_update( new_group_members_dns = [ "uid=" + user + ",ou=users,dc=yunohost,dc=org" for user in new_group_members ] - new_attr_dict["member"] = set(new_group_members_dns) - new_attr_dict["memberUid"] = set(new_group_members) + new_attr_dict["member"] = list(set(new_group_members_dns)) + new_attr_dict["memberUid"] = list(set(new_group_members)) # Check the whole alias situation if add_mailalias: @@ -1273,7 +1272,7 @@ def user_group_update( if set(new_group_mail) != set(current_group_mail): logger.info(m18n.n("group_update_aliases", group=groupname)) - new_attr_dict["mail"] = set(new_group_mail) + new_attr_dict["mail"] = list(set(new_group_mail)) if new_attr_dict["mail"] and "mailGroup" not in group["objectClass"]: new_attr_dict["objectClass"] = group["objectClass"] + ["mailGroup"] @@ -1292,6 +1291,11 @@ def user_group_update( except Exception as e: raise YunohostError("group_update_failed", group=groupname, error=e) + if groupname == "admins" and remove: + from yunohost.authenticators.ldap_admin import Authenticator as AdminAuth + for user in users_to_remove: + AdminAuth.invalidate_all_sessions_for_user(user) + if sync_perm: permission_sync_to_user() @@ -1326,8 +1330,10 @@ def user_group_update( return user_group_info(groupname) + return None + -def user_group_info(groupname): +def user_group_info(groupname: str) -> dict[str, Any]: """ Get user informations @@ -1363,7 +1369,7 @@ def user_group_info(groupname): } -def user_group_add(groupname, usernames, force=False, sync_perm=True): +def user_group_add(groupname: str, usernames: list[str], force: bool = False, sync_perm: bool = True) -> Optional[dict[str, Any]]: """ Add user(s) to a group @@ -1375,7 +1381,7 @@ def user_group_add(groupname, usernames, force=False, sync_perm=True): return user_group_update(groupname, add=usernames, force=force, sync_perm=sync_perm) -def user_group_remove(groupname, usernames, force=False, sync_perm=True): +def user_group_remove(groupname: str, usernames: list[str], force: bool = False, sync_perm: bool = True) -> Optional[dict[str, Any]]: """ Remove user(s) from a group @@ -1389,30 +1395,27 @@ def user_group_remove(groupname, usernames, force=False, sync_perm=True): ) -def user_group_add_mailalias(groupname, aliases, force=False): - return user_group_update( - groupname, add_mailalias=aliases, force=force, sync_perm=False - ) +def user_group_add_mailalias(groupname: str, aliases: list[str], force: bool = False) -> Optional[dict[str, Any]]: + return user_group_update(groupname, add_mailalias=aliases, force=force, sync_perm=False) -def user_group_remove_mailalias(groupname, aliases, force=False): - return user_group_update( - groupname, remove_mailalias=aliases, force=force, sync_perm=False - ) +def user_group_remove_mailalias(groupname: str, aliases: list[str], force: bool = False) -> Optional[dict[str, Any]]: + return user_group_update(groupname, remove_mailalias=aliases, force=force, sync_perm=False) # # Permission subcategory # - -def user_permission_list(short=False, full=False, apps=[]): +# FIXME: missing return type +def user_permission_list(short: bool = False, full: bool = False, apps: list[str] = []): from yunohost.permission import user_permission_list return user_permission_list(short, full, absolute_urls=True, apps=apps) -def user_permission_update(permission, label=None, show_tile=None, sync_perm=True): +# FIXME: missing return type +def user_permission_update(permission: str, label: Optional[str] = None, show_tile: Optional[bool] = None, sync_perm: bool = True): from yunohost.permission import user_permission_update return user_permission_update( @@ -1420,7 +1423,8 @@ def user_permission_update(permission, label=None, show_tile=None, sync_perm=Tru ) -def user_permission_add(permission, names, protected=None, force=False, sync_perm=True): +# FIXME: missing return type +def user_permission_add(permission: str, names: list[str], protected: Optional[bool] = None, force: bool = False, sync_perm: bool = True): from yunohost.permission import user_permission_update return user_permission_update( @@ -1428,8 +1432,9 @@ def user_permission_add(permission, names, protected=None, force=False, sync_per ) +# FIXME: missing return type def user_permission_remove( - permission, names, protected=None, force=False, sync_perm=True + permission: str, names: list[str], protected: Optional[bool] = None, force: bool = False, sync_perm: bool = True ): from yunohost.permission import user_permission_update @@ -1438,13 +1443,15 @@ def user_permission_remove( ) -def user_permission_reset(permission, sync_perm=True): +# FIXME: missing return type +def user_permission_reset(permission: str, sync_perm: bool = True): from yunohost.permission import user_permission_reset return user_permission_reset(permission, sync_perm=sync_perm) -def user_permission_info(permission): +# FIXME: missing return type +def user_permission_info(permission: str): from yunohost.permission import user_permission_info return user_permission_info(permission) @@ -1456,15 +1463,15 @@ def user_permission_info(permission): import yunohost.ssh -def user_ssh_list_keys(username): +def user_ssh_list_keys(username: str) -> dict[str, dict[str, str]]: return yunohost.ssh.user_ssh_list_keys(username) -def user_ssh_add_key(username, key, comment): +def user_ssh_add_key(username: str, key: str, comment: Optional[str] = None) -> None: return yunohost.ssh.user_ssh_add_key(username, key, comment) -def user_ssh_remove_key(username, key): +def user_ssh_remove_key(username: str, key: str) -> None: return yunohost.ssh.user_ssh_remove_key(username, key) @@ -1472,37 +1479,7 @@ def user_ssh_remove_key(username, key): # End SSH subcategory # - -def _hash_user_password(password): - """ - This function computes and return a salted hash for the password in input. - This implementation is inspired from [1]. - - The hash follows SHA-512 scheme from Linux/glibc. - Hence the {CRYPT} and $6$ prefixes - - {CRYPT} means it relies on the OS' crypt lib - - $6$ corresponds to SHA-512, the strongest hash available on the system - - The salt is generated using random.SystemRandom(). It is the crypto-secure - pseudo-random number generator according to the python doc [2] (c.f. the - red square). It internally relies on /dev/urandom - - The salt is made of 16 characters from the set [./a-zA-Z0-9]. This is the - max sized allowed for salts according to [3] - - [1] https://www.redpill-linpro.com/techblog/2016/08/16/ldap-password-hash.html - [2] https://docs.python.org/2/library/random.html - [3] https://www.safaribooksonline.com/library/view/practical-unix-and/0596003234/ch04s03.html - """ - - char_set = string.ascii_uppercase + string.ascii_lowercase + string.digits + "./" - salt = "".join([random.SystemRandom().choice(char_set) for x in range(16)]) - - salt = "$6$" + salt + "$" - return "{CRYPT}" + crypt.crypt(str(password), salt) - - -def _update_admins_group_aliases(old_main_domain, new_main_domain): +def _update_admins_group_aliases(old_main_domain: str, new_main_domain: str) -> None: current_admin_aliases = user_group_info("admins")["mail-aliases"] aliases_to_remove = [ diff --git a/src/utils/configpanel.py b/src/utils/configpanel.py index b2f9688682..a17a634741 100644 --- a/src/utils/configpanel.py +++ b/src/utils/configpanel.py @@ -19,36 +19,403 @@ import glob import os import re -import urllib.parse from collections import OrderedDict -from typing import Union +from logging import getLogger +from typing import TYPE_CHECKING, Any, Iterator, Literal, Sequence, Type, Union, cast + +from pydantic import BaseModel, Extra, validator from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize from moulinette.utils.filesystem import mkdir, read_toml, read_yaml, write_to_yaml -from moulinette.utils.log import getActionLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.form import ( - OPTIONS, - BaseChoicesOption, + AnyOption, BaseInputOption, - BaseOption, + BaseReadonlyOption, FileOption, + OptionsModel, OptionType, - ask_questions_and_parse_answers, + Translation, + build_form, evaluate_simple_js_expression, + parse_prefilled_values, + prompt_or_validate_form, ) from yunohost.utils.i18n import _value_for_locale -logger = getActionLogger("yunohost.configpanel") +if TYPE_CHECKING: + from pydantic.fields import ModelField + from pydantic.typing import AbstractSetIntStr, MappingIntStrAny + + from yunohost.utils.form import FormModel, Hooks + from yunohost.log import OperationLogger + +if TYPE_CHECKING: + from moulinette.utils.log import MoulinetteLogger + + logger = cast(MoulinetteLogger, getLogger("yunohost.configpanel")) +else: + logger = getLogger("yunohost.configpanel") + + +# ╭───────────────────────────────────────────────────────╮ +# │ ╭╮╮╭─╮┌─╮┌─╴╷ ╭─╴ │ +# │ ││││ ││ │├─╴│ ╰─╮ │ +# │ ╵╵╵╰─╯└─╯╰─╴╰─╴╶─╯ │ +# ╰───────────────────────────────────────────────────────╯ + CONFIG_PANEL_VERSION_SUPPORTED = 1.0 +class ContainerModel(BaseModel): + id: str + name: Union[Translation, None] = None + services: list[str] = [] + help: Union[Translation, None] = None + + def translate(self, i18n_key: Union[str, None] = None) -> None: + """ + Translate `ask` and `name` attributes of panels and section. + This is in place mutation. + """ + + for key in ("help", "name"): + value = getattr(self, key) + if value: + setattr(self, key, _value_for_locale(value)) + elif m18n.key_exists(f"{i18n_key}_{self.id}_{key}"): + setattr(self, key, m18n.n(f"{i18n_key}_{self.id}_{key}")) + + +class SectionModel(ContainerModel, OptionsModel): + """ + Sections are, basically, options grouped together. Sections are `dict`s defined inside a Panel and require a unique id (in the below example, the id is `customization` prepended by the panel's id `main`). Keep in mind that this combined id will be used in CLI to refer to the section, so choose something short and meaningfull. Also make sure to not make a typo in the panel id, which would implicitly create an other entire panel. + + If at least one `button` is present it then become an action section. + Options in action sections are not considered settings and therefor are not saved, they are more like parameters that exists only during the execution of an action. + FIXME i'm not sure we have this in code. + + #### Examples + ```toml + [main] + + [main.customization] + name.en = "Advanced configuration" + name.fr = "Configuration avancée" + help = "Every form items in this section are not saved." + services = ["__APP__", "nginx"] + + [main.customization.option_id] + type = "string" + # …refer to Options doc + ``` + + #### Properties + - `name` (optional): `Translation` or `str`, displayed as the section's title if any + - `help`: `Translation` or `str`, text to display before the first option + - `services` (optional): `list` of services names to `reload-or-restart` when any option's value contained in the section changes + - `"__APP__` will refer to the app instance name + - `optional`: `bool` (default: `true`), set the default `optional` prop of all Options in the section + - `visible`: `bool` or `JSExpression` (default: `true`), allow to conditionally display a section depending on user's answers to previous questions. + - Be careful that the `visible` property should only refer to **previous** options's value. Hence, it should not make sense to have a `visible` property on the very first section. + """ + + visible: Union[bool, str] = True + optional: bool = True + is_action_section: bool = False + bind: Union[str, None] = None + + class Config: + @staticmethod + def schema_extra(schema: dict[str, Any]) -> None: + del schema["properties"]["id"] + options = schema["properties"].pop("options") + del schema["required"] + schema["additionalProperties"] = options["items"] + + # Don't forget to pass arguments to super init + def __init__( + self, + id: str, + name: Union[Translation, None] = None, + services: list[str] = [], + help: Union[Translation, None] = None, + visible: Union[bool, str] = True, + optional: bool = True, + bind: Union[str, None] = None, + **kwargs, + ) -> None: + options = self.options_dict_to_list(kwargs, optional=optional) + is_action_section = any( + [option["type"] == OptionType.button for option in options] + ) + ContainerModel.__init__( # type: ignore + self, + id=id, + name=name, + services=services, + help=help, + visible=visible, + bind=bind, + options=options, + is_action_section=is_action_section, + ) + + def is_visible(self, context: dict[str, Any]) -> bool: + if isinstance(self.visible, bool): + return self.visible + + return evaluate_simple_js_expression(self.visible, context=context) + + def translate(self, i18n_key: Union[str, None] = None) -> None: + """ + Call to `Container`'s `translate` for self translation + + Call to `OptionsContainer`'s `translate_options` for options translation + """ + super().translate(i18n_key) + self.translate_options(i18n_key) + + +class PanelModel(ContainerModel): + """ + Panels are, basically, sections grouped together. Panels are `dict`s defined inside a ConfigPanel file and require a unique id (in the below example, the id is `main`). Keep in mind that this id will be used in CLI to refer to the panel, so choose something short and meaningfull. + + #### Examples + ```toml + [main] + name.en = "Main configuration" + name.fr = "Configuration principale" + help = "" + services = ["__APP__", "nginx"] + + [main.customization] + # …refer to Sections doc + ``` + #### Properties + - `name`: `Translation` or `str`, displayed as the panel title + - `help` (optional): `Translation` or `str`, text to display before the first section + - `services` (optional): `list` of services names to `reload-or-restart` when any option's value contained in the panel changes + - `"__APP__` will refer to the app instance name + - `actions`: FIXME not sure what this does + """ + + # FIXME what to do with `actions? + actions: dict[str, Translation] = {"apply": {"en": "Apply"}} + bind: Union[str, None] = None + sections: list[SectionModel] + + class Config: + extra = Extra.allow + + @staticmethod + def schema_extra(schema: dict[str, Any]) -> None: + del schema["properties"]["id"] + del schema["properties"]["sections"] + del schema["required"] + schema["additionalProperties"] = {"$ref": "#/definitions/SectionModel"} + + # Don't forget to pass arguments to super init + def __init__( + self, + id: str, + name: Union[Translation, None] = None, + services: list[str] = [], + help: Union[Translation, None] = None, + bind: Union[str, None] = None, + **kwargs, + ) -> None: + sections = [data | {"id": name} for name, data in kwargs.items()] + super().__init__( # type: ignore + id=id, name=name, services=services, help=help, bind=bind, sections=sections + ) + + def translate(self, i18n_key: Union[str, None] = None) -> None: + """ + Recursivly mutate translatable attributes to their translation + """ + super().translate(i18n_key) + + for section in self.sections: + section.translate(i18n_key) + + +class ConfigPanelModel(BaseModel): + """ + This is the 'root' level of the config panel toml file + + #### Examples + + ```toml + version = 1.0 + + [config] + # …refer to Panels doc + ``` + + #### Properties + + - `version`: `float` (default: `1.0`), version that the config panel supports in terms of features. + - `i18n` (optional): `str`, an i18n property that let you internationalize options text. + - However this feature is only available in core configuration panel (like `yunohost domain config`), prefer the use `Translation` in `name`, `help`, etc. + + """ + + version: float = CONFIG_PANEL_VERSION_SUPPORTED + i18n: Union[str, None] = None + panels: list[PanelModel] + + class Config: + arbitrary_types_allowed = True + extra = Extra.allow + + @staticmethod + def schema_extra(schema: dict[str, Any]) -> None: + """Update the schema to the expected input + In actual TOML definition, schema is like: + ```toml + [panel_1] + [panel_1.section_1] + [panel_1.section_1.option_1] + ``` + Which is equivalent to `{"panel_1": {"section_1": {"option_1": {}}}}` + so `section_id` (and `option_id`) are additional property of `panel_id`, + which is convinient to write but not ideal to iterate. + In ConfigPanelModel we gather additional properties of panels, sections + and options as lists so that structure looks like: + `{"panels`: [{"id": "panel_1", "sections": [{"id": "section_1", "options": [{"id": "option_1"}]}]}] + """ + del schema["properties"]["panels"] + del schema["required"] + schema["additionalProperties"] = {"$ref": "#/definitions/PanelModel"} + + # Don't forget to pass arguments to super init + def __init__( + self, + version: float, + i18n: Union[str, None] = None, + **kwargs, + ) -> None: + panels = [data | {"id": name} for name, data in kwargs.items()] + super().__init__(version=version, i18n=i18n, panels=panels) + + @property + def sections(self) -> Iterator[SectionModel]: + """Convinient prop to iter on all sections""" + for panel in self.panels: + for section in panel.sections: + yield section + + @property + def options(self) -> Iterator[AnyOption]: + """Convinient prop to iter on all options""" + for section in self.sections: + for option in section.options: + yield option + + def get_panel(self, panel_id: str) -> Union[PanelModel, None]: + for panel in self.panels: + if panel.id == panel_id: + return panel + return None + + def get_section(self, section_id: str) -> Union[SectionModel, None]: + for section in self.sections: + if section.id == section_id: + return section + return None + + def get_option(self, option_id: str) -> Union[AnyOption, None]: + for option in self.options: + if option.id == option_id: + return option + # FIXME raise error? + return None + + @property + def services(self) -> list[str]: + services = set() + for panel in self.panels: + services |= set(panel.services) + for section in panel.sections: + services |= set(section.services) + + services_ = list(services) + services_.sort(key="nginx".__eq__) + return services_ + + def iter_children( + self, + trigger: list[Literal["panel", "section", "option", "action"]] = ["option"], + ): + for panel in self.panels: + if "panel" in trigger: + yield (panel, None, None) + for section in panel.sections: + if "section" in trigger: + yield (panel, section, None) + if "action" in trigger: + for option in section.options: + if option.type is OptionType.button: + yield (panel, section, option) + if "option" in trigger: + for option in section.options: + yield (panel, section, option) + + def translate(self) -> None: + """ + Recursivly mutate translatable attributes to their translation + """ + for panel in self.panels: + panel.translate(self.i18n) + + @validator("version", always=True) + def check_version(cls, value: float, field: "ModelField") -> float: + if value < CONFIG_PANEL_VERSION_SUPPORTED: + raise ValueError( + f"Config panels version '{value}' are no longer supported." + ) + + return value + + +# ╭───────────────────────────────────────────────────────╮ +# │ ╭─╴╭─╮╭╮╷┌─╴╶┬╴╭─╮ ╶┬╴╭╮╮┌─╮╷ │ +# │ │ │ ││││├─╴ │ │╶╮ │ │││├─╯│ │ +# │ ╰─╴╰─╯╵╰╯╵ ╶┴╴╰─╯ ╶┴╴╵╵╵╵ ╰─╴ │ +# ╰───────────────────────────────────────────────────────╯ + +if TYPE_CHECKING: + FilterKey = Sequence[Union[str, None]] + RawConfig = OrderedDict[str, Any] + RawSettings = dict[str, Any] + ConfigPanelGetMode = Literal["classic", "full", "export"] + + +def parse_filter_key(key: Union[str, None] = None) -> "FilterKey": + if key and key.count(".") > 2: + raise YunohostError( + f"The filter key {key} has too many sub-levels, the max is 3.", + raw_msg=True, + ) + + if not key: + return (None, None, None) + keys = key.split(".") + return tuple(keys[i] if len(keys) > i else None for i in range(3)) + + class ConfigPanel: entity_type = "config" save_path_tpl: Union[str, None] = None config_path_tpl = "/usr/share/yunohost/config_{entity_type}.toml" save_mode = "full" + settings_must_be_defined: bool = False + filter_key: "FilterKey" = (None, None, None) + config: Union[ConfigPanelModel, None] = None + form: Union["FormModel", None] = None + raw_settings: "RawSettings" = {} + hooks: "Hooks" = {} @classmethod def list(cls): @@ -67,7 +434,9 @@ def list(cls): entities = [] return entities - def __init__(self, entity, config_path=None, save_path=None, creation=False): + def __init__( + self, entity, config_path=None, save_path=None, creation=False + ) -> None: self.entity = entity self.config_path = config_path if not config_path: @@ -77,9 +446,6 @@ def __init__(self, entity, config_path=None, save_path=None, creation=False): self.save_path = save_path if not save_path and self.save_path_tpl: self.save_path = self.save_path_tpl.format(entity=entity) - self.config = {} - self.values = {} - self.new_values = {} if ( self.save_path @@ -103,91 +469,87 @@ def __init__(self, entity, config_path=None, save_path=None, creation=False): and re.match("^(validate|post_ask)__", func) } - def get(self, key="", mode="classic"): - self.filter_key = key or "" + def get( + self, key: Union[str, None] = None, mode: "ConfigPanelGetMode" = "classic" + ) -> Any: + self.filter_key = parse_filter_key(key) + self.config, self.form = self._get_config_panel(prevalidate=False) - # Read config panel toml - self._get_config_panel() - - if not self.config: - raise YunohostValidationError("config_no_panel") - - # Read or get values and hydrate the config - self._get_raw_settings() - self._hydrate() + panel_id, section_id, option_id = self.filter_key # In 'classic' mode, we display the current value if key refer to an option - if self.filter_key.count(".") == 2 and mode == "classic": - option = self.filter_key.split(".")[-1] - value = self.values.get(option, None) + if option_id and mode == "classic": + option = self.config.get_option(option_id) - option_type = None - for _, _, option_ in self._iterate(): - if option_["id"] == option: - option_type = OPTIONS[option_["type"]] - break + if option is None: + # FIXME i18n + raise YunohostValidationError( + f"Couldn't find any option with id {option_id}", raw_msg=True + ) - return option_type.normalize(value) if option_type else value + if isinstance(option, BaseReadonlyOption): + return None + + return option.normalize(self.form[option_id], option) # Format result in 'classic' or 'export' mode + self.config.translate() logger.debug(f"Formating result in '{mode}' mode") - result = {} - for panel, section, option in self._iterate(): - if section["is_action_section"] and mode != "full": - continue - - key = f"{panel['id']}.{section['id']}.{option['id']}" - if mode == "export": - result[option["id"]] = option.get("current_value") - continue - - ask = None - if "ask" in option: - ask = _value_for_locale(option["ask"]) - elif "i18n" in self.config: - ask = m18n.n(self.config["i18n"] + "_" + option["id"]) - - if mode == "full": - option["ask"] = ask - question_class = OPTIONS[option.get("type", OptionType.string)] - # FIXME : maybe other properties should be taken from the question, not just choices ?. - if issubclass(question_class, BaseChoicesOption): - option["choices"] = question_class(option).choices - if issubclass(question_class, BaseInputOption): - option["default"] = question_class(option).default - option["pattern"] = question_class(option).pattern - else: - result[key] = {"ask": ask} - if "current_value" in option: - question_class = OPTIONS[option.get("type", OptionType.string)] - if hasattr(question_class, "humanize"): - result[key]["value"] = question_class.humanize( - option["current_value"], option - ) - else: - result[key]["value"] = option["current_value"] - - # FIXME: semantics, technically here this is not about a prompt... - if getattr(question_class, "hide_user_input_in_prompt", None): - result[key][ - "value" - ] = "**************" # Prevent displaying password in `config get` if mode == "full": - return self.config - else: + result = self.config.dict(exclude_none=True) + + for panel in result["panels"]: + for section in panel["sections"]: + for opt in section["options"]: + instance = self.config.get_option(opt["id"]) + if isinstance(instance, BaseInputOption): + opt["value"] = instance.normalize( + self.form[opt["id"]], instance + ) return result - def set( - self, key=None, value=None, args=None, args_file=None, operation_logger=None - ): - self.filter_key = key or "" + result = OrderedDict() - # Read config panel toml - self._get_config_panel() + for panel in self.config.panels: + for section in panel.sections: + if section.is_action_section and mode != "full": + continue - if not self.config: - raise YunohostValidationError("config_no_panel") + for option in section.options: + # FIXME not sure why option resolves as possibly `None` + option = cast(AnyOption, option) + + if mode == "export": + if isinstance(option, BaseInputOption): + result[option.id] = self.form[option.id] + continue + + if mode == "classic": + key = f"{panel.id}.{section.id}.{option.id}" + result[key] = {"ask": option.ask} + + if isinstance(option, BaseInputOption): + result[key]["value"] = option.humanize( + self.form[option.id], option + ) + if option.type is OptionType.password: + result[key][ + "value" + ] = "**************" # Prevent displaying password in `config get` + + return result + + def set( + self, + key: Union[str, None] = None, + value: Any = None, + args: Union[str, None] = None, + args_file: Union[str, None] = None, + operation_logger: Union["OperationLogger", None] = None, + ) -> None: + self.filter_key = parse_filter_key(key) + panel_id, section_id, option_id = self.filter_key if (args is not None or args_file is not None) and value is not None: raise YunohostValidationError( @@ -195,24 +557,35 @@ def set( raw_msg=True, ) - if self.filter_key.count(".") != 2 and value is not None: + if not option_id and value is not None: raise YunohostValidationError("config_cant_set_value_on_section") # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") - self._parse_pre_answered(args, value, args_file) + if option_id and value is not None: + prefilled_answers = {option_id: value} + else: + prefilled_answers = parse_prefilled_values(args, args_file) + + self.config, self.form = self._get_config_panel() + # FIXME find a better way to exclude previous settings + previous_settings = self.form.dict() - # Read or get values and hydrate the config - self._get_raw_settings() - self._hydrate() - BaseOption.operation_logger = operation_logger - self._ask() + # FIXME Not sure if this is need (redact call to operation logger does it on all the instances) + # BaseOption.operation_logger = operation_logger + + self.form = self._ask( + self.config, + self.form, + prefilled_answers=prefilled_answers, + hooks=self.hooks, + ) if operation_logger: operation_logger.start() try: - self._apply() + self._apply(self.form, self.config, previous_settings) except YunohostError: raise # Script got manually interrupted ... @@ -238,46 +611,59 @@ def set( self._reload_services() logger.success("Config updated as expected") - operation_logger.success() - def list_actions(self): + if operation_logger: + operation_logger.success() + + def list_actions(self) -> dict[str, str]: actions = {} # FIXME : meh, loading the entire config panel is again going to cause # stupid issues for domain (e.g loading registrar stuff when willing to just list available actions ...) - self.filter_key = "" - self._get_config_panel() - for panel, section, option in self._iterate(): - if option["type"] == OptionType.button: - key = f"{panel['id']}.{section['id']}.{option['id']}" - actions[key] = _value_for_locale(option["ask"]) + self.config, self.form = self._get_config_panel() + + for panel, section, option in self.config.iter_children(): + if option.type == OptionType.button: + key = f"{panel.id}.{section.id}.{option.id}" + actions[key] = _value_for_locale(option.ask) return actions - def run_action(self, action=None, args=None, args_file=None, operation_logger=None): + def run_action( + self, + key: Union[str, None] = None, + args: Union[str, None] = None, + args_file: Union[str, None] = None, + operation_logger: Union["OperationLogger", None] = None, + ) -> None: # # FIXME : this stuff looks a lot like set() ... # + panel_id, section_id, action_id = parse_filter_key(key) + # since an action may require some options from its section, + # remove the action_id from the filter + self.filter_key = (panel_id, section_id, None) - self.filter_key = ".".join(action.split(".")[:2]) - action_id = action.split(".")[2] - - # Read config panel toml - self._get_config_panel() + self.config, self.form = self._get_config_panel() # FIXME: should also check that there's indeed a key called action - if not self.config: - raise YunohostValidationError(f"No action named {action}", raw_msg=True) + if not action_id or not self.config.get_option(action_id): + raise YunohostValidationError(f"No action named {action_id}", raw_msg=True) # Import and parse pre-answered options logger.debug("Import and parse pre-answered options") - self._parse_pre_answered(args, None, args_file) + prefilled_answers = parse_prefilled_values(args, args_file) + + self.form = self._ask( + self.config, + self.form, + prefilled_answers=prefilled_answers, + action_id=action_id, + hooks=self.hooks, + ) - # Read or get values and hydrate the config - self._get_raw_settings() - self._hydrate() - BaseOption.operation_logger = operation_logger - self._ask(action=action_id) + # FIXME Not sure if this is need (redact call to operation logger does it on all the instances) + # BaseOption.operation_logger = operation_logger # FIXME: here, we could want to check constrains on # the action's visibility / requirements wrt to the answer to questions ... @@ -286,21 +672,21 @@ def run_action(self, action=None, args=None, args_file=None, operation_logger=No operation_logger.start() try: - self._run_action(action_id) + self._run_action(self.form, action_id) except YunohostError: raise # Script got manually interrupted ... # N.B. : KeyboardInterrupt does not inherit from Exception except (KeyboardInterrupt, EOFError): error = m18n.n("operation_interrupted") - logger.error(m18n.n("config_action_failed", action=action, error=error)) + logger.error(m18n.n("config_action_failed", action=key, error=error)) raise # Something wrong happened in Yunohost's code (most probably hook_exec) except Exception: import traceback error = m18n.n("unexpected_error", error="\n" + traceback.format_exc()) - logger.error(m18n.n("config_action_failed", action=action, error=error)) + logger.error(m18n.n("config_action_failed", action=key, error=error)) raise finally: # Delete files uploaded from API @@ -311,382 +697,241 @@ def run_action(self, action=None, args=None, args_file=None, operation_logger=No # FIXME: i18n logger.success(f"Action {action_id} successful") - operation_logger.success() - def _get_raw_config(self): - return read_toml(self.config_path) - - def _get_config_panel(self): - # Split filter_key - filter_key = self.filter_key.split(".") if self.filter_key != "" else [] - if len(filter_key) > 3: - raise YunohostError( - f"The filter key {filter_key} has too many sub-levels, the max is 3.", - raw_msg=True, - ) + if operation_logger: + operation_logger.success() + def _get_raw_config(self) -> "RawConfig": if not os.path.exists(self.config_path): - logger.debug(f"Config panel {self.config_path} doesn't exists") - return None + raise YunohostValidationError("config_no_panel") - toml_config_panel = self._get_raw_config() + return read_toml(self.config_path) - # Check TOML config panel is in a supported version - if float(toml_config_panel["version"]) < CONFIG_PANEL_VERSION_SUPPORTED: - logger.error( - f"Config panels version {toml_config_panel['version']} are not supported" + def _get_raw_settings(self) -> "RawSettings": + if not self.save_path or not os.path.exists(self.save_path): + return {} + + return read_yaml(self.save_path) or {} + + def _get_partial_raw_config(self) -> "RawConfig": + def filter_keys( + data: "RawConfig", + key: str, + model: Union[Type[ConfigPanelModel], Type[PanelModel], Type[SectionModel]], + ) -> "RawConfig": + # filter in keys defined in model, filter out panels/sections/options that aren't `key` + return OrderedDict( + {k: v for k, v in data.items() if k in model.__fields__ or k == key} ) - return None - - # Transform toml format into internal format - format_description = { - "root": { - "properties": ["version", "i18n"], - "defaults": {"version": 1.0}, - }, - "panels": { - "properties": ["name", "services", "actions", "help", "bind"], - "defaults": { - "services": [], - "actions": {"apply": {"en": "Apply"}}, - }, - }, - "sections": { - "properties": [ - "name", - "services", - "optional", - "help", - "visible", - "bind", - ], - "defaults": { - "name": "", - "services": [], - "optional": True, - "is_action_section": False, - }, - }, - "options": { - "properties": [ - "ask", - "type", - "bind", - "help", - "example", - "default", - "style", - "icon", - "placeholder", - "visible", - "optional", - "choices", - "yes", - "no", - "pattern", - "limit", - "min", - "max", - "step", - "accept", - "redact", - "filter", - "readonly", - "enabled", - # "confirm", # TODO: to ask confirmation before running an action - ], - "defaults": {}, - }, - } - def _build_internal_config_panel(raw_infos, level): - """Convert TOML in internal format ('full' mode used by webadmin) - Here are some properties of 1.0 config panel in toml: - - node properties and node children are mixed, - - text are in english only - - some properties have default values - This function detects all children nodes and put them in a list - """ + raw_config = self._get_raw_config() - defaults = format_description[level]["defaults"] - properties = format_description[level]["properties"] + panel_id, section_id, option_id = self.filter_key - # Start building the ouput (merging the raw infos + defaults) - out = {key: raw_infos.get(key, value) for key, value in defaults.items()} - - # Now fill the sublevels (+ apply filter_key) - i = list(format_description).index(level) - sublevel = list(format_description)[i + 1] if level != "options" else None - search_key = filter_key[i] if len(filter_key) > i else False + try: + if panel_id: + raw_config = filter_keys(raw_config, panel_id, ConfigPanelModel) - for key, value in raw_infos.items(): - # Key/value are a child node - if ( - isinstance(value, OrderedDict) - and key not in properties - and sublevel - ): - # We exclude all nodes not referenced by the filter_key - if search_key and key != search_key: - continue - subnode = _build_internal_config_panel(value, sublevel) - subnode["id"] = key - if level == "root": - subnode.setdefault("name", {"en": key.capitalize()}) - elif level == "sections": - subnode["name"] = key # legacy - subnode.setdefault("optional", raw_infos.get("optional", True)) - # If this section contains at least one button, it becomes an "action" section - if subnode.get("type") == OptionType.button: - out["is_action_section"] = True - out.setdefault(sublevel, []).append(subnode) - # Key/value are a property - else: - if key not in properties: - logger.warning(f"Unknown key '{key}' found in config panel") - # Todo search all i18n keys - out[key] = ( - value - if key not in ["ask", "help", "name"] or isinstance(value, dict) - else {"en": value} + if section_id: + raw_config[panel_id] = filter_keys( + raw_config[panel_id], section_id, PanelModel ) - return out - - self.config = _build_internal_config_panel(toml_config_panel, "root") - try: - self.config["panels"][0]["sections"][0]["options"][0] - except (KeyError, IndexError): + if option_id: + raw_config[panel_id][section_id] = filter_keys( + raw_config[panel_id][section_id], option_id, SectionModel + ) + except KeyError: raise YunohostValidationError( - "config_unknown_filter_key", filter_key=self.filter_key + "config_unknown_filter_key", + filter_key=".".join([k for k in self.filter_key if k]), ) - # List forbidden keywords from helpers and sections toml (to avoid conflict) - forbidden_keywords = [ - "old", - "app", - "changed", - "file_hash", - "binds", - "types", - "formats", - "getter", - "setter", - "short_setting", - "type", - "bind", - "nothing_changed", - "changes_validated", - "result", - "max_progression", - ] - forbidden_keywords += format_description["sections"] - - for _, _, option in self._iterate(): - if option["id"] in forbidden_keywords: - raise YunohostError("config_forbidden_keyword", keyword=option["id"]) - - return self.config - - def _get_default_values(self): - return { - option["id"]: option["default"] - for _, _, option in self._iterate() - if "default" in option - } + return raw_config - def _get_raw_settings(self): - """ - Retrieve entries in YAML file - And set default values if needed - """ + def _get_partial_raw_settings_and_mutate_config( + self, config: ConfigPanelModel + ) -> tuple[ConfigPanelModel, "RawSettings"]: + raw_settings = self._get_raw_settings() + # Save `raw_settings` for diff at `_apply` + self.raw_settings = raw_settings + values = {} + + for _, section, option in config.iter_children(): + value = data = raw_settings.get(option.id, getattr(option, "default", None)) - # Inject defaults if needed (using the magic .update() ;)) - self.values = self._get_default_values() - - # Retrieve entries in the YAML - if os.path.exists(self.save_path) and os.path.isfile(self.save_path): - self.values.update(read_yaml(self.save_path) or {}) - - def _hydrate(self): - # Hydrating config panel with current value - for _, section, option in self._iterate(): - if option["id"] not in self.values: - allowed_empty_types = { - OptionType.alert, - OptionType.display_text, - OptionType.markdown, - OptionType.file, - OptionType.button, - } - - if section["is_action_section"] and option.get("default") is not None: - self.values[option["id"]] = option["default"] - elif ( - option["type"] in allowed_empty_types - or option.get("bind") == "null" - ): + if isinstance(option, BaseInputOption) and option.id not in raw_settings: + if option.default is not None: + value = option.default + elif option.type is OptionType.file or option.bind == "null": continue - else: + elif self.settings_must_be_defined: raise YunohostError( - f"Config panel question '{option['id']}' should be initialized with a value during install or upgrade. (Or maybe the bind key wasn't found?)", + f"Config panel question '{option.id}' should be initialized with a value during install or upgrade.", raw_msg=True, ) - value = self.values[option["id"]] - - # Allow to use value instead of current_value in app config script. - # e.g. apps may write `echo 'value: "foobar"'` in the config file (which is more intuitive that `echo 'current_value: "foobar"'` - # For example hotspot used it... - # See https://github.com/YunoHost/yunohost/pull/1546 - if ( - isinstance(value, dict) - and "value" in value - and "current_value" not in value - ): - value["current_value"] = value["value"] - - # In general, the value is just a simple value. - # Sometimes it could be a dict used to overwrite the option itself - value = value if isinstance(value, dict) else {"current_value": value} - option.update(value) - self.values[option["id"]] = value.get("current_value") + if isinstance(data, dict): + # Settings data if gathered from bash "ynh_app_config_show" + # may be a custom getter that returns a dict with `value` or `current_value` + # and other attributes meant to override those of the option. + + if "value" in data: + value = data.pop("value") + + # Allow to use value instead of current_value in app config script. + # e.g. apps may write `echo 'value: "foobar"'` in the config file (which is more intuitive that `echo 'current_value: "foobar"'` + # For example hotspot used it... + # See https://github.com/YunoHost/yunohost/pull/1546 + # FIXME do we still need the `current_value`? + if "current_value" in data: + value = data.pop("current_value") + + # Mutate other possible option attributes + for k, v in data.items(): + setattr(option, k, v) + + if isinstance(option, BaseInputOption): # or option.bind == "null": + values[option.id] = value + + return (config, values) + + def _get_config_panel( + self, prevalidate: bool = False + ) -> tuple[ConfigPanelModel, "FormModel"]: + raw_config = self._get_partial_raw_config() + config = ConfigPanelModel(**raw_config) + config, raw_settings = self._get_partial_raw_settings_and_mutate_config(config) + config.translate() + Settings = build_form(config.options) + settings = ( + Settings(**raw_settings) + if prevalidate + else Settings.construct(**raw_settings) + ) - return self.values + try: + config.panels[0].sections[0].options[0] + except (KeyError, IndexError): + raise YunohostValidationError( + "config_unknown_filter_key", filter_key=self.filter_key + ) - def _ask(self, action=None): + return (config, settings) + + def _ask( + self, + config: ConfigPanelModel, + form: "FormModel", + prefilled_answers: dict[str, Any] = {}, + action_id: Union[str, None] = None, + hooks: "Hooks" = {}, + ) -> "FormModel": + # FIXME could be turned into a staticmethod logger.debug("Ask unanswered question and prevalidate data") - if "i18n" in self.config: - for panel, section, option in self._iterate(): - if "ask" not in option: - option["ask"] = m18n.n(self.config["i18n"] + "_" + option["id"]) - # auto add i18n help text if present in locales - if m18n.key_exists(self.config["i18n"] + "_" + option["id"] + "_help"): - option["help"] = m18n.n( - self.config["i18n"] + "_" + option["id"] + "_help" - ) + interactive = Moulinette.interface.type == "cli" and os.isatty(1) + verbose = action_id is None or len(list(config.options)) > 1 - def display_header(message): - """CLI panel/section header display""" - if Moulinette.interface.type == "cli" and self.filter_key.count(".") < 2: - Moulinette.display(colorize(message, "purple")) + if interactive: + config.translate() - for panel, section, obj in self._iterate(["panel", "section"]): - if ( - section - and section.get("visible") - and not evaluate_simple_js_expression( - section["visible"], context=self.future_values + for panel in config.panels: + if interactive and verbose: + Moulinette.display( + colorize(f"\n{'='*40}\n>>>> {panel.name}\n{'='*40}", "purple") ) - ): - continue - # Ugly hack to skip action section ... except when when explicitly running actions - if not action: - if section and section["is_action_section"]: + # A section or option may only evaluate its conditions (`visible` + # and `enabled`) with its panel's local context that is built + # prompt after prompt. + # That means that a condition can only reference options of its + # own panel and options that are previously defined. + context: dict[str, Any] = {} + + for section in panel.sections: + if ( + action_id is None and section.is_action_section + ) or not section.is_visible(context): continue - if panel == obj: - name = _value_for_locale(panel["name"]) - display_header(f"\n{'='*40}\n>>>> {name}\n{'='*40}") - else: - name = _value_for_locale(section["name"]) - if name: - display_header(f"\n# {name}") - elif section: + if interactive and verbose and section.name: + Moulinette.display(colorize(f"\n# {section.name}", "purple")) + # filter action section options in case of multiple buttons - section["options"] = [ + options = [ option - for option in section["options"] - if option.get("type", OptionType.string) != OptionType.button - or option["id"] == action + for option in section.options + if option.type is not OptionType.button or option.id == action_id ] - if panel == obj: - continue - - # Check and ask unanswered questions - prefilled_answers = self.args.copy() - prefilled_answers.update(self.new_values) - - questions = ask_questions_and_parse_answers( - {question["id"]: question for question in section["options"]}, - prefilled_answers=prefilled_answers, - current_values=self.values, - hooks=self.hooks, - ) - self.new_values.update( - { - question.id: question.value - for question in questions - if not question.readonly and question.value is not None - } - ) - - @property - def future_values(self): - return {**self.values, **self.new_values} - - def __getattr__(self, name): - if "new_values" in self.__dict__ and name in self.new_values: - return self.new_values[name] - - if "values" in self.__dict__ and name in self.values: - return self.values[name] - - return self.__dict__[name] - - def _parse_pre_answered(self, args, value, args_file): - args = urllib.parse.parse_qs(args or "", keep_blank_values=True) - self.args = {key: ",".join(value_) for key, value_ in args.items()} - - if args_file: - # Import YAML / JSON file but keep --args values - self.args = {**read_yaml(args_file), **self.args} + form = prompt_or_validate_form( + options, + form, + prefilled_answers=prefilled_answers, + context=context, + hooks=hooks, + ) - if value is not None: - self.args = {self.filter_key.split(".")[-1]: value} + return form - def _apply(self): + def _apply( + self, + form: "FormModel", + config: ConfigPanelModel, + previous_settings: dict[str, Any], + exclude: Union["AbstractSetIntStr", "MappingIntStrAny", None] = None, + ) -> None: + """ + Save settings in yaml file. + If `save_mode` is `"diff"` (which is the default), only values that are + different from their default value will be saved. + """ logger.info("Saving the new configuration...") + dir_path = os.path.dirname(os.path.realpath(self.save_path)) if not os.path.exists(dir_path): mkdir(dir_path, mode=0o700) - values_to_save = self.future_values - if self.save_mode == "diff": - defaults = self._get_default_values() - values_to_save = { - k: v for k, v in values_to_save.items() if defaults.get(k) != v + exclude_defaults = self.save_mode == "diff" + # get settings keys filtered by filter_key + partial_settings_keys = form.__fields__.keys() + # get filtered settings + partial_settings = form.dict(exclude_defaults=exclude_defaults, exclude=exclude) # type: ignore + # get previous settings that we will updated with new settings + current_settings = self.raw_settings.copy() + + if exclude: + current_settings = { + key: value + for key, value in current_settings.items() + if key not in exclude } + for key in partial_settings_keys: + if ( + exclude_defaults + and key not in partial_settings + and key in current_settings + ): + del current_settings[key] + elif key in partial_settings: + current_settings[key] = partial_settings[key] + # Save the settings to the .yaml file - write_to_yaml(self.save_path, values_to_save) + write_to_yaml(self.save_path, current_settings) - def _reload_services(self): + def _run_action(self, form: "FormModel", action_id: str) -> None: + raise NotImplementedError() + + def _reload_services(self) -> None: from yunohost.service import service_reload_or_restart - services_to_reload = set() - for panel, section, obj in self._iterate(["panel", "section", "option"]): - services_to_reload |= set(obj.get("services", [])) + services_to_reload = self.config.services if self.config else [] - services_to_reload = list(services_to_reload) - services_to_reload.sort(key="nginx".__eq__) if services_to_reload: logger.info("Reloading services...") for service in services_to_reload: if hasattr(self, "entity"): service = service.replace("__APP__", self.entity) service_reload_or_restart(service) - - def _iterate(self, trigger=["option"]): - for panel in self.config.get("panels", []): - if "panel" in trigger: - yield (panel, None, panel) - for section in panel.get("sections", []): - if "section" in trigger: - yield (panel, section, section) - if "option" in trigger: - for option in section.get("options", []): - yield (panel, section, option) diff --git a/src/utils/form.py b/src/utils/form.py index 06bb38c557..93e5a59eb0 100644 --- a/src/utils/form.py +++ b/src/utils/form.py @@ -17,6 +17,7 @@ # along with this program. If not, see . # import ast +import datetime import operator as op import os import re @@ -24,19 +25,46 @@ import tempfile import urllib.parse from enum import Enum -from typing import Any, Callable, Dict, List, Literal, Mapping, Optional, Union +from logging import getLogger +from typing import ( + TYPE_CHECKING, + cast, + overload, + Annotated, + Any, + Callable, + Iterable, + Literal, + Mapping, + Type, + Union, +) + +from pydantic import ( + BaseModel, + Extra, + ValidationError, + create_model, + validator, + root_validator, +) +from pydantic.color import Color +from pydantic.fields import Field +from pydantic.networks import EmailStr, HttpUrl +from pydantic.types import constr from moulinette import Moulinette, m18n from moulinette.interfaces.cli import colorize -from moulinette.utils.filesystem import read_file, write_to_file -from moulinette.utils.log import getActionLogger +from moulinette.utils.filesystem import read_yaml, write_to_file from yunohost.log import OperationLogger from yunohost.utils.error import YunohostError, YunohostValidationError from yunohost.utils.i18n import _value_for_locale -logger = getActionLogger("yunohost.form") +if TYPE_CHECKING: + from pydantic.fields import ModelField, FieldInfo + +logger = getLogger("yunohost.form") -Context = dict[str, Any] # ╭───────────────────────────────────────────────────────╮ # │ ┌─╴╷ ╷╭─┐╷ │ @@ -93,11 +121,11 @@ def evaluate_simple_ast(node, context=None): ): # left = evaluate_simple_ast(node.left, context) right = evaluate_simple_ast(node.right, context) - if type(node.op) == ast.Add: + if type(node.op) is ast.Add: if isinstance(left, str) or isinstance(right, str): # support 'I am ' + 42 left = str(left) right = str(right) - elif type(left) != type(right): # support "111" - "1" -> 110 + elif type(left) is type(right): # support "111" - "1" -> 110 left = float(left) right = float(right) @@ -117,7 +145,7 @@ def evaluate_simple_ast(node, context=None): left = float(left) right = float(right) except ValueError: - return type(operator) == ast.NotEq + return type(operator) is ast.NotEq try: return operators[type(operator)](left, right) except TypeError: # support "e" > 1 -> False like in JS @@ -181,7 +209,7 @@ def js_to_python(expr): return py_expr -def evaluate_simple_js_expression(expr, context={}): +def evaluate_simple_js_expression(expr: str, context: dict[str, Any] = {}) -> bool: if not expr.strip(): return False node = ast.parse(js_to_python(expr), mode="eval").body @@ -231,6 +259,12 @@ class OptionType(str, Enum): group = "group" +READONLY_TYPES = { + OptionType.display_text, + OptionType.markdown, + OptionType.alert, + OptionType.button, +} FORBIDDEN_READONLY_TYPES = { OptionType.password, OptionType.app, @@ -239,28 +273,152 @@ class OptionType(str, Enum): OptionType.group, } +# To simplify AppConfigPanel bash scripts, we've chosen to use question +# short_ids as global variables. The consequence is that there is a risk +# of collision with other variables, notably different global variables +# used to expose old values or the type of a question... +# In addition to conflicts with bash variables, there is a direct +# conflict with the TOML properties of sections, so the keywords `name`, +# `visible`, `services`, `optional` and `help` cannot be used either. +FORBIDDEN_KEYWORDS = { + "old", + "app", + "changed", + "file_hash", + "binds", + "types", + "formats", + "getter", + "setter", + "short_setting", + "type", + "bind", + "nothing_changed", + "changes_validated", + "result", + "max_progression", + "name", + "visible", + "services", + "optional", + "help", +} -class BaseOption: - def __init__( - self, - question: Dict[str, Any], - ): - self.id = question["id"] - self.type = question.get("type", OptionType.string) - self.visible = question.get("visible", True) - - self.readonly = question.get("readonly", False) - if self.readonly and self.type in FORBIDDEN_READONLY_TYPES: - # FIXME i18n - raise YunohostError( - "config_forbidden_readonly_type", - type=self.type, - id=self.id, - ) +Context = dict[str, Any] +Translation = Union[dict[str, str], str] +JSExpression = str +Values = dict[str, Any] +Mode = Literal["python", "bash"] + + +class Pattern(BaseModel): + regexp: str + error: Translation = "pydantic.value_error.str.regex" # FIXME add generic i18n key + + +class BaseOption(BaseModel): + """ + Options are fields declaration that renders as form items, button, alert or text in the web-admin and printed or prompted in CLI. + They are used in app manifests to declare the before installation form and in config panels. + + [Have a look at the app config panel doc](/packaging_apps_config_panels) for details about Panels and Sections. + + ! IMPORTANT: as for Panels and Sections you have to choose an id, but this one should be unique in all this document, even if the question is in an other panel. + + #### Example + + ```toml + [section.my_option_id] + type = "string" + # ask as `str` + ask = "The text in english" + # ask as `dict` + ask.en = "The text in english" + ask.fr = "Le texte en français" + # advanced props + visible = "my_other_option_id != 'success'" + readonly = true + # much advanced: config panel only? + bind = "null" + ``` + + #### Properties + + - `type`: the actual type of the option, such as 'markdown', 'password', 'number', 'email', ... + - `ask`: `Translation` (default to the option's `id` if not defined): + - text to display as the option's label for inputs or text to display for readonly options + - in config panels, questions are displayed on the left side and therefore have not much space to be rendered. Therefore, it is better to use a short question, and use the `help` property to provide additional details if necessary. + - `visible` (optional): `bool` or `JSExpression` (default: `true`) + - define if the option is diplayed/asked + - if `false` and used alongside `readonly = true`, you get a context only value that can still be used in `JSExpression`s + - `readonly` (optional): `bool` (default: `false`, forced to `true` for readonly types): + - If `true` for input types: forbid mutation of its value + - `bind` (optional): `Binding`, config panels only! A powerful feature that let you configure how and where the setting will be read, validated and written + - if not specified, the value will be read/written in the app `settings.yml` + - if `"null"`: + - the value will not be stored at all (can still be used in context evaluations) + - if in `scripts/config` there's a function named: + - `get__my_option_id`: the value will be gathered from this custom getter + - `set__my_option_id`: the value will be passed to this custom setter where you can do whatever you want with the value + - `validate__my_option_id`: the value will be passed to this custom validator before any custom setter + - if `bind` is a file path: + - if the path starts with `:`, the value be saved as its id's variable/property counterpart + - this only works for first level variables/properties and simple types (no array) + - else the value will be stored as the whole content of the file + - you can use `__FINALPATH__` or `__INSTALL_DIR__` in your path to point to dynamic install paths + - FIXME are other global variables accessible? + - [refer to `bind` doc for explaination and examples](#read-and-write-values-the) + """ + + type: OptionType + id: str + mode: Mode = "bash" # TODO use "python" as default mode with AppConfigPanel setuping it to "bash" + ask: Union[Translation, None] + readonly: bool = False + visible: Union[JSExpression, bool] = True + bind: Union[str, None] = None + name: Union[str, None] = None # LEGACY (replaced by `id`) + + class Config: + arbitrary_types_allowed = True + use_enum_values = True + validate_assignment = True + extra = Extra.forbid + + @staticmethod + def schema_extra(schema: dict[str, Any]) -> None: + del schema["properties"]["id"] + del schema["properties"]["name"] + schema["required"] = [ + required for required in schema.get("required", []) if required != "id" + ] + if not schema["required"]: + del schema["required"] - self.ask = question.get("ask", self.id) - if not isinstance(self.ask, dict): - self.ask = {"en": self.ask} + @validator("id", pre=True) + def check_id_is_not_forbidden(cls, value: str) -> str: + if value in FORBIDDEN_KEYWORDS: + raise ValueError(m18n.n("config_forbidden_keyword", keyword=value)) + return value + + # FIXME Legacy, is `name` still needed? + @validator("name") + def apply_legacy_name(cls, value: Union[str, None], values: Values) -> str: + if value is None: + return values["id"] + return value + + @validator("readonly", pre=True) + def can_be_readonly(cls, value: bool, values: Values) -> bool: + if value is True and values["type"] in FORBIDDEN_READONLY_TYPES: + raise ValueError( + m18n.n( + "config_forbidden_readonly_type", + type=values["type"], + id=values["id"], + ) + ) + return value def is_visible(self, context: Context) -> bool: if isinstance(self.visible, bool): @@ -268,8 +426,10 @@ def is_visible(self, context: Context) -> bool: return evaluate_simple_js_expression(self.visible, context=context) - def _get_prompt_message(self) -> str: - return _value_for_locale(self.ask) + def _get_prompt_message(self, value: None) -> str: + # force type to str + # `OptionsModel.translate_options()` should have been called before calling this method + return cast(str, self.ask) # ╭───────────────────────────────────────────────────────╮ @@ -278,51 +438,131 @@ def _get_prompt_message(self) -> str: class BaseReadonlyOption(BaseOption): - def __init__(self, question): - super().__init__(question) - self.readonly = True + readonly: Literal[True] = True class DisplayTextOption(BaseReadonlyOption): + """ + Display simple multi-line content. + + #### Example + + ```toml + [section.my_option_id] + type = "display_text" + ask = "Simple text rendered as is." + ``` + """ + type: Literal[OptionType.display_text] = OptionType.display_text class MarkdownOption(BaseReadonlyOption): + """ + Display markdown multi-line content. + Markdown is currently only rendered in the web-admin + + #### Example + ```toml + [section.my_option_id] + type = "display_text" + ask = "Text **rendered** in markdown." + ``` + """ + type: Literal[OptionType.markdown] = OptionType.markdown +class State(str, Enum): + success = "success" + info = "info" + warning = "warning" + danger = "danger" + + class AlertOption(BaseReadonlyOption): + """ + Alerts displays a important message with a level of severity. + You can use markdown in `ask` but will only be rendered in the web-admin. + + #### Example + + ```toml + [section.my_option_id] + type = "alert" + ask = "The configuration seems to be manually modified..." + style = "warning" + icon = "warning" + ``` + #### Properties + + - [common properties](#common-properties) + - `style`: any of `"success|info|warning|danger"` (default: `"info"`) + - `icon` (optional): any icon name from [Fork Awesome](https://forkaweso.me/Fork-Awesome/icons/) + - Currently only displayed in the web-admin + """ + type: Literal[OptionType.alert] = OptionType.alert + style: State = State.info + icon: Union[str, None] = None + + def _get_prompt_message(self, value: None) -> str: + colors = { + State.success: "green", + State.info: "cyan", + State.warning: "yellow", + State.danger: "red", + } + message = m18n.g(self.style) if self.style != State.danger else m18n.n("danger") + return f"{colorize(message, colors[self.style])} {self.ask}" - def __init__(self, question): - super().__init__(question) - self.style = question.get("style", "info") - def _get_prompt_message(self) -> str: - text = _value_for_locale(self.ask) +class ButtonOption(BaseReadonlyOption): + """ + Triggers actions. + Available only in config panels. + Renders as a `button` in the web-admin and can be called with `yunohost [app|domain|settings] action run ` in CLI. + + Every options defined in an action section (a config panel section with at least one `button`) is guaranted to be shown/asked to the user and available in `scripts/config`'s scope. + [check examples in advanced use cases](#actions). + + #### Example + + ```toml + [section.my_option_id] + type = "button" + ask = "Break the system" + style = "danger" + icon = "bug" + # enabled only if another option's value (a `boolean` for example) is positive + enabled = "aknowledged" + ``` + + To be able to trigger an action we have to add a bash function starting with `run__` in your `scripts/config` + + ```bash + run__my_action_id() { + ynh_print_info "Running 'my_action_id' action" + } + ``` - if self.style in ["success", "info", "warning", "danger"]: - color = { - "success": "green", - "info": "cyan", - "warning": "yellow", - "danger": "red", - } - prompt = m18n.g(self.style) if self.style != "danger" else m18n.n("danger") - return colorize(prompt, color[self.style]) + f" {text}" - else: - return text + #### Properties + - [common properties](#common-properties) + - `bind`: forced to `"null"` + - `style`: any of `"success|info|warning|danger"` (default: `"success"`) + - `enabled`: `JSExpression` or `bool` (default: `true`) + - when used with `JSExpression` you can enable/disable the button depending on context + - `icon` (optional): any icon name from [Fork Awesome](https://forkaweso.me/Fork-Awesome/icons/) + - Currently only displayed in the web-admin + """ -class ButtonOption(BaseReadonlyOption): type: Literal[OptionType.button] = OptionType.button - enabled = True - - def __init__(self, question): - super().__init__(question) - self.help = question.get("help") - self.style = question.get("style", "success") - self.enabled = question.get("enabled", True) + bind: Literal["null"] = "null" + help: Union[Translation, None] = None + style: State = State.success + icon: Union[str, None] = None + enabled: Union[JSExpression, bool] = True def is_enabled(self, context: Context) -> bool: if isinstance(self.enabled, bool): @@ -337,29 +577,54 @@ def is_enabled(self, context: Context) -> bool: class BaseInputOption(BaseOption): - hide_user_input_in_prompt = False - pattern: Optional[Dict] = None - - def __init__(self, question: Dict[str, Any]): - super().__init__(question) - self.default = question.get("default", None) - self.optional = question.get("optional", False) - self.pattern = question.get("pattern", self.pattern) - self.help = question.get("help") - self.redact = question.get("redact", False) - # .current_value is the currently stored value - self.current_value = question.get("current_value") - # .value is the "proposed" value which we got from the user - self.value = question.get("value") - # Use to return several values in case answer is in mutipart - self.values: Dict[str, Any] = {} - - # Empty value is parsed as empty string - if self.default == "": - self.default = None + """ + Rest of the option types available are considered `inputs`. + + #### Example + + ```toml + [section.my_option_id] + type = "string" + # …any common props… + + optional = false + redact = false + default = "some default string" + help = "You can enter almost anything!" + example = "an example string" + placeholder = "write something…" + ``` + + #### Properties + + - [common properties](#common-properties) + - `optional`: `bool` (default: `false`, but `true` in config panels) + - `redact`: `bool` (default: `false`), to redact the value in the logs when the value contain private information + - `default`: depends on `type`, the default value to assign to the option + - in case of readonly values, you can use this `default` to assign a value (or return a dynamic `default` from a custom getter) + - `help` (optional): `Translation`, to display a short help message in cli and web-admin + - `example` (optional): `str`, to display an example value in web-admin only + - `placeholder` (optional): `str`, shown in the web-admin fields only + """ + + help: Union[Translation, None] = None + example: Union[str, None] = None + placeholder: Union[str, None] = None + redact: bool = False + optional: bool = False # FIXME keep required as default? + default: Any = None + _annotation: Any = Any + _none_as_empty_str: bool = True + + @validator("default", pre=True) + def check_empty_default(value: Any) -> Any: + if value == "": + return None + return value @staticmethod - def humanize(value, option={}): + def humanize(value: Any, option={}) -> str: + if value is None: + return "" return str(value) @staticmethod @@ -368,40 +633,97 @@ def normalize(value, option={}): value = value.strip() return value - def _get_prompt_message(self) -> str: - message = super()._get_prompt_message() + @property + def _dynamic_annotation(self) -> Any: + """ + Returns the expected type of an Option's value. + This may be dynamic based on constraints. + """ + return self._annotation + + @property + def _validators(self) -> dict[str, Callable]: + return { + "pre": self._value_pre_validator, + "post": self._value_post_validator, + } + + def _get_field_attrs(self) -> dict[str, Any]: + """ + Returns attributes to build a `pydantic.Field`. + This may contains non `Field` attrs that will end up in `Field.extra`. + Those extra can be used as constraints in custom validators and ends up + in the JSON Schema. + """ + # TODO + # - help + # - placeholder + attrs: dict[str, Any] = { + "redact": self.redact, # extra + "none_as_empty_str": self._none_as_empty_str, + } + + if self.readonly: + attrs["allow_mutation"] = False + + if self.example: + attrs["examples"] = [self.example] + + if self.default is not None: + attrs["default"] = self.default + else: + attrs["default"] = ... if not self.optional else None + + return attrs + + def _as_dynamic_model_field(self) -> tuple[Any, "FieldInfo"]: + """ + Return a tuple of a type and a Field instance to be injected in a + custom form declaration. + """ + attrs = self._get_field_attrs() + anno = ( + self._dynamic_annotation + if not self.optional + else Union[self._dynamic_annotation, None] + ) + field = Field(default=attrs.pop("default", None), **attrs) + + return (anno, field) + + def _get_prompt_message(self, value: Any) -> str: + message = super()._get_prompt_message(value) if self.readonly: message = colorize(message, "purple") - return f"{message} {self.humanize(self.current_value)}" + return f"{message} {self.humanize(value, self)}" return message - def _value_pre_validator(self): - if self.value in [None, ""] and not self.optional: - raise YunohostValidationError("app_argument_required", name=self.id) + @classmethod + def _value_pre_validator(cls, value: Any, field: "ModelField") -> Any: + if value == "": + return None + + return value - # we have an answer, do some post checks - if self.value not in [None, ""]: - if self.pattern and not re.match(self.pattern["regexp"], str(self.value)): - raise YunohostValidationError( - self.pattern["error"], - name=self.id, - value=self.value, - ) + @classmethod + def _value_post_validator(cls, value: Any, field: "ModelField") -> Any: + extras = field.field_info.extra - def _value_post_validator(self): - if not self.redact: - return self.value + if value is None and extras["none_as_empty_str"]: + value = "" + + if not extras.get("redact"): + return value # Tell the operation_logger to redact all password-type / secret args # Also redact the % escaped version of the password that might appear in # the 'args' section of metadata (relevant for password with non-alphanumeric char) data_to_redact = [] - if self.value and isinstance(self.value, str): - data_to_redact.append(self.value) - if self.current_value and isinstance(self.current_value, str): - data_to_redact.append(self.current_value) + if value and isinstance(value, str): + data_to_redact.append(value) + data_to_redact += [ urllib.parse.quote(data) for data in data_to_redact @@ -411,76 +733,209 @@ def _value_post_validator(self): for operation_logger in OperationLogger._instances: operation_logger.data_to_redact.extend(data_to_redact) - return self.value + return value # ─ STRINGS ─────────────────────────────────────────────── class BaseStringOption(BaseInputOption): - default_value = "" + default: Union[str, None] + pattern: Union[Pattern, None] = None + _annotation = str + + @property + def _dynamic_annotation(self) -> Type[str]: + if self.pattern: + return constr(regex=self.pattern.regexp) + + return self._annotation + + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + + if self.pattern: + attrs["regex_error"] = self.pattern.error # extra + + return attrs class StringOption(BaseStringOption): + r""" + Ask for a simple string. + + #### Example + ```toml + [section.my_option_id] + type = "string" + default = "E10" + pattern.regexp = '^[A-F]\d\d$' + pattern.error = "Provide a room like F12 : one uppercase and 2 numbers" + ``` + + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.string] = OptionType.string class TextOption(BaseStringOption): + """ + Ask for a multiline string. + Renders as a `textarea` in the web-admin and by opening a text editor on the CLI. + + #### Example + ```toml + [section.my_option_id] + type = "text" + default = "multi\\nline\\ncontent" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.text] = OptionType.text +FORBIDDEN_PASSWORD_CHARS = r"{}" + + class PasswordOption(BaseInputOption): + """ + Ask for a password. + The password is tested as a regular user password (at least 8 chars) + + #### Example + ```toml + [section.my_option_id] + type = "password" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: forced to `""` + - `redact`: forced to `true` + - `example`: forbidden + """ + type: Literal[OptionType.password] = OptionType.password - hide_user_input_in_prompt = True - default_value = "" - forbidden_chars = "{}" + example: Literal[None] = None + default: Literal[None] = None + redact: Literal[True] = True + _annotation = str + _forbidden_chars: str = FORBIDDEN_PASSWORD_CHARS - def __init__(self, question): - super().__init__(question) - self.redact = True - if self.default is not None: - raise YunohostValidationError( - "app_argument_password_no_default", name=self.id - ) + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + + attrs["forbidden_chars"] = self._forbidden_chars # extra - def _value_pre_validator(self): - super()._value_pre_validator() + return attrs - if self.value not in [None, ""]: - if any(char in self.value for char in self.forbidden_chars): + @classmethod + def _value_pre_validator( + cls, value: Union[str, None], field: "ModelField" + ) -> Union[str, None]: + value = super()._value_pre_validator(value, field) + + if value is not None and value != "": + forbidden_chars: str = field.field_info.extra["forbidden_chars"] + if any(char in value for char in forbidden_chars): raise YunohostValidationError( - "pattern_password_app", forbidden_chars=self.forbidden_chars + "pattern_password_app", forbidden_chars=forbidden_chars ) # If it's an optional argument the value should be empty or strong enough from yunohost.utils.password import assert_password_is_strong_enough - assert_password_is_strong_enough("user", self.value) + assert_password_is_strong_enough("user", value) + + return value -class ColorOption(BaseStringOption): +class ColorOption(BaseInputOption): + """ + Ask for a color represented as a hex value (with possibly an alpha channel). + Renders as color picker in the web-admin and as a prompt that accept named color like `yellow` in CLI. + + #### Example + ```toml + [section.my_option_id] + type = "color" + default = "#ff0" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ + type: Literal[OptionType.color] = OptionType.color - pattern = { - "regexp": r"^#[ABCDEFabcdef\d]{3,6}$", - "error": "config_validate_color", # i18n: config_validate_color - } + default: Union[str, None] + _annotation = Color + + @staticmethod + def humanize(value: Union[Color, str, None], option={}) -> str: + if isinstance(value, Color): + value.as_named(fallback=True) + + return super(ColorOption, ColorOption).humanize(value, option) + + @staticmethod + def normalize(value: Union[Color, str, None], option={}) -> str: + if isinstance(value, Color): + return value.as_hex() + + return super(ColorOption, ColorOption).normalize(value, option) + + @classmethod + def _value_post_validator( + cls, value: Union[Color, None], field: "ModelField" + ) -> Union[str, None]: + if isinstance(value, Color): + return value.as_hex() + + return super()._value_post_validator(value, field) # ─ NUMERIC ─────────────────────────────────────────────── class NumberOption(BaseInputOption): - type: Literal[OptionType.number, OptionType.range] = OptionType.number - default_value = None + """ + Ask for a number (an integer). + + #### Example + ```toml + [section.my_option_id] + type = "number" + default = 100 + min = 50 + max = 200 + step = 5 + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `type`: `number` or `range` (input or slider in the web-admin) + - `min` (optional): minimal int value inclusive + - `max` (optional): maximal int value inclusive + - `step` (optional): currently only used in the webadmin as the `` step jump + """ - def __init__(self, question): - super().__init__(question) - self.min = question.get("min", None) - self.max = question.get("max", None) - self.step = question.get("step", None) + # `number` and `range` are exactly the same, but `range` does render as a slider in web-admin + type: Literal[OptionType.number, OptionType.range] = OptionType.number + default: Union[int, None] + min: Union[int, None] = None + max: Union[int, None] = None + step: Union[int, None] = None + _annotation = int + _none_as_empty_str = False @staticmethod - def normalize(value, option={}): + def normalize(value, option={}) -> Union[int, None]: if isinstance(value, int): return value @@ -493,52 +948,70 @@ def normalize(value, option={}): if value in [None, ""]: return None - option = option.__dict__ if isinstance(option, BaseOption) else option + option = option.dict() if isinstance(option, BaseOption) else option raise YunohostValidationError( "app_argument_invalid", name=option.get("id"), error=m18n.n("invalid_number"), ) - def _value_pre_validator(self): - super()._value_pre_validator() - if self.value in [None, ""]: - return + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + attrs["ge"] = self.min + attrs["le"] = self.max + attrs["step"] = self.step # extra - if self.min is not None and int(self.value) < self.min: - raise YunohostValidationError( - "app_argument_invalid", - name=self.id, - error=m18n.n("invalid_number_min", min=self.min), - ) + return attrs - if self.max is not None and int(self.value) > self.max: - raise YunohostValidationError( - "app_argument_invalid", - name=self.id, - error=m18n.n("invalid_number_max", max=self.max), - ) + @classmethod + def _value_pre_validator( + cls, value: Union[int, None], field: "ModelField" + ) -> Union[int, None]: + value = super()._value_pre_validator(value, field) + + if value is None: + return None + + return value # ─ BOOLEAN ─────────────────────────────────────────────── class BooleanOption(BaseInputOption): - type: Literal[OptionType.boolean] = OptionType.boolean - default_value = 0 - yes_answers = ["1", "yes", "y", "true", "t", "on"] - no_answers = ["0", "no", "n", "false", "f", "off"] + """ + Ask for a boolean. + Renders as a switch in the web-admin and a yes/no prompt in CLI. + + #### Example + ```toml + [section.my_option_id] + type = "boolean" + default = 1 + yes = "agree" + no = "disagree" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `0` + - `yes` (optional): (default: `1`) define as what the thruthy value should output + - can be `true`, `True`, `"yes"`, etc. + - `no` (optional): (default: `0`) define as what the thruthy value should output + - can be `0`, `"false"`, `"n"`, etc. + """ - def __init__(self, question): - super().__init__(question) - self.yes = question.get("yes", 1) - self.no = question.get("no", 0) - if self.default is None: - self.default = self.no + type: Literal[OptionType.boolean] = OptionType.boolean + yes: Any = 1 + no: Any = 0 + default: Union[bool, int, str, None] = 0 + _annotation = Union[bool, int, str] + _yes_answers: set[str] = {"1", "yes", "y", "true", "t", "on"} + _no_answers: set[str] = {"0", "no", "n", "false", "f", "off"} + _none_as_empty_str = False @staticmethod - def humanize(value, option={}): - option = option.__dict__ if isinstance(option, BaseOption) else option + def humanize(value, option={}) -> str: + option = option.dict() if isinstance(option, BaseOption) else option yes = option.get("yes", 1) no = option.get("no", 0) @@ -560,8 +1033,8 @@ def humanize(value, option={}): ) @staticmethod - def normalize(value, option={}): - option = option.__dict__ if isinstance(option, BaseOption) else option + def normalize(value, option={}) -> Any: + option = option.dict() if isinstance(option, BaseOption) else option if isinstance(value, str): value = value.strip() @@ -569,8 +1042,8 @@ def normalize(value, option={}): technical_yes = option.get("yes", 1) technical_no = option.get("no", 0) - no_answers = BooleanOption.no_answers - yes_answers = BooleanOption.yes_answers + no_answers = BooleanOption._no_answers + yes_answers = BooleanOption._yes_answers assert ( str(technical_yes).lower() not in no_answers @@ -579,8 +1052,8 @@ def normalize(value, option={}): str(technical_no).lower() not in yes_answers ), f"'no' value can't be in {yes_answers}" - no_answers += [str(technical_no).lower()] - yes_answers += [str(technical_yes).lower()] + no_answers.add(str(technical_no).lower()) + yes_answers.add(str(technical_yes).lower()) strvalue = str(value).lower() @@ -602,63 +1075,145 @@ def normalize(value, option={}): def get(self, key, default=None): return getattr(self, key, default) - def _get_prompt_message(self): - message = super()._get_prompt_message() + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() + attrs["parse"] = { # extra + True: self.yes, + False: self.no, + } + return attrs + + def _get_prompt_message(self, value: Union[bool, None]) -> str: + message = super()._get_prompt_message(value) if not self.readonly: message += " [yes | no]" return message + @classmethod + def _value_post_validator( + cls, value: Union[bool, None], field: "ModelField" + ) -> Any: + if isinstance(value, bool): + return field.field_info.extra["parse"][value] + + return super()._value_post_validator(value, field) + # ─ TIME ────────────────────────────────────────────────── -class DateOption(BaseStringOption): +class DateOption(BaseInputOption): + """ + Ask for a date in the form `"2025-06-14"`. + Renders as a date-picker in the web-admin and a regular prompt in CLI. + + Can also take a timestamp as value that will output as an ISO date string. + + #### Example + ```toml + [section.my_option_id] + type = "date" + default = "2070-12-31" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ + type: Literal[OptionType.date] = OptionType.date - pattern = { - "regexp": r"^\d{4}-\d\d-\d\d$", - "error": "config_validate_date", # i18n: config_validate_date - } + default: Union[str, None] + _annotation = datetime.date - def _value_pre_validator(self): - from datetime import datetime + @classmethod + def _value_post_validator( + cls, value: Union[datetime.date, None], field: "ModelField" + ) -> Union[str, None]: + if isinstance(value, datetime.date): + return value.isoformat() - super()._value_pre_validator() + return super()._value_post_validator(value, field) - if self.value not in [None, ""]: - try: - datetime.strptime(self.value, "%Y-%m-%d") - except ValueError: - raise YunohostValidationError("config_validate_date") +class TimeOption(BaseInputOption): + """ + Ask for an hour in the form `"22:35"`. + Renders as a date-picker in the web-admin and a regular prompt in CLI. + + #### Example + ```toml + [section.my_option_id] + type = "time" + default = "12:26" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ -class TimeOption(BaseStringOption): type: Literal[OptionType.time] = OptionType.time - pattern = { - "regexp": r"^(?:\d|[01]\d|2[0-3]):[0-5]\d$", - "error": "config_validate_time", # i18n: config_validate_time - } + default: Union[str, int, None] + _annotation = datetime.time + + @classmethod + def _value_post_validator( + cls, value: Union[datetime.date, None], field: "ModelField" + ) -> Union[str, None]: + if isinstance(value, datetime.time): + # FIXME could use `value.isoformat()` to get `%H:%M:%S` + return value.strftime("%H:%M") + + return super()._value_post_validator(value, field) # ─ LOCATIONS ───────────────────────────────────────────── -class EmailOption(BaseStringOption): +class EmailOption(BaseInputOption): + """ + Ask for an email. Validation made with [python-email-validator](https://github.com/JoshData/python-email-validator) + + #### Example + ```toml + [section.my_option_id] + type = "email" + default = "Abc.123@test-example.com" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + """ + type: Literal[OptionType.email] = OptionType.email - pattern = { - "regexp": r"^.+@.+", - "error": "config_validate_email", # i18n: config_validate_email - } + default: Union[EmailStr, None] + _annotation = EmailStr -class WebPathOption(BaseInputOption): +class WebPathOption(BaseStringOption): + """ + Ask for an web path (the part of an url after the domain). Used by default in app install to define from where the app will be accessible. + + #### Example + ```toml + [section.my_option_id] + type = "path" + default = "/" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.path] = OptionType.path - default_value = "" @staticmethod - def normalize(value, option={}): - option = option.__dict__ if isinstance(option, BaseOption) else option + def normalize(value, option={}) -> str: + option = option.dict() if isinstance(option, BaseOption) else option + + if value is None: + value = "" if not isinstance(value, str): raise YunohostValidationError( @@ -684,102 +1239,206 @@ def normalize(value, option={}): class URLOption(BaseStringOption): + """ + Ask for any url. + + #### Example + ```toml + [section.my_option_id] + type = "url" + default = "https://example.xn--zfr164b/@handle/" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `pattern` (optional): `Pattern`, a regex to match the value against + """ + type: Literal[OptionType.url] = OptionType.url - pattern = { - "regexp": r"^https?://.*$", - "error": "config_validate_url", # i18n: config_validate_url - } + _annotation = HttpUrl + + @classmethod + def _value_post_validator( + cls, value: Union[HttpUrl, None], field: "ModelField" + ) -> Union[str, None]: + if isinstance(value, HttpUrl): + return str(value) + return super()._value_post_validator(value, field) # ─ FILE ────────────────────────────────────────────────── class FileOption(BaseInputOption): + r""" + Ask for file. + Renders a file prompt in the web-admin and ask for a path in CLI. + + #### Example + ```toml + [section.my_option_id] + type = "file" + accept = ".json" + # bind the file to a location to save the file there + bind = "/tmp/my_file.json" + ``` + #### Properties + - [common inputs properties](#common-inputs-properties) + - `default`: `""` + - `accept`: a comma separated list of extension to accept like `".conf, .ini` + - /!\ currently only work on the web-admin + """ + type: Literal[OptionType.file] = OptionType.file - upload_dirs: List[str] = [] + # `FilePath` for CLI (path must exists and must be a file) + # `bytes` for API (a base64 encoded file actually) + accept: Union[list[str], None] = None # currently only used by the web-admin + default: Union[str, None] + _annotation = str # TODO could be Path at some point + _upload_dirs: set[str] = set() + + @property + def _validators(self) -> dict[str, Callable]: + return { + "pre": self._value_pre_validator, + "post": ( + self._bash_value_post_validator + if self.mode == "bash" + else self._python_value_post_validator + ), + } + + def _get_field_attrs(self) -> dict[str, Any]: + attrs = super()._get_field_attrs() - def __init__(self, question): - super().__init__(question) - self.accept = question.get("accept", "") + if self.accept: + attrs["accept"] = self.accept # extra + + attrs["bind"] = self.bind + + return attrs @classmethod - def clean_upload_dirs(cls): + def clean_upload_dirs(cls) -> None: # Delete files uploaded from API - for upload_dir in cls.upload_dirs: + for upload_dir in cls._upload_dirs: if os.path.exists(upload_dir): shutil.rmtree(upload_dir) - def _value_pre_validator(self): - if self.value is None: - self.value = self.current_value + @classmethod + def _base_value_post_validator( + cls, value: Any, field: "ModelField" + ) -> tuple[bytes, str | None]: + import mimetypes + from pathlib import Path + from magic import Magic + from base64 import b64decode - super()._value_pre_validator() + if Moulinette.interface.type != "api": + path = Path(value) + if not (path.exists() and path.is_absolute() and path.is_file()): + raise YunohostValidationError("File doesn't exists", raw_msg=True) + content = path.read_bytes() + else: + content = b64decode(value) - # Validation should have already failed if required - if self.value in [None, ""]: - return self.value + accept_list = field.field_info.extra.get("accept") + mimetype = Magic(mime=True).from_buffer(content) - if Moulinette.interface.type != "api": - if not os.path.exists(str(self.value)) or not os.path.isfile( - str(self.value) - ): - raise YunohostValidationError( - "app_argument_invalid", - name=self.id, - error=m18n.n("file_does_not_exist", path=str(self.value)), - ) + if accept_list and mimetype not in accept_list: + raise YunohostValidationError( + f"Unsupported file type '{mimetype}', expected a type among '{', '.join(accept_list)}'.", raw_msg=True + ) - def _value_post_validator(self): - from base64 import b64decode + ext = mimetypes.guess_extension(mimetype) - if not self.value: + return content, ext + + @classmethod + def _bash_value_post_validator(cls, value: Any, field: "ModelField") -> str: + """File handling for "bash" config panels (app)""" + if not value: return "" + content, _ = cls._base_value_post_validator(value, field) + upload_dir = tempfile.mkdtemp(prefix="ynh_filequestion_") _, file_path = tempfile.mkstemp(dir=upload_dir) - FileOption.upload_dirs += [upload_dir] + FileOption._upload_dirs.add(upload_dir) - logger.debug(f"Saving file {self.id} for file question into {file_path}") + logger.debug(f"Saving file {field.name} for file question into {file_path}") - def is_file_path(s): - return isinstance(s, str) and s.startswith("/") and os.path.exists(s) + write_to_file(file_path, content, file_mode="wb") - if Moulinette.interface.type != "api" or is_file_path(self.value): - content = read_file(str(self.value), file_mode="rb") - else: - content = b64decode(self.value) + return file_path - write_to_file(file_path, content, file_mode="wb") + @classmethod + def _python_value_post_validator(cls, value: Any, field: "ModelField") -> str: + """File handling for "python" config panels""" - self.value = file_path + from pathlib import Path + import hashlib - return self.value + if not value: + return "" + + bind = field.field_info.extra["bind"] + + # to avoid "filename too long" with b64 content + if len(value.encode("utf-8")) < 255: + # Check if value is an already hashed and saved filepath + path = Path(value) + if path.exists() and value == bind.format( + filename=path.stem, ext=path.suffix + ): + return value + + content, ext = cls._base_value_post_validator(value, field) + + m = hashlib.sha256() + m.update(content) + sha256sum = m.hexdigest() + filename = Path(bind.format(filename=sha256sum, ext=ext)) + filename.write_bytes(content) + + return str(filename) # ─ CHOICES ─────────────────────────────────────────────── class BaseChoicesOption(BaseInputOption): - def __init__( - self, - question: Dict[str, Any], - ): - super().__init__(question) - # Don't restrict choices if there's none specified - self.choices = question.get("choices", None) + # FIXME probably forbid choices to be None? + filter: Union[JSExpression, None] = None # filter before choices + # We do not declare `choices` here to be able to declare other fields before `choices` and acces their values in `choices` validators + # choices: Union[dict[str, Any], list[Any], None] - def _get_prompt_message(self) -> str: - message = super()._get_prompt_message() + @validator("choices", pre=True, check_fields=False) + def parse_comalist_choices(value: Any) -> Union[dict[str, Any], list[Any], None]: + if isinstance(value, str): + values = [value.strip() for value in value.split(",")] + return [value for value in values if value] + return value - if self.readonly: - message = message - choice = self.current_value + @property + def _dynamic_annotation(self) -> Union[object, Type[str]]: + if self.choices is not None: + choices = ( + self.choices if isinstance(self.choices, list) else self.choices.keys() + ) + return Literal[tuple(choices)] + + return self._annotation - if isinstance(self.choices, dict) and choice is not None: - choice = self.choices[choice] + def _get_prompt_message(self, value: Any) -> str: + message = super()._get_prompt_message(value) - return f"{colorize(message, 'purple')} {choice}" + if self.readonly: + if isinstance(self.choices, dict) and value is not None: + value = self.choices[value] + + return f"{colorize(message, 'purple')} {value}" if self.choices: # Prevent displaying a shitload of choices @@ -789,112 +1448,200 @@ def _get_prompt_message(self) -> str: if isinstance(self.choices, dict) else self.choices ) - choices_to_display = choices[:20] + splitted_choices = choices[:20] remaining_choices = len(choices[20:]) if remaining_choices > 0: - choices_to_display += [ + splitted_choices += [ m18n.n("other_available_options", n=remaining_choices) ] - choices_to_display = " | ".join( - str(choice) for choice in choices_to_display - ) + choices_to_display = " | ".join(str(choice) for choice in splitted_choices) return f"{message} [{choices_to_display}]" return message - def _value_pre_validator(self): - super()._value_pre_validator() - - # we have an answer, do some post checks - if self.value not in [None, ""]: - if self.choices and self.value not in self.choices: - raise YunohostValidationError( - "app_argument_choice_invalid", - name=self.id, - value=self.value, - choices=", ".join(str(choice) for choice in self.choices), - ) - class SelectOption(BaseChoicesOption): + """ + Ask for value from a limited set of values. + Renders as a regular `