From 9b25faca743b21d05e4fbe2a620e6abcb94ddbca Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Tue, 24 Sep 2024 09:48:27 +0100 Subject: [PATCH 1/8] Better missing GUI requirements message on macOS Fixes #286 --- emailproxy.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/emailproxy.py b/emailproxy.py index c991c43..d4738b3 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-09-12' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-09-24' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -2601,7 +2601,8 @@ def __init__(self, args=None): if self.args.gui and len(MISSING_GUI_REQUIREMENTS) > 0: Log.error('Unable to load all GUI requirements:', MISSING_GUI_REQUIREMENTS, '- did you mean to run in', - '`--no-gui` mode? If not, please run `python -m pip install -r requirements-gui.txt`') + '`--no-gui` mode? If not, please run `python -m pip install -r requirements-gui.txt` or install', + 'from PyPI with GUI requirements included: `python -m pip install emailproxy[gui]`') self.exit(None) return @@ -2622,6 +2623,9 @@ def __init__(self, args=None): # noinspection PyUnresolvedReferences,PyAttributeOutsideInit def init_platforms(self): if sys.platform == 'darwin' and self.args.gui: + if len(MISSING_GUI_REQUIREMENTS) > 0: + return # skip - we will exit anyway due to missing requirements (with a more helpful error message) + # hide dock icon (but not LSBackgroundOnly as we need input via webview) info = AppKit.NSBundle.mainBundle().infoDictionary() info['LSUIElement'] = '1' @@ -3417,7 +3421,8 @@ def exit(self, icon, restart_callback=None): AppConfig.save() - if sys.platform == 'darwin' and self.args.gui: + # attribute existence check is needed here and below because we may exit before init_platforms() has run + if sys.platform == 'darwin' and self.args.gui and hasattr(self, 'macos_reachability_target'): # noinspection PyUnresolvedReferences SystemConfiguration.SCNetworkReachabilityUnscheduleFromRunLoop(self.macos_reachability_target, SystemConfiguration.CFRunLoopGetCurrent(), @@ -3451,7 +3456,8 @@ def exit(self, icon, restart_callback=None): restart_callback() # macOS Launch Agents need reloading when changed; unloading exits immediately so this must be our final action - if sys.platform == 'darwin' and self.args.gui and self.macos_unload_plist_on_exit: + if sys.platform == 'darwin' and self.args.gui and ( + hasattr(self, 'macos_unload_plist_on_exit') and self.macos_unload_plist_on_exit): self.macos_launchctl('unload') From 9f2b988f8462c82a9b73c3f8f996f6787157a463 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Tue, 24 Sep 2024 13:02:38 +0100 Subject: [PATCH 2/8] Escape square brackets in PyPI GUI variant command See #286 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0eaabb8..0a296a7 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Begin by downloading the proxy via one of the following methods:
  1. Pick a pre-built release for your platform (macOS or Windows; no installation needed); or,
    2. -
  2. Install from PyPI: set up using python -m pip install emailproxy[gui], download the sample emailproxy.config file, then python -m emailproxy to run; or,
    3. +
  3. Install from PyPI: set up using python -m pip install emailproxy\[gui\], download the sample emailproxy.config file, then python -m emailproxy to run; or,
  4. Clone or download (and star :-) the GitHub repository, then: python -m pip install -r requirements-core.txt -r requirements-gui.txt to install requirements, and python emailproxy.py to run.
From e4dbd6490258a42197df690110a66e8edf7be4b1 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Fri, 4 Oct 2024 14:09:37 +0100 Subject: [PATCH 3/8] Minor lint improvements --- emailproxy.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/emailproxy.py b/emailproxy.py index d4738b3..28c8910 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-09-24' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-10-04' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -1153,7 +1153,7 @@ def get_service_account_authorisation_token(key_type, key_path_or_contents, oaut """Requests an authorisation token via a Service Account key (currently Google Cloud only)""" import json try: - import requests + import requests # noqa: F401 - requests is required as the default transport for google-auth import google.oauth2.service_account import google.auth.transport.requests except ModuleNotFoundError as e: @@ -1359,8 +1359,8 @@ def handle_error(self): 'CERTIFICATE_VERIFY_FAILED', 'TLSV1_ALERT_PROTOCOL_VERSION', 'TLSV1_ALERT_UNKNOWN_CA', 'UNSUPPORTED_PROTOCOL', 'record layer failure', APP_PACKAGE] error_type, value = Log.get_last_error() - if error_type == OSError and value.errno == 0 or issubclass(error_type, ssl.SSLError) and \ - any(i in value.args[1] for i in ssl_errors) or error_type == FileNotFoundError: + if error_type is OSError and value.errno == 0 or issubclass(error_type, ssl.SSLError) and \ + any(i in value.args[1] for i in ssl_errors) or error_type is FileNotFoundError: Log.error('Caught connection error in', self.info_string(), ':', error_type, 'with message:', value) if hasattr(self, 'custom_configuration') and hasattr(self, 'proxy_type'): if self.proxy_type == 'SMTP': @@ -1485,7 +1485,7 @@ def handle_close(self): error_type, value = Log.get_last_error() if error_type and value: message = 'Caught connection error (client)' - if error_type == ConnectionResetError: + if error_type is ConnectionResetError: message = '%s [ Are you attempting an encrypted connection to a non-encrypted server? ]' % message Log.info(self.info_string(), message, '-', error_type.__name__, ':', value) self.close() @@ -1943,9 +1943,9 @@ def send(self, byte_data, censor_log=False): def handle_error(self): error_type, value = Log.get_last_error() - if error_type == TimeoutError and value.errno == errno.ETIMEDOUT or \ + if error_type is TimeoutError and value.errno == errno.ETIMEDOUT or \ issubclass(error_type, ConnectionError) and value.errno in [errno.ECONNRESET, errno.ECONNREFUSED] or \ - error_type == OSError and value.errno in [0, errno.ENETDOWN, errno.ENETUNREACH, errno.EHOSTDOWN, + error_type is OSError and value.errno in [0, errno.ENETDOWN, errno.ENETUNREACH, errno.EHOSTDOWN, errno.EHOSTUNREACH]: # TimeoutError 60 = 'Operation timed out'; ConnectionError 54 = 'Connection reset by peer', 61 = 'Connection # refused; OSError 0 = 'Error' (typically network failure), 50 = 'Network is down', 51 = 'Network is @@ -1965,7 +1965,7 @@ def handle_close(self): error_type, value = Log.get_last_error() if error_type and value: message = 'Caught connection error (server)' - if error_type == OSError and value.errno in [errno.ENOTCONN, 10057]: + if error_type is OSError and value.errno in [errno.ENOTCONN, 10057]: # OSError 57 or 10057 = 'Socket is not connected' message = '%s [ Client attempted to send command without waiting for server greeting ]' % message Log.info(self.info_string(), message, '-', error_type.__name__, ':', value) @@ -2382,9 +2382,9 @@ def restart(self): def handle_error(self): error_type, value = Log.get_last_error() if error_type == socket.gaierror and value.errno in [-2, 8, 11001] or \ - error_type == TimeoutError and value.errno == errno.ETIMEDOUT or \ + error_type is TimeoutError and value.errno == errno.ETIMEDOUT or \ issubclass(error_type, ConnectionError) and value.errno in [errno.ECONNRESET, errno.ECONNREFUSED] or \ - error_type == OSError and value.errno in [0, errno.EINVAL, errno.ENETDOWN, errno.EHOSTUNREACH]: + error_type is OSError and value.errno in [0, errno.EINVAL, errno.ENETDOWN, errno.EHOSTUNREACH]: # gaierror -2 or 8 = 'nodename nor servname provided, or not known' / 11001 = 'getaddrinfo failed' (caused # by getpeername() failing due to no connection); TimeoutError 60 = 'Operation timed out'; ConnectionError # 54 = 'Connection reset by peer', 61 = 'Connection refused; OSError 0 = 'Error' (local SSL failure), @@ -2874,7 +2874,7 @@ def create_authorisation_menu(self): else: usernames = [] for request in self.authorisation_requests: - if not request['username'] in usernames: + if request['username'] not in usernames: items.append(pystray.MenuItem(request['username'], self.authorise_account)) usernames.append(request['username']) items.append(pystray.Menu.SEPARATOR) From 31eac06104e5c47daf3a9a3863d1d172f4818710 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 21 Oct 2024 11:21:46 +0100 Subject: [PATCH 4/8] Readme improvements; minor lint fixes --- README.md | 25 ++++++++++++++----------- emailproxy.config | 11 +++++++++-- emailproxy.py | 6 +++++- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0a296a7..af830c3 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The proxy works in the background with a menu bar/taskbar helper or as a headles ### Example use-cases - You need to use an Office 365 email account, but don't get on with Outlook. -The email client you like doesn't support OAuth 2.0, which became mandatory [in January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437) ([September 2025 for SMTP](https://techcommunity.microsoft.com/t5/exchange-team-blog/exchange-online-to-retire-basic-auth-for-client-submission-smtp/ba-p/4114750)). +The email client you like doesn't support OAuth 2.0, which became mandatory [in January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437) ([September 2024 for free Hotmail/Outlook accounts](https://support.microsoft.com/en-us/office/modern-authentication-methods-now-needed-to-continue-syncing-outlook-email-in-non-microsoft-email-apps-c5d65390-9676-4763-b41f-d7986499a90d); [September 2025 for O365 SMTP](https://techcommunity.microsoft.com/t5/exchange-team-blog/exchange-online-to-retire-basic-auth-for-client-submission-smtp/ba-p/4114750)). - You used to use Gmail via IMAP/POP/SMTP with your raw account credentials (i.e., your real password), but cannot do this now that Google has disabled this method, and don't want to use an [App Password](https://support.google.com/accounts/answer/185833) (or cannot enable this option). - You have an account already set up in an email client, and you need to switch it to OAuth 2.0 authentication. You can edit the server details, but the client forces you to delete and re-add the account to enable OAuth 2.0, and you don't want to do this. @@ -32,7 +32,7 @@ Begin by downloading the proxy via one of the following methods: Next, edit the sample `emailproxy.config` file to add configuration details for each email server and account that you want to use with the proxy. -[Guidance and example account configurations](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) are provided for Office 365, Gmail and several other providers, though you will need to insert your own client credentials for each one (see the [client credentials documentation](#oauth-20-client-credentials) for guidance). +[Guidance and example account configurations](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) are provided for Office 365, Gmail and several other providers, though you will need to insert your own client credentials for each one (see the [client credentials documentation](#oauth-20-client-credentials) below for guidance). You can remove details from the sample configuration file for services you don't use, or add additional ones for any other OAuth 2.0-authenticated IMAP/POP/SMTP servers you would like to use with the proxy. You can now start the proxy: depending on which installation option you chose, either launch the application or use the appropriate run command listed above. @@ -71,7 +71,8 @@ The [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/b - Office 365: register a new [Microsoft identity application](https://learn.microsoft.com/entra/identity-platform/quickstart-register-app) - Gmail / Google Workspace: register a [Google API desktop app client](https://developers.google.com/identity/protocols/oauth2/native-app) -- AOL and Yahoo Mail (and subproviders such as AT&T) are not currently allowing new client registrations with the OAuth email scope – the only option here is to reuse the credentials from an existing client that does have this permission. +- Outlook / Hotmail (free accounts): because you are not the administrator for these Microsoft-operated domains, the only option is to reuse an existing client ID – see, for example, [Thunderbird](https://blog.thunderbird.net/2023/01/important-message-for-microsoft-office-365-enterprise-users/), or the links above +- AOL and Yahoo Mail (and subproviders such as AT&T) are not currently allowing new client registrations with the OAuth email scope – the only option here is to reuse the credentials from an existing client that does have this permission The proxy supports [Google Cloud service accounts](https://cloud.google.com/iam/docs/service-account-overview) for access to Google Workspace Gmail. It also supports the [client credentials grant (CCG)](https://learn.microsoft.com/entra/identity-platform/v2-oauth2-client-creds-grant-flow) and [resource owner password credentials grant (ROPCG)](https://learn.microsoft.com/entra/identity-platform/v2-oauth-ropc) OAuth 2.0 flows, and [certificate credentials (JWT)](https://learn.microsoft.com/entra/identity-platform/certificate-credentials). @@ -134,8 +135,8 @@ See the [optional arguments and configuration](#optional-arguments-and-configura If your network requires connections to use an existing proxy, you can instruct the script to use this by setting the [proxy handler](https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler) environment variable `https_proxy` (and/or `http_proxy`) – for example, `https_proxy=localhost python -m emailproxy`. -After installing its requirements, the proxy script can be packaged as a single self-contained executable using [pyinstaller](https://pyinstaller.org/) if desired: `pyinstaller --onefile emailproxy.py`. -If you are using the GUI version of the proxy, you may need to add `--hidden-import timeago.locales.en_short` until [this `timeago` issue](https://github.com/hustcc/timeago/issues/40) is resolved. +After installing its requirements, the proxy script can be packaged as a single self-contained executable using [Nuitka](https://nuitka.net/) (`nuitka --standalone --macos-create-app-bundle emailproxy.py`) or [pyinstaller](https://pyinstaller.org/) (`pyinstaller --onefile emailproxy.py`). +If you are using pyinstaller and the GUI version of the proxy, you may need to add `--hidden-import timeago.locales.en_short` until [this `timeago` issue](https://github.com/hustcc/timeago/issues/40) is resolved. Python 3.7 or later is required to run the proxy. The [python2 branch](https://github.com/simonrob/email-oauth2-proxy/tree/python2) provides minimal compatibility with python 2.7, but with a limited feature set, and no ongoing maintenance. @@ -151,7 +152,7 @@ The method to achieve this differs depending on whether you are using macOS, Win On macOS, the file `~/Library/LaunchAgents/ac.robinson.email-oauth2-proxy.plist` is used to configure automatic starting of the proxy. If you stop the proxy's service (i.e., `Quit Email OAuth 2.0 Proxy` from the menu bar), you can restart it using `launchctl start ac.robinson.email-oauth2-proxy` from a terminal. -You can stop, disable or remove the service from your startup items either via the menu bar icon option, or using `launchctl unload [plist path]`. +You can stop, disable or remove the service from your startup items either via the menu bar icon option, or using `launchctl unload `_`[plist path]`_. If you edit the plist file manually, make sure you `unload` and then `load` it to update the system with your changes. If the `Start at login` option appears not to be working for you on macOS, see the [known issues section](#known-issues) for potential solutions. @@ -189,7 +190,7 @@ The easiest approach here is to use [OpenSSL](https://www.openssl.org/): `openss If you are having trouble actually connecting to the proxy, it is always worth double-checking the `local_address` values that you are using. The [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) sets this parameter to `127.0.0.1` for all servers. If you remove this value and do not provide your own, the proxy defaults to `::` – in most cases this resolves to `localhost` for both IPv4 and IPv6 configurations, but it is possible that this differs depending on your environment. -If you are unable to connect to the proxy from your client, it is always worth first specifying this value explicitly – see the [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) for further details about how to do this. +If you are unable to connect to the proxy from your email client, first try specifying this value explicitly – see the [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) for further details about how to do this. Please try setting and connecting to both IPv4 (i.e., `127.0.0.1`) and IPv6 (i.e., `::1`) loopback addresses before reporting any connection issues with the proxy. ### Dependencies and setup @@ -202,7 +203,7 @@ This is caused by missing dependencies for [pystray](https://github.com/moses-pa See the [pywebview dependencies](https://pywebview.flowrl.com/guide/installation.html#dependencies) and [pystray FAQ](https://pystray.readthedocs.io/en/latest/faq.html) pages and [existing](https://github.com/simonrob/email-oauth2-proxy/issues/1#issuecomment-831746642) [closed issues](https://github.com/simonrob/email-oauth2-proxy/issues/136#issuecomment-1430417456) in this repository for a summary and suggestions about how to resolve this. A similar issue may occur on Windows with the [pythonnet](https://github.com/pythonnet/pythonnet) package, which is required by [pywebview](https://github.com/r0x0r/pywebview). -If you are unable to resolve this by following the [pythonnet installation instructions](https://github.com/pythonnet/pythonnet/wiki/Installation), you may find that installing a [prebuilt wheel](https://www.lfd.uci.edu/~gohlke/pythonlibs/#pythonnet) helps fix the issue. +The [pythonnet installation instructions](https://github.com/pythonnet/pythonnet/wiki/Installation) may offer alternative ways to install this package if the default installation fails. Note that the public releases of pythonnet can take some time to be compatible with the latest major python release, so it can be worth using a slightly older version of python, or a pre-release version of pythonnet. ### Known issues @@ -219,7 +220,7 @@ Once this has been approved, the proxy's menu bar icon will appear as normal. In some cases — particularly when running the proxy in a virtual environment, or using the built-in macOS python, rather than the python.org version, or installations managed by, e.g., homebrew, pyenv, etc. — the permission prompt does not appear. If this happens it is worth first trying to `unload` and then `load` the service via `launchctl`. If this still does not cause the prompt to appear, the only currently-known resolution is to run the proxy outside of a virtual environment and manually grant Full Disk Access to your python executable via the privacy settings in the macOS System Preferences. -You may also need to edit the proxy's launch agent plist file, which is found at the location given in the command above, to set the path to your python executable – it must be the real path rather than a symlink (the `readlink` command can help here). +You may also need to edit the proxy's launch agent plist file, which is found at the location given [in the command above](#starting-the-proxy-automatically), to set the path to your python executable – it must be the real path rather than a symlink (the `readlink` command can help here). Fortunately this is a one-time fix, and once the proxy loads successfully via this method you will not need to adjust its startup configuration again (except perhaps when upgrading to a newer major macOS version, in which case just repeat the procedure). ### Other problems @@ -227,11 +228,13 @@ Please feel free to [open an issue](https://github.com/simonrob/email-oauth2-pro ## Advanced features -The [plugins variant](https://github.com/simonrob/email-oauth2-proxy/tree/plugins) has an additional feature that enables the use of separate scripts to modify IMAP/POP/SMTP commands when they are received from the client or server before passing through to the other side of the connection. +The [plugins variant of the proxy](https://github.com/simonrob/email-oauth2-proxy/tree/plugins) has an additional feature that enables the use of separate scripts to modify IMAP/POP/SMTP commands when they are received from the client or server before passing through to the other side of the connection. This allows a wide range of additional capabilities or triggers to be added the proxy. + For example, the [IMAPIgnoreSentMessageUpload plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/IMAPIgnoreSentMessageUpload.py) intercepts any client commands to add emails to the IMAP sent messages mailbox, which resolves message duplication issues for servers that automatically do this when emails are received via SMTP (e.g., Office 365, Gmail, etc.). -The [IMAPCleanO365ATPLinks plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/IMAPCleanO365ATPLinks.py) restores "Safe Links" modified by Microsoft Defender for Office 365 to their original URLs. +The [IMAPCleanO365ATPLinks plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/IMAPCleanO365ATPLinks.py) restores "Safe Links" modified by Microsoft Defender for Office 365 to their original URLs, while the [IMAPRegexContentReplacer plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/IMAPRegexContentReplacer.py) lets you match and remove/replace any content in the message. The [SMTPBlackHole plugin](https://github.com/simonrob/email-oauth2-proxy/blob/plugins/plugins/SMTPBlackHole.py) gives the impression emails are being sent but actually silently discards them, which is useful for testing email sending tools. + See the [documentation and examples](https://github.com/simonrob/email-oauth2-proxy/tree/plugins/plugins) for further details, additional sample plugins and setup instructions. diff --git a/emailproxy.config b/emailproxy.config index 18b9aa6..d84f226 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -199,6 +199,13 @@ redirect_uri = http://localhost client_id = *** your client id here *** client_secret = *** your client secret here *** +[your.free.outlook.or.hotmail.address@outlook.com] +permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/authorize +token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token +oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access +client_id = *** your client id here - note that as you are not the administrator of Hotmail.com / Outlook.com, you will need to reuse an existing client ID (see the proxy's readme) *** +redirect_uri = https://localhost + [your.email@gmail.com] permission_url = https://accounts.google.com/o/oauth2/auth token_url = https://oauth2.googleapis.com/token @@ -212,7 +219,7 @@ permission_url = https://api.login.yahoo.com/oauth2/request_auth token_url = https://api.login.yahoo.com/oauth2/get_token oauth2_scope = mail-w redirect_uri = http://localhost -client_id = *** your client id here *** +client_id = *** your client id here - note that as new client registrations are not permitted for Yahoo, you will need to reuse an existing client ID (see the proxy's readme) *** client_secret = *** your client secret here *** [your.email@aol.com] @@ -220,7 +227,7 @@ permission_url = https://api.login.aol.com/oauth2/request_auth token_url = https://api.login.aol.com/oauth2/get_token oauth2_scope = mail-w redirect_uri = http://localhost -client_id = *** your client id here *** +client_id = *** your client id here - note that as new client registrations are not permitted for AOL, you will need to reuse an existing client ID (see the proxy's readme) *** client_secret = *** your client secret here *** [ccg.flow.configured.address@your-tenant.com] diff --git a/emailproxy.py b/emailproxy.py index 28c8910..5a9d2e5 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-10-04' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-10-21' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -1105,6 +1105,7 @@ def get_oauth2_authorisation_code(permission_url, redirect_uri, redirect_listen_ time.sleep(1) @staticmethod + # pylint: disable-next=too-many-positional-arguments def get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_secret, jwt_client_assertion, authorisation_code, oauth2_scope, oauth2_flow, username, password): """Requests OAuth 2.0 access and refresh tokens from token_url using the given client_id, client_secret, @@ -1182,6 +1183,7 @@ def get_service_account_authorisation_token(key_type, key_path_or_contents, oaut return {'access_token': credentials.token, 'expires_in': int(credentials.expiry.timestamp() - time.time())} @staticmethod + # pylint: disable-next=too-many-positional-arguments def refresh_oauth2_access_token(token_url, client_id, client_secret, jwt_client_assertion, username, refresh_token): """Obtains a new access token from token_url using the given client_id, client_secret and refresh token, returning a dict with 'access_token', 'expires_in', and 'refresh_token' on success; exception on failure""" @@ -1385,6 +1387,7 @@ class OAuth2ClientConnection(SSLAsyncoreDispatcher): """The base client-side connection that is subclassed to handle IMAP/POP/SMTP client interaction (note that there is some protocol-specific code in here, but it is not essential, and only used to avoid logging credentials)""" + # pylint: disable-next=too-many-positional-arguments def __init__(self, proxy_type, connection_socket, socket_map, proxy_parent, custom_configuration): SSLAsyncoreDispatcher.__init__(self, connection_socket=connection_socket, socket_map=socket_map) self.receive_buffer = b'' @@ -1829,6 +1832,7 @@ def send_authentication_request(self): class OAuth2ServerConnection(SSLAsyncoreDispatcher): """The base server-side connection that is subclassed to handle IMAP/POP/SMTP server interaction""" + # pylint: disable-next=too-many-positional-arguments def __init__(self, proxy_type, connection_socket, socket_map, proxy_parent, custom_configuration): SSLAsyncoreDispatcher.__init__(self, socket_map=socket_map) # note: establish connection later due to STARTTLS self.receive_buffer = b'' From f0bf373efc78be6717b385c12f17bed3a39b0af1 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Tue, 5 Nov 2024 21:39:38 +0000 Subject: [PATCH 5/8] Add support for device authorisation grant - see #302 --- emailproxy.config | 13 +++- emailproxy.py | 186 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 150 insertions(+), 49 deletions(-) diff --git a/emailproxy.config b/emailproxy.config index d84f226..dde604f 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -155,6 +155,10 @@ documentation = Accounts are specified using your email address as the section h attempts before the first valid login, pre-encrypting account entries is highly recommended. See the example script at https://github.com/simonrob/email-oauth2-proxy/issues/61#issuecomment-1259110336. + - The proxy supports the device authorisation grant (DAG) OAuth 2.0 flow (RFC 8628), which may better suit headless + systems. To use this flow, set `oauth2_flow = device`. With this flow, the proxy receives authorisation responses + directly from the service provider, so no `redirect_uri` is needed. An example account configuration is given below. + Gmail customisation: - The proxy supports the use of service accounts with Gmail for Google Workspace (note: normal Gmail accounts do not support this method). To use this option, add an account entry as normal, but do not add a `permission_url` value @@ -203,7 +207,7 @@ client_secret = *** your client secret here *** permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/authorize token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access -client_id = *** your client id here - note that as you are not the administrator of Hotmail.com / Outlook.com, you will need to reuse an existing client ID (see the proxy's readme) *** +client_id = *** your client id here - note that as you are not the administrator of Hotmail.com / Outlook.com, you will likely need to reuse an existing client ID (see the proxy's readme) *** redirect_uri = https://localhost [your.email@gmail.com] @@ -230,6 +234,13 @@ redirect_uri = http://localhost client_id = *** your client id here - note that as new client registrations are not permitted for AOL, you will need to reuse an existing client ID (see the proxy's readme) *** client_secret = *** your client secret here *** +[dag.flow.configured.address@outlook.com] +permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/devicecode +token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token +oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access +oauth2_flow = device +client_id = *** your client id here *** + [ccg.flow.configured.address@your-tenant.com] documentation = *** note: this is an advanced O365 account example; in most cases you want the version above instead *** token_url = https://login.microsoftonline.com/*** your tenant id here ***/oauth2/v2.0/token diff --git a/emailproxy.py b/emailproxy.py index 5a9d2e5..0b60db2 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-10-21' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-11-05' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -210,32 +210,38 @@ class NSObject: Ovp0yY9EkQZ8XELHSa+x0S9OAm75cT+F+UFm+vhbmClQLCtF+SnMNAji11lcz5orzCQopo21KJIn3FB37iuaJ9yRd+4zuicsSINViSesyEgbMtQcZgIE TyNBsIQrXgdVS3h2hGdf+Apf4eIIF+ub16FYBhQd4ci3IiAOBP8/z+kNGUS6hBN6UlIAAAAASUVORK5CYII=''' # 22px SF Symbols lock.fill -EXTERNAL_AUTH_HTML = ''' -

Login authorisation request for %s

+ document.execCommand('copy');document.body.removeChild(copySource);source.innerText='✔'; + window.setTimeout(()=>source.innerText='⧉',1000)} +

Login authorisation request for %s

Click the following link to open your browser and approve the request:

%s - ⧉

-

After logging in and successfully authorising your account, paste and submit the - resulting URL from the browser's address bar using the box at the bottom of this page to allow the %s script to - transparently handle login requests on your behalf in future.

-

Note that your browser may show a navigation error (e.g., "localhost refused to connect") after + ⧉

''' + +EXTERNAL_AUTH_HTML = EXTERNAL_AUTH_HTML_BASE + '''

After logging in and successfully + authorising your account, paste and submit the resulting URL from the browser's address bar using the box at the + bottom of this page to allow the %s script to transparently handle login requests on your behalf in future.

+

Note that your browser may show a navigation error (e.g., “localhost refused to connect”) after successfully logging in, but the final URL is the only important part, and as long as this begins with the correct redirection URI and contains a valid authorisation code your email client's request will succeed.''' + ( ' If you are using Windows, submitting can take a few seconds.' if sys.platform == 'win32' else '') + '''

According to your proxy configuration file, the expected URL will be of the form:

- 

%s [...] code=[code] [...]

-
+ 

%s […] code=[code] […]

+
''' +EXTERNAL_AUTH_DAG_HTML = EXTERNAL_AUTH_HTML_BASE + '''

Enter the following code when + prompted:

%s
⧉ +

You can close this window once authorisation is complete.

''' + EXITING = False # used to check whether to restart failed threads - is set to True if the user has requested to exit @@ -855,13 +861,16 @@ def get_oauth2_credentials(username, password, reload_remote_accounts=True): if not access_token: auth_result = None if permission_url: # O365 CCG/ROPCG and Google service accounts skip authorisation; no permission_url - oauth2_flow = 'authorization_code' + if oauth2_flow != 'device': # the device flow is a poll-based method with asynchronous interaction + oauth2_flow = 'authorization_code' + permission_url = OAuth2Helper.construct_oauth2_permission_url(permission_url, redirect_uri, client_id, oauth2_scope, username) # note: get_oauth2_authorisation_code is a blocking call (waiting on user to provide code) success, auth_result = OAuth2Helper.get_oauth2_authorisation_code(permission_url, redirect_uri, - redirect_listen_address, username) + redirect_listen_address, username, + oauth2_flow) if not success: Log.info('Authorisation result error for account', username, '- aborting login.', auth_result) @@ -873,6 +882,7 @@ def get_oauth2_credentials(username, password, reload_remote_accounts=True): '`oauth2_flow` value is specified when using a method that does not require a ' '`permission_url`' % (APP_NAME, username)) + # note: get_oauth2_authorisation_tokens may be a blocking call (DAG flow retries until user code entry) response = OAuth2Helper.get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_secret, jwt_client_assertion, auth_result, oauth2_scope, oauth2_flow, @@ -1041,13 +1051,38 @@ def construct_oauth2_permission_url(permission_url, redirect_uri, client_id, sco 'Sign in with another account' option)""" params = {'client_id': client_id, 'redirect_uri': redirect_uri, 'scope': scope, 'response_type': 'code', 'access_type': 'offline', 'login_hint': username} + if not redirect_uri: # unlike other interactive flows, DAG doesn't involve a (known) final redirect + del params['redirect_uri'] param_pairs = ['%s=%s' % (param, OAuth2Helper.oauth2_url_escape(value)) for param, value in params.items()] return '%s?%s' % (permission_url, '&'.join(param_pairs)) @staticmethod - def get_oauth2_authorisation_code(permission_url, redirect_uri, redirect_listen_address, username): + def start_device_authorisation_grant(permission_url): + """Requests the device authorisation grant flow URI and user code - see https://tools.ietf.org/html/rfc8628""" + try: + response = urllib.request.urlopen( + urllib.request.Request(permission_url, headers={'User-Agent': APP_NAME}), + timeout=AUTHENTICATION_TIMEOUT).read() + parsed_result = json.loads(response) + verification_uri = parsed_result.get('verification_uri_complete', parsed_result['verification_uri']) + user_code = parsed_result['user_code'] + return True, parsed_result, verification_uri, user_code + except urllib.error.HTTPError as e: + return False, json.loads(e.read()), None, None + + @staticmethod + def get_oauth2_authorisation_code(permission_url, redirect_uri, redirect_listen_address, username, oauth2_flow): """Submit an authorisation request to the parent app and block until it is provided (or the request fails)""" - token_request = {'permission_url': permission_url, 'redirect_uri': redirect_uri, + + # the device authorisation grant flow requires a bit of pre-precessing to get the actual permission URL + user_code, device_grant_result = None, None + if oauth2_flow == 'device': + success, device_grant_result, permission_url, user_code = OAuth2Helper.start_device_authorisation_grant( + permission_url) + if not success: + return device_grant_result + + token_request = {'permission_url': permission_url, 'user_code': user_code, 'redirect_uri': redirect_uri, 'redirect_listen_address': redirect_listen_address, 'username': username, 'expired': False} REQUEST_QUEUE.put(token_request) response_queue_reference = RESPONSE_QUEUE # referenced locally to avoid inserting into the new queue on restart @@ -1078,6 +1113,9 @@ def get_oauth2_authorisation_code(permission_url, redirect_uri, redirect_listen_ threading.Thread(target=OAuth2Helper.start_redirection_receiver_server, args=(data,), name='EmailOAuth2Proxy-auth-%s' % data['username'], daemon=True).start() + if oauth2_flow == 'device': + return True, device_grant_result + else: if 'response_url' in data and OAuth2Helper.match_redirect_uri(token_request['redirect_uri'], data['response_url']): @@ -1093,7 +1131,7 @@ def get_oauth2_authorisation_code(permission_url, redirect_uri, redirect_listen_ if 'error' in response: message = 'OAuth 2.0 authorisation error for account %s: ' % data['username'] message += response['error'] - message += '; %s' % response['error_description'] if 'error_description' in response else '' + message += '; %s' % response.get('error_description', '') return False, message return (False, 'OAuth 2.0 authorisation response for account %s has neither code nor error ' 'message' % data['username']) @@ -1107,7 +1145,7 @@ def get_oauth2_authorisation_code(permission_url, redirect_uri, redirect_listen_ @staticmethod # pylint: disable-next=too-many-positional-arguments def get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_secret, jwt_client_assertion, - authorisation_code, oauth2_scope, oauth2_flow, username, password): + authorisation_result, oauth2_scope, oauth2_flow, username, password): """Requests OAuth 2.0 access and refresh tokens from token_url using the given client_id, client_secret, authorisation_code and redirect_uri, returning a dict with 'access_token', 'expires_in', and 'refresh_token' on success, or throwing an exception on failure (e.g., HTTP 400)""" @@ -1115,8 +1153,9 @@ def get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_s return OAuth2Helper.get_service_account_authorisation_token(client_id, client_secret, oauth2_scope, username) - params = {'client_id': client_id, 'client_secret': client_secret, 'code': authorisation_code, + params = {'client_id': client_id, 'client_secret': client_secret, 'code': authorisation_result, 'redirect_uri': redirect_uri, 'grant_type': oauth2_flow} + expires_in = AUTHENTICATION_TIMEOUT if not client_secret: del params['client_secret'] # client secret can be optional for O365, but we don't want a None entry @@ -1133,20 +1172,44 @@ def get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_s if oauth2_flow != 'authorization_code': del params['code'] # CCG/ROPCG flows have no code, but we need the scope and (for ROPCG) username+password params['scope'] = oauth2_scope - if oauth2_flow == 'password': + if oauth2_flow == 'device': + params['grant_type'] = 'urn:ietf:params:oauth:grant-type:device_code' + params['device_code'] = authorisation_result['device_code'] + expires_in = authorisation_result['expires_in'] + elif oauth2_flow == 'password': params['username'] = username params['password'] = password if not redirect_uri: del params['redirect_uri'] # redirect_uri is not typically required in non-code flows; remove if empty - try: - response = urllib.request.urlopen( - urllib.request.Request(token_url, data=urllib.parse.urlencode(params).encode('utf-8'), - headers={'User-Agent': APP_NAME}), timeout=AUTHENTICATION_TIMEOUT).read() - return json.loads(response) - except urllib.error.HTTPError as e: - e.message = json.loads(e.read()) - Log.debug('Error requesting access token for account', username, '- received invalid response:', e.message) - raise e + + # the device flow requires repeatedly polling the service while the result is pending + expires_at = time.time() + expires_in + while time.time() < expires_at and not EXITING: + try: + # in all flows except DAG, we make one attempt only + response = urllib.request.urlopen( + urllib.request.Request(token_url, data=urllib.parse.urlencode(params).encode('utf-8'), + headers={'User-Agent': APP_NAME}), timeout=AUTHENTICATION_TIMEOUT).read() + return json.loads(response) + + except urllib.error.HTTPError as e: + e.message = json.loads(e.read()) + if oauth2_flow == 'device' and e.code == 400: + if e.message['error'] == 'slow_down': + authorisation_result['interval'] *= 2 + continue + if e.message['error'] == 'authorization_pending': + Log.debug('Waiting for device flow confirmation for account', username, '- retrying in', + '%ds; timeout in %ds' % (authorisation_result['interval'], expires_at - time.time())) + time.sleep(authorisation_result['interval']) + continue + + Log.debug('Error requesting access token for account', username, '- received invalid response:', + e.message) + raise e + + if oauth2_flow == 'device' and not EXITING: + raise TimeoutError('The device authorisation grant flow request timed out') # noinspection PyUnresolvedReferences @staticmethod @@ -2915,11 +2978,18 @@ def authorise_account(self, _, item): self.notify(APP_NAME, 'There are no pending authorisation requests') def create_authorisation_window(self, request): - # note that the webview title *must* end with a space and then the email address/username + # note that the webview title *must* contain the email address/username as it is used to confirm success window_title = 'Authorise your account: %s' % request['username'] + if request['user_code']: + window_title += ' with code: %s' % request['user_code'] + if self.args.external_auth: - auth_page = EXTERNAL_AUTH_HTML % (request['username'], request['permission_url'], request['permission_url'], - request['permission_url'], APP_NAME, request['redirect_uri']) + if request['user_code']: + auth_page = EXTERNAL_AUTH_DAG_HTML % ( + request['username'], *[request['permission_url']] * 3, *[request['user_code']] * 2) + else: + auth_page = EXTERNAL_AUTH_HTML % ( + request['username'], *[request['permission_url']] * 3, APP_NAME, request['redirect_uri']) authorisation_window = webview.create_window(window_title, html=auth_page, on_top=True, text_select=True) else: authorisation_window = webview.create_window(window_title, request['permission_url'], on_top=True) @@ -2972,8 +3042,8 @@ def authorisation_window_loaded(self): url = window.get_current_url() # noinspection PyUnresolvedReferences - username = window.get_title(window).split(' ')[-1] # see note above: title *must* match this format - if not url or not username: + username = next((a for a in window.get_title(window).split(' ') if '@' in a), None) + if not url or not username: # see note in create_authorisation_window: title *must* match username format continue # skip any invalid windows # respond to both the original request and any duplicates in the list @@ -2981,8 +3051,9 @@ def authorisation_window_loaded(self): for request in self.authorisation_requests[:]: # iterate over a copy; remove from original if request['username'] == username and OAuth2Helper.match_redirect_uri(request['redirect_uri'], url): Log.info('Returning authorisation request result for', request['username']) - RESPONSE_QUEUE.put( - {'permission_url': request['permission_url'], 'response_url': url, 'username': username}) + if not request['user_code']: # the device authorisation grant flow does not require a response + RESPONSE_QUEUE.put( + {'permission_url': request['permission_url'], 'response_url': url, 'username': username}) self.authorisation_requests.remove(request) completed_request = request else: @@ -2992,7 +3063,9 @@ def authorisation_window_loaded(self): if completed_request is None: continue # no requests processed for this window - nothing to do yet - window.destroy() + # the device authorisation grant flow will not normally have a matching `redirect_uri` + elif not completed_request['user_code']: + window.destroy() self.icon.update_menu() # note that in this part of the interaction we don't actually check the *use* of the authorisation code, @@ -3178,10 +3251,9 @@ def notify(self, title, text): self.icon.remove_notification() self.icon.notify('%s: %s' % (title, text)) - else: - Log.info(title, text) # last resort + Log.info(text) # duplicate to log for, e.g., local server auth mode when GUI is present else: - Log.info(title, text) + Log.info(text) def stop_servers(self): global RESPONSE_QUEUE @@ -3381,22 +3453,40 @@ def post_create(self, icon): Log.info('Authorisation request received for', data['username'], '(local server auth mode)' if self.args.local_server_auth else '(external auth mode)' if self.args.external_auth else '(interactive mode)') + + user_code_notification = None + if data['user_code']: + RESPONSE_QUEUE.put(data) # device flow does not require a user response; here we only notify + user_code_notification = 'Visit %s and use code %s' % (data['permission_url'], data['user_code']) + if self.args.local_server_auth: self.notify(APP_NAME, 'Local server auth mode: please authorise a request for account %s' % data['username']) - data['local_server_auth'] = True - RESPONSE_QUEUE.put(data) # local server auth is handled by the client/server connections + if user_code_notification: + # note: local server auth mode doesn't make any sense with the device flow as it is poll-based, + # so here we notify and return the request as normal, but don't actually start the local server + self.notify(APP_NAME, user_code_notification) + else: + data['local_server_auth'] = True + RESPONSE_QUEUE.put(data) # local server auth is handled by the client/server connections + elif self.args.external_auth and not self.args.gui: - if sys.stdin and sys.stdin.isatty(): + can_auth_interactively = sys.stdin and sys.stdin.isatty() + if can_auth_interactively: self.notify(APP_NAME, 'No-GUI external auth mode: please authorise a request for account ' '%s' % data['username']) - self.terminal_external_auth_prompt(data) + if user_code_notification: + self.notify(APP_NAME, user_code_notification) else: + self.terminal_external_auth_prompt(data) + if not can_auth_interactively: Log.error('Not running interactively; unable to handle no-GUI external auth request') + elif icon: self.authorisation_requests.append(data) icon.update_menu() # force refresh the menu self.notify(APP_NAME, 'Please authorise your account %s from the menu' % data['username']) + else: for request in self.authorisation_requests[:]: # iterate over a copy; remove from original if request['permission_url'] == data['permission_url']: From 2eb92d0473cb2b72d46247563c7e28cf0e99257e Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Wed, 6 Nov 2024 20:08:12 +0000 Subject: [PATCH 6/8] Fix issue with pywebview 5.3+ and navigation redirection --- emailproxy.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/emailproxy.py b/emailproxy.py index 5a9d2e5..8da1b10 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-10-21' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-11-06' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -2424,13 +2424,15 @@ class ProvisionalNavigationBrowserDelegate: # callbacks, but using that means that window.get_current_url() returns None when the loaded handler is called def webView_didStartProvisionalNavigation_(self, web_view, _nav): # called when a user action (i.e., clicking our external authorisation mode submit button) redirects locally - browser_view_instance = webview.platforms.cocoa.BrowserView.get_instance('webkit', web_view) + browser_view_instance = webview.platforms.cocoa.BrowserView.get_instance( + ProvisionalNavigationBrowserDelegate.pywebview_attr, web_view) if browser_view_instance: browser_view_instance.loaded.set() def webView_didReceiveServerRedirectForProvisionalNavigation_(self, web_view, _nav): # called when the server initiates a local redirect - browser_view_instance = webview.platforms.cocoa.BrowserView.get_instance('webkit', web_view) + browser_view_instance = webview.platforms.cocoa.BrowserView.get_instance( + ProvisionalNavigationBrowserDelegate.pywebview_attr, web_view) if browser_view_instance: browser_view_instance.loaded.set() @@ -2440,7 +2442,7 @@ def performKeyEquivalent_(self, event): event.keyCode() == 12 and self.window().firstResponder(): self.window().performClose_(event) return True - return webview.platforms.cocoa.BrowserView.WebKitHost.performKeyEquivalentBase_(self, event) + return False if sys.platform == 'darwin': # noinspection PyUnresolvedReferences @@ -2942,18 +2944,16 @@ def handle_authorisation_windows(self): # pywebview window can get into a state in which http://localhost navigation, rather than failing, just hangs # noinspection PyPackageRequirements import webview.platforms.cocoa + pywebview_version = packaging.version.Version(importlib_metadata.version('pywebview')) + ProvisionalNavigationBrowserDelegate.pywebview_attr = 'webkit' if pywebview_version < packaging.version.Version( + '5.3') else 'webview' setattr(webview.platforms.cocoa.BrowserView.BrowserDelegate, 'webView_didStartProvisionalNavigation_', ProvisionalNavigationBrowserDelegate.webView_didStartProvisionalNavigation_) setattr(webview.platforms.cocoa.BrowserView.BrowserDelegate, 'webView_didReceiveServerRedirectForProvisional' 'Navigation_', ProvisionalNavigationBrowserDelegate.webView_didReceiveServerRedirectForProvisionalNavigation_) - try: - setattr(webview.platforms.cocoa.BrowserView.WebKitHost, 'performKeyEquivalentBase_', - webview.platforms.cocoa.BrowserView.WebKitHost.performKeyEquivalent_) - setattr(webview.platforms.cocoa.BrowserView.WebKitHost, 'performKeyEquivalent_', - ProvisionalNavigationBrowserDelegate.performKeyEquivalent_) - except TypeError: - pass + setattr(webview.platforms.cocoa.BrowserView.WebKitHost, 'performKeyEquivalent_', + ProvisionalNavigationBrowserDelegate.performKeyEquivalent_) # also needed only on macOS because otherwise closing the last remaining webview window exits the application dummy_window = webview.create_window('%s hidden (dummy) window' % APP_NAME, html='', hidden=True) From ab0fbd300632f430962759f4c9970a57d55e406f Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 11 Nov 2024 09:13:24 +0000 Subject: [PATCH 7/8] Handle optional DAG request interval --- emailproxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/emailproxy.py b/emailproxy.py index 0b60db2..6ef5286 100644 --- a/emailproxy.py +++ b/emailproxy.py @@ -6,7 +6,7 @@ __author__ = 'Simon Robinson' __copyright__ = 'Copyright (c) 2024 Simon Robinson' __license__ = 'Apache 2.0' -__version__ = '2024-11-05' # ISO 8601 (YYYY-MM-DD) +__version__ = '2024-11-11' # ISO 8601 (YYYY-MM-DD) __package_version__ = '.'.join([str(int(i)) for i in __version__.split('-')]) # for pyproject.toml usage only import abc @@ -1176,6 +1176,7 @@ def get_oauth2_authorisation_tokens(token_url, redirect_uri, client_id, client_s params['grant_type'] = 'urn:ietf:params:oauth:grant-type:device_code' params['device_code'] = authorisation_result['device_code'] expires_in = authorisation_result['expires_in'] + authorisation_result['interval'] = authorisation_result.get('interval', 5) # see RFC 8628, Section 3.2 elif oauth2_flow == 'password': params['username'] = username params['password'] = password From 910c393f400ffc35202f89e38284f0199130cdf3 Mon Sep 17 00:00:00 2001 From: Simon Robinson Date: Mon, 11 Nov 2024 11:22:52 +0000 Subject: [PATCH 8/8] Improve documentation and example configurations --- README.md | 54 +++++++++++----------- emailproxy.config | 114 +++++++++++++++++++++++++++------------------- 2 files changed, 93 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index af830c3..6271ed3 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ The proxy works in the background with a menu bar/taskbar helper or as a headles ### Example use-cases - You need to use an Office 365 email account, but don't get on with Outlook. -The email client you like doesn't support OAuth 2.0, which became mandatory [in January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437) ([September 2024 for free Hotmail/Outlook accounts](https://support.microsoft.com/en-us/office/modern-authentication-methods-now-needed-to-continue-syncing-outlook-email-in-non-microsoft-email-apps-c5d65390-9676-4763-b41f-d7986499a90d); [September 2025 for O365 SMTP](https://techcommunity.microsoft.com/t5/exchange-team-blog/exchange-online-to-retire-basic-auth-for-client-submission-smtp/ba-p/4114750)). +The email client you like doesn't support OAuth 2.0, which became mandatory [in January 2023](https://techcommunity.microsoft.com/t5/exchange-team-blog/basic-authentication-deprecation-in-exchange-online-september/ba-p/3609437) ([September 2024 for personal Hotmail/Outlook accounts](https://support.microsoft.com/en-us/office/modern-authentication-methods-now-needed-to-continue-syncing-outlook-email-in-non-microsoft-email-apps-c5d65390-9676-4763-b41f-d7986499a90d); [September 2025 for O365 SMTP](https://techcommunity.microsoft.com/t5/exchange-team-blog/exchange-online-to-retire-basic-auth-for-client-submission-smtp/ba-p/4114750)). - You used to use Gmail via IMAP/POP/SMTP with your raw account credentials (i.e., your real password), but cannot do this now that Google has disabled this method, and don't want to use an [App Password](https://support.google.com/accounts/answer/185833) (or cannot enable this option). - You have an account already set up in an email client, and you need to switch it to OAuth 2.0 authentication. You can edit the server details, but the client forces you to delete and re-add the account to enable OAuth 2.0, and you don't want to do this. - You have made your own script or application that sends or receives email, but it doesn't support OAuth 2.0, and you don't want to have to modify it to implement this. - You work with multiple services or applications that use IMAP/POP/SMTP, and you don't want to have to set up OAuth 2.0 independently for each one. -In all of these cases and more, this proxy can help. -Follow the instructions here to get started, and please [open an issue](https://github.com/simonrob/email-oauth2-proxy/issues) with any problems or suggestions. +In all of these cases and more, this proxy can help – just follow the instructions below to get started. +Visit the [Discussions pages](https://github.com/simonrob/email-oauth2-proxy/discussions) for help with any configuration or setup problems, or [open an issue](https://github.com/simonrob/email-oauth2-proxy/issues) to report bugs or make suggestions. For commercial support or feature requests, please also consider [sponsoring this project](https://github.com/sponsors/simonrob?frequency=one-time). @@ -32,7 +32,7 @@ Begin by downloading the proxy via one of the following methods: Next, edit the sample `emailproxy.config` file to add configuration details for each email server and account that you want to use with the proxy. -[Guidance and example account configurations](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) are provided for Office 365, Gmail and several other providers, though you will need to insert your own client credentials for each one (see the [client credentials documentation](#oauth-20-client-credentials) below for guidance). +[Guidance and example account configurations](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) are provided for Office 365, Gmail and several other providers, though you will need to insert your own client credentials for each one (see the [client credentials documentation](#oauth-20-client-credentials) below for help doing this). You can remove details from the sample configuration file for services you don't use, or add additional ones for any other OAuth 2.0-authenticated IMAP/POP/SMTP servers you would like to use with the proxy. You can now start the proxy: depending on which installation option you chose, either launch the application or use the appropriate run command listed above. @@ -58,25 +58,26 @@ After your accounts are fully set-up and authorised, no further proxy interactio It will notify you if this is the case. ### OAuth 2.0 client credentials -As part of the proxy setup process you need to provide an OAuth 2.0 `client_id` and `client_secret` to allow it to authenticate with email servers on your behalf. +As part of the proxy setup process you need to provide an OAuth 2.0 `client_id` and (in many cases) a `client_secret` to allow it to authenticate with email servers on your behalf. If you have an existing client ID and secret for a desktop app, you can use these directly in the proxy. If this is not possible, you can also reuse the client ID and secret from any email client that supports IMAP/POP/SMTP OAuth 2.0 authentication with the email server you would like to connect to (such as [the](https://github.com/mozilla/releases-comm-central/blob/812b7c9068ca5cac0580b0ddbea8e34c141cd441/mailnews/base/src/OAuth2Providers.jsm) [many](https://github.com/mozilla/releases-comm-central/blob/master/mailnews/base/src/OAuth2Providers.sys.mjs) [existing](https://github.com/Foundry376/Mailspring/blob/master/app/internal_packages/onboarding/lib/onboarding-constants.ts) [open](https://gitlab.gnome.org/GNOME/evolution-data-server/-/blob/master/CMakeLists.txt) [source](https://gitlab.gnome.org/GNOME/gnome-online-accounts/-/blob/master/meson_options.txt) [clients](https://github.com/M66B/FairEmail/blob/master/app/src/main/res/xml/providers.xml) with OAuth 2.0 support), but please do this with care and restraint as access through reused tokens will be associated with the token owner rather than your own client. -If you do not have access to credentials for an existing client you will need to register your own. +If you do not want to use credentials from an existing client you will need to register your own. The process to do this is different for each provider, but the registration guides for several common ones are linked here. In all cases, when registering, make sure your client is set up to use an OAuth scope that will give it permission to access IMAP/POP/SMTP as desired. It is also highly recommended to use a scope that will grant "offline" access (i.e., a way to [refresh the OAuth 2.0 authentication token](https://oauth.net/2/refresh-tokens/) without user intervention). The [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) provides example scope values for several common providers. -- Office 365: register a new [Microsoft identity application](https://learn.microsoft.com/entra/identity-platform/quickstart-register-app) -- Gmail / Google Workspace: register a [Google API desktop app client](https://developers.google.com/identity/protocols/oauth2/native-app) -- Outlook / Hotmail (free accounts): because you are not the administrator for these Microsoft-operated domains, the only option is to reuse an existing client ID – see, for example, [Thunderbird](https://blog.thunderbird.net/2023/01/important-message-for-microsoft-office-365-enterprise-users/), or the links above -- AOL and Yahoo Mail (and subproviders such as AT&T) are not currently allowing new client registrations with the OAuth email scope – the only option here is to reuse the credentials from an existing client that does have this permission +- Office 365: register a new [Microsoft identity application](https://learn.microsoft.com/entra/identity-platform/quickstart-register-app). +- Gmail / Google Workspace: register a [Google API desktop app client](https://developers.google.com/identity/protocols/oauth2/native-app). +- Outlook / Hotmail (personal accounts): If you are part of the Microsoft 365 Developer Programme or have an Azure account (including free accounts), you can create your own app registration in the Entra admin centre – see [this discussion](https://github.com/simonrob/email-oauth2-proxy/discussions/301) for a guide. +If not, you will need to reuse an existing client ID – see, for example, [this sample configuration](https://github.com/simonrob/email-oauth2-proxy/issues/297#issuecomment-2424200404). +- AOL and Yahoo Mail (and subproviders such as AT&T) are not currently allowing new client registrations with the OAuth email scope – the only option here is to reuse the credentials from an existing client that does have this permission. The proxy supports [Google Cloud service accounts](https://cloud.google.com/iam/docs/service-account-overview) for access to Google Workspace Gmail. -It also supports the [client credentials grant (CCG)](https://learn.microsoft.com/entra/identity-platform/v2-oauth2-client-creds-grant-flow) and [resource owner password credentials grant (ROPCG)](https://learn.microsoft.com/entra/identity-platform/v2-oauth-ropc) OAuth 2.0 flows, and [certificate credentials (JWT)](https://learn.microsoft.com/entra/identity-platform/certificate-credentials). -Please note that currently only Office 365 is known to support the CCG, ROPCG and certificate credentials methods. +It also supports the [client credentials grant (CCG)](https://learn.microsoft.com/entra/identity-platform/v2-oauth2-client-creds-grant-flow), [resource owner password credentials grant (ROPCG)](https://learn.microsoft.com/entra/identity-platform/v2-oauth-ropc) and [device authorisation grant (DAG)](https://tools.ietf.org/html/rfc8628) OAuth 2.0 flows, and [certificate credentials (JWT)](https://learn.microsoft.com/entra/identity-platform/certificate-credentials). +Please note that currently only Office 365 / Outlook is known to support the CCG, ROPCG, DAG and certificate credentials methods. See the [sample configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) for further details. @@ -84,7 +85,7 @@ See the [sample configuration file](https://github.com/simonrob/email-oauth2-pro When starting the proxy there are several optional arguments that can be set to customise its behaviour. - `--no-gui` will launch the proxy without an icon, which allows it to be run as a `systemctl` service as demonstrated in [this example](https://github.com/simonrob/email-oauth2-proxy/issues/2#issuecomment-839713677), or fully headless as demonstrated in [various](https://github.com/michaelstepner/email-oauth2-proxy-aws) [other](https://github.com/blacktirion/email-oauth2-proxy-docker) subprojects. -Please note that unless you also specify one of the authorisation options below, this mode is only of use if you have already authorised your accounts through the proxy in GUI mode, or are importing a pre-authorised proxy configuration file from elsewhere. +Please note that unless you also specify one of the authorisation options below, or are using an OAuth 2.0 flow that does not require user authorisation, this mode is only of use if you have already authorised your accounts through the proxy in GUI mode, or are loading a proxy configuration file that already contains the cached authorisation tokens. If you do not set `--external-auth` or `--local-server-auth`, accounts that have not yet been authorised (or for whatever reason require re-authorisation) will time out when authenticating, and an error will be printed to the log. - `--external-auth` configures the proxy to present an account authorisation URL to be opened in an external browser and wait for you to copy+paste the post-authorisation result. @@ -104,7 +105,7 @@ See the [sample configuration file](https://github.com/simonrob/email-oauth2-pro - `--config-file` allows you to specify the location of a [configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) that the proxy should load. If this argument is not provided, the proxy will look for `emailproxy.config` in its working directory. By default, the proxy also saves cached OAuth 2.0 tokens back to this file, so it must be writable. -See the `--cache-store` option, if you would rather store configuration and cached values separately. +See the `--cache-store` option if you would rather store configuration and cached values separately. - `--cache-store` is used to specify a separate location in which to cache authorised OAuth 2.0 tokens and associated metadata. The value of this argument can either be the full path to a local file (which must be writable), or an identifier for an external store such as a secrets manager (see the [advanced configuration](#advanced-configuration) section). @@ -119,24 +120,17 @@ This argument is identical to enabling debug mode from the proxy's menu bar icon If needed, debug mode can also be toggled at runtime by sending the signal `SIGUSR1` (e.g.: `pkill -SIGUSR1 -f emailproxy`). ### Advanced configuration -The [example configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) contains further documentation for various additional features of the proxy, including catch-all (wildcard) accounts, locally-encrypted connections and advanced Office 365 OAuth 2.0 flows. - -The proxy caches authenticated OAuth 2.0 tokens and associated metadata back to its own configuration file by default, but can alternatively be configured to use either a separate local file or a secrets manager service for this purpose. -Currently only [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/) is supported for remote token storage. -To use this feature, set the [`--cache-store`](#optional-arguments-and-configuration) parameter to either a full ARN or a secret name, prefixing the value with `aws:` to identify its type to the proxy. -You must also install the AWS SDK for Python: `python -m pip install boto3` and [set up authentication credentials](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html#configuration) (including a region). -The minimum required permissions for the associated AWS IAM user are `secretsmanager:GetSecretValue` and `secretsmanager:PutSecretValue`. -If the named AWS Secret does not yet exist, the proxy will attempt to create it; here, the `secretsmanager:CreateSecret` permission is also required. +The [example configuration file](https://github.com/simonrob/email-oauth2-proxy/blob/main/emailproxy.config) contains further documentation for many additional features of the proxy, including catch-all (wildcard) accounts, locally-encrypted connections, advanced OAuth 2.0 flows, integration with a secrets manager and more. If you are using the proxy in a non-GUI environment it is possible to skip installation of dependencies that apply only to the interactive version. To do this, install via `python -m pip install emailproxy` (i.e., without the `[gui]` variant option), and pass the [`--no-gui`](#optional-arguments-and-configuration) argument when starting the proxy. -Please note that the proxy was designed as a GUI-based tool from the outset due to the inherently interactive nature of OAuth 2.0 authorisation, and there are limits to its ability to support fully no-GUI operation. +Please note that the proxy was designed as a GUI-based tool from the outset due to the inherently interactive nature of the most common OAuth 2.0 authorisation flows, and there are limits to its ability to support fully no-GUI operation. See the [optional arguments and configuration](#optional-arguments-and-configuration) section of this file for further details. If your network requires connections to use an existing proxy, you can instruct the script to use this by setting the [proxy handler](https://docs.python.org/3/library/urllib.request.html#urllib.request.ProxyHandler) environment variable `https_proxy` (and/or `http_proxy`) – for example, `https_proxy=localhost python -m emailproxy`. -After installing its requirements, the proxy script can be packaged as a single self-contained executable using [Nuitka](https://nuitka.net/) (`nuitka --standalone --macos-create-app-bundle emailproxy.py`) or [pyinstaller](https://pyinstaller.org/) (`pyinstaller --onefile emailproxy.py`). -If you are using pyinstaller and the GUI version of the proxy, you may need to add `--hidden-import timeago.locales.en_short` until [this `timeago` issue](https://github.com/hustcc/timeago/issues/40) is resolved. +After installing its requirements, the proxy script can be packaged as a single self-contained executable using [Nuitka](https://nuitka.net/) (`nuitka --standalone --macos-create-app-bundle emailproxy.py`) or [pyinstaller](https://pyinstaller.org/) (`pyinstaller --onefile emailproxy.py`[[1]](#f1)). +A pyinstaller-packaged version is provided automatically for each [release](https://github.com/simonrob/email-oauth2-proxy/releases). Python 3.7 or later is required to run the proxy. The [python2 branch](https://github.com/simonrob/email-oauth2-proxy/tree/python2) provides minimal compatibility with python 2.7, but with a limited feature set, and no ongoing maintenance. @@ -212,7 +206,7 @@ On Windows this is normally limited to keyboard shortcuts (i.e., copy/paste), bu As a workaround, the proxy will enable pywebview's debug mode when you run the proxy itself in debug mode, which should allow you to use the right-click context menu to copy/paste to enter text. If you are unable to proceed with popup-based authentication even with this workaround, it is worth trying the proxy's `--external-auth` or `--local-server-auth` options. -- On more recent macOS versions (10.14 and later), you may find that when first running the proxy as a service you need to manually load its launch agent in order to trigger a file access permission prompt. +- On macOS (10.14 and later), you may find that when first running the proxy as a service you need to manually load its launch agent in order to trigger a file access permission prompt. You will know intervention is necessary if the proxy exits (rather than restarts) the first time you click `Start at login` from its menu bar icon. To resolve this, exit the proxy and then run `launchctl load ~/Library/LaunchAgents/ac.robinson.email-oauth2-proxy.plist` from a terminal. A permission pop-up should appear requesting file access for python. @@ -241,13 +235,13 @@ See the [documentation and examples](https://github.com/simonrob/email-oauth2-pr ## Potential improvements (pull requests welcome) - Full feature parity on different platforms (e.g., live menu updating; monitoring network status; clickable notifications) - Switch to asyncio? (with Python 3.12, [PEP 594](https://peps.python.org/pep-0594/) removed the asyncore package that the proxy is built upon – currently mitigated by the use of [pyasyncore](https://pypi.org/project/pyasyncore/)) -- STARTTLS for IMAP/POP? +- Remote STARTTLS for IMAP/POP? ## Related projects and alternatives Michael Stepner has created a [Terraform configuration](https://github.com/michaelstepner/email-oauth2-proxy-aws) that helps run this proxy on a lightweight cloud server (AWS EC2). Thiago Macieira has provided a [makefile and systemd configuration files](https://github.com/thiagomacieira/email-oauth2-proxy/tree/Add_a_Makefile_and_systemd_configuration_files_to_install_system_wide). -For Docker, blacktirion has an [example configuration](https://github.com/blacktirion/email-oauth2-proxy-docker). +For Docker, Moriah Morgan has an [example configuration](https://github.com/blacktirion/email-oauth2-proxy-docker). If you already use postfix, the [sasl-xoauth2](https://github.com/tarickb/sasl-xoauth2) plugin is probably a better solution than running this proxy. Similarly, if you use an application that is able to handle OAuth 2.0 tokens but just cannot retrieve them itself, then [pizauth](https://github.com/ltratt/pizauth), [mailctl](https://github.com/pdobsan/mailctl) or [oauth-helper-office-365](https://github.com/ahrex/oauth-helper-office-365) may be more appropriate. @@ -260,3 +254,7 @@ This proxy was developed to work around these limitations for providers that do ## License [Apache 2.0](https://github.com/simonrob/email-oauth2-proxy/blob/main/LICENSE) + + +--- +1. If you are packaging the GUI version of the proxy using pyinstaller, you may need to add `--hidden-import timeago.locales.en_short` until [this `timeago` issue](https://github.com/hustcc/timeago/issues/40) is resolved. diff --git a/emailproxy.config b/emailproxy.config index dde604f..71d1d28 100644 --- a/emailproxy.config +++ b/emailproxy.config @@ -18,7 +18,7 @@ documentation = Local servers are specified as demonstrated below where, for exa port must be above 1023 (unless the proxy script is run via sudo), below 65536, and unique across local servers. Multiple accounts can share the same server, however. Each server section must specify the `server_address` and `server_port` of the remote server that it will be proxying - you can obtain these values from your email provider, - or use the details below for Office 365 and/or Gmail. + or use the details below (examples are given for Office 365 / Outlook and Gmail). To allow the proxy to operate, your email client must be set up to use an unencrypted connection for IMAP/SMTP/POP (i.e., no STARTTLS or SSL/TLS, just plain login credentials). The proxy will create a secure connection on your @@ -48,17 +48,26 @@ documentation = Local servers are specified as demonstrated below where, for exa /etc/letsencrypt/live/mail.example.net/privkey.pem) for the server you are using the proxy with, and it will use these to set up a secure connection between itself and your email client. + Advanced feature - proxy plugins: + - Plugins are an advanced feature that enable the use of separate scripts to modify IMAP/POP/SMTP commands when they + are received from the client or server before passing through to the other side of the connection. For more details + about how to install and enable this feature, see the additional documentation and range of sample plugins explained + at https://github.com/simonrob/email-oauth2-proxy/tree/plugins/plugins#email-oauth-20-proxy-plugins + [IMAP-1993] +documentation = *** note: this server will work for both Office 365 and personal Outlook/Hotmail accounts *** server_address = outlook.office365.com server_port = 993 local_address = 127.0.0.1 [POP-1995] +documentation = *** note: this server will work for both Office 365 and personal Outlook/Hotmail accounts *** server_address = outlook.office365.com server_port = 995 local_address = 127.0.0.1 [SMTP-1587] +documentation = *** note: this server will work for both Office 365 and personal Outlook/Hotmail accounts *** server_address = smtp-mail.outlook.com server_port = 587 server_starttls = True @@ -87,9 +96,9 @@ documentation = Accounts are specified using your email address as the section h OAuth 2.0 flow you are using, other values may also be required (see examples below). If you are adding an account for a service other than the examples shown below then the provider's documentation should provide these details. - You will also need to add your own `client_id` and `client_secret` values as indicated below. These can either be - reused from an existing source (such as another email client that supports OAuth 2.0), or you can register and use - your own desktop app API client credentials. See https://developers.google.com/identity/protocols/oauth2/native-app + You will also need to add your own `client_id` and (in many cases) `client_secret` values as indicated below. These + can either be reused from an existing source (such as an email client that supports OAuth 2.0), or you can register + your own desktop app client credentials. See https://developers.google.com/identity/protocols/oauth2/native-app and the Microsoft link below for details. Multiple accounts on the same server can use the same values for the `client_id` and `client_secret` properties; just duplicate these in each account's entry below, or see the advanced `allow_catch_all_accounts` option. Note that while there are example account configurations for AOL and Yahoo Mail @@ -99,8 +108,9 @@ documentation = Accounts are specified using your email address as the section h Once the proxy is correctly configured, after the first successful use of an account its access token details will be cached for future use, encrypted with the IMAP/POP/SMTP password you used in your email client. By default this configuration file is reused for caching (so it must be writable), but you can specify a different location or - method using the proxy's `--cache-store` parameter. You should not add or edit cached values manually (i.e., - `token_salt`, `access_token`, `access_token_expiry`, `refresh_token` and `last_activity`); the proxy handles this. + method using the proxy's `--cache-store` parameter. See below for advanced use of this option to integrate with a + secrets manager service. You should not add or edit cached values manually (i.e.,`token_salt`, `token_iterations`, + `access_token`, `access_token_expiry`, `refresh_token` and `last_activity`); the proxy handles this. The password used in your email client is not used for authentication with the actual email server (this is done via OAuth 2.0 in a web browser), so it can be different to your real account password, which is helpful for debugging. @@ -108,33 +118,34 @@ documentation = Accounts are specified using your email address as the section h password to avoid repeated re-authentication requests (which is the proxy's default behaviour when credential decryption fails). See the proxy's README.md file for more information and the end of this file for further options. - Office 365 customisation: - - Unlike other providers, Office 365 requires an OAuth 2.0 scope that explicitly specifies `offline_access` (shown - in the example below) in order to allow the proxy to refresh its access token on your behalf. The proxy will still - work if this parameter is not included, but you will need to re-authenticate extremely often (about once per hour). + Office 365 / Outlook customisation: + - Unlike other providers, Office 365 / Outlook requires an OAuth 2.0 scope that explicitly grants `offline_access` + (shown in the examples below) in order to allow the proxy to refresh its access token on your behalf. The proxy will + still without this parameter, but you will need to re-authenticate extremely often (about once per hour). - - The example Office 365 configuration entries below use an OAuth 2.0 scope that clearly specifies IMAP, POP and - SMTP permission. If you do not require one or more of these protocols, you may remove the relevant values to ensure - the access tokens obtained on your behalf are as precisely-targeted as possible. Conversely, it is also possible to - replace these specific scopes with the more generic `https://outlook.office365.com/.default`. Switching to a broader - scope value may also be needed if you are using a version of O365 delivered by a regional provider (e.g., 21Vianet). - See: https://github.com/simonrob/email-oauth2-proxy/issues/255 for more details and discussion. + - The example Office 365 / Outlook configuration entries below use an OAuth 2.0 scope that clearly specifies IMAP, + POP and SMTP permission. If you do not require one or more of these protocols, you may remove the relevant values to + ensure access tokens obtained on your behalf are as precisely-targeted as possible. Conversely, it is also possible + to replace these specific scopes with the more generic `https://outlook.office365.com/.default`. Switching to a + broader scope value may also be needed if you are using Microsoft services delivered by a regional provider (e.g., + 21Vianet). See: https://github.com/simonrob/email-oauth2-proxy/issues/255 for more details and discussion. - By default, new Entra (Azure AD) clients are accessible only within your own tenant. If you are registering a new client to use with the proxy (and do not want to make it available outside your own organisation) you will need to - replace `common` with your tenant ID in the Office 365 `permission_url` and `token_url` values below. Alternatively, - you can reuse credentials from an existing client registration (see the proxy's README.md file), or configure your - client as a multi-tenant application. For more detail about this, and guides for setting up your desktop app client, - see the documentation at https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app. + replace `common` with your tenant ID in the `permission_url` and `token_url` values below. Alternatively, you can + reuse credentials from an existing client registration (see the proxy's README.md file), or configure your client as + a multi-tenant application. For more detail about this, and guides for setting up your desktop app client, see the + documentation at https://learn.microsoft.com/en-us/entra/identity-platform/quickstart-register-app. - Office 365 shared mailboxes are supported: add an account entry here using the email address of the shared mailbox as the account name. When asked to authenticate, log in as the user that access has been delegated to. - Note that Office 365 no-longer supports the `authorised.user@example.com/delegated.mailbox` username syntax here. + Note that Office 365 no-longer supports the `authorised.user@example.com/delegated.mailbox` username syntax. - - It is possible to create Office 365 clients that do not require a secret to be sent. If this is the case for your - setup, delete the `client_secret` line from your account's configuration entry (do not leave the default value). + - It is possible to create Office 365 / Outlook OAuth 2.0 clients that do not require a secret to be sent. If this + is the case for your setup, delete the `client_secret` line from your account's configuration entry (do not leave + the default value). - - To use O365 certificate credentials instead of a client secret, delete the `client_secret` line and instead + - To use Office 365 certificate credentials instead of a client secret, delete the `client_secret` line and instead provide a `jwt_certificate_path` (e.g., /path/to/certificate.pem) and `jwt_key_path` (e.g., /path/to/key.pem). Further documentation and examples can be found at https://github.com/simonrob/email-oauth2-proxy/pull/247. @@ -147,17 +158,18 @@ documentation = Accounts are specified using your email address as the section h - WARNING: Please note that by default the CCG flow has essentially no local access control when creating new accounts (no user consent is involved, so the proxy cannot validate login attempts unless an account entry - already exists its configuration file). Using the CCG flow with the proxy in a publicly-accessible context is - not advised. This is especially important when using the proxy's catch-all feature (which is likely to be the - case given the typical use-cases for the CCG flow). Because of this, you are highly encouraged to enable the - proxy's secret encryption option - see `encrypt_client_secret_on_first_use` at the end of this file. In + already exists in its configuration file). Using the CCG flow with the proxy in a publicly-accessible context + is not advised. This is especially important when using the proxy's catch-all feature (which is likely to be + the case given the typical use-cases for the CCG flow). Because of this, you are highly encouraged to enable + the proxy's secret encryption option - see `encrypt_client_secret_on_first_use` at the end of this file. In addition, if you are using the proxy in an environment where there is any possibility of malicious access attempts before the first valid login, pre-encrypting account entries is highly recommended. See the example script at https://github.com/simonrob/email-oauth2-proxy/issues/61#issuecomment-1259110336. - - The proxy supports the device authorisation grant (DAG) OAuth 2.0 flow (RFC 8628), which may better suit headless - systems. To use this flow, set `oauth2_flow = device`. With this flow, the proxy receives authorisation responses - directly from the service provider, so no `redirect_uri` is needed. An example account configuration is given below. + - The proxy supports the device authorisation grant (DAG) OAuth 2.0 flow, which may better suit headless systems + (currently only known to be available for Office 365 / Outlook). To use this flow, set `oauth2_flow = device`. With + this flow, the proxy receives authorisation responses directly from the service provider, so no `redirect_uri` is + needed. An example account configuration is given below. Gmail customisation: - The proxy supports the use of service accounts with Gmail for Google Workspace (note: normal Gmail accounts do not @@ -168,11 +180,13 @@ documentation = Accounts are specified using your email address as the section h full path to the JSON key file. To include the key directly, set `client_id = key`, then paste the full contents of your service account's JSON key as the value for `client_secret`, making sure all lines are indented by at least one space (so that the proxy can tell they are all part of one value). An example is given for both methods towards the - end of the sample account entries below. + end of the sample account entries below. Note that when creating the account entry here and when logging in from an + email client, the username you should use is the account you are trying to access, not the service account user + (i.e., do not use the auto-generated address that is similar to `your-user@your-project.iam.gserviceaccount.com`). - - WARNING: Please note that the same potential security issues outlined above with O365's CCG flow also apply to - the service account method: there is essentially no local access control when creating new accounts. Using a - service account with the proxy in a publicly-accessible context is not advised. You are highly encouraged to + - WARNING: Please note that the same potential security issues outlined above with O365's CCG flow also apply + to the service account method: there is essentially no local access control when creating new accounts. Using + a service account with the proxy in a publicly-accessible context is not advised. You are highly encouraged to enable the proxy's secret encryption option (see `encrypt_client_secret_on_first_use` at the end of this file) and consider pre-encrypting account entries. A sample pre-encryption script is provided for reference at https://github.com/simonrob/email-oauth2-proxy/issues/212#issuecomment-1867557029 @@ -181,7 +195,7 @@ documentation = Accounts are specified using your email address as the section h - For most configurations the default `redirect_uri` value of `http://localhost` is correct, unless you have explicitly set the OAuth 2.0 client configuration with your provider to use a different address for this purpose (e.g., redirecting via an external domain). If this is the case, you will need to manually redirect this to the - proxy. Please note that in most cases the address is indeed `http://localhost`, not `https`. + proxy. Please note that in most cases the address is indeed `http://`, not `https://`. - When using the `--local-server-auth` option you will need to either run the proxy with additional privileges to use the implicit default port 80 (e.g., via sudo) or specify a different port (and/or a different host if needed) - @@ -195,20 +209,25 @@ documentation = Accounts are specified using your email address as the section h the same address. To avoid clashes, it is recommended that each account has a unique `redirect_uri` (or `redirect_listen_address`) value, for example by using a different port for each account. -[your.office365.address@example.com] + Integration with a secrets manager: + - The proxy caches authenticated OAuth 2.0 tokens and associated metadata back into this configuration file by + default, but can alternatively be configured to use either a separate local file (via `--cache-store /path/to/file`) + or a secrets manager service for remote token storage. Currently only AWS Secrets Manager is supported. To use this + feature, set the proxy's `--cache-store` parameter to either a full AWS ARN or a secret name, prefixing the value + with `aws:` to identify its type to the proxy. If not already present, you must also install the AWS SDK for Python + (`python -m pip install boto3`) and set up authentication credentials (including a region) - see the documentation + at https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html#configuration. The minimum required + permissions for the associated AWS IAM user are `secretsmanager:GetSecretValue` and `secretsmanager:PutSecretValue`. + If the named AWS Secret does not yet exist, the proxy will attempt to create it; in this case, the permission + `secretsmanager:CreateSecret` is also required. + +[your.office365.or.outlook.address@example.com] permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/authorize token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access redirect_uri = http://localhost client_id = *** your client id here *** -client_secret = *** your client secret here *** - -[your.free.outlook.or.hotmail.address@outlook.com] -permission_url = https://login.microsoftonline.com/common/oauth2/v2.0/authorize -token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token -oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access -client_id = *** your client id here - note that as you are not the administrator of Hotmail.com / Outlook.com, you will likely need to reuse an existing client ID (see the proxy's readme) *** -redirect_uri = https://localhost +client_secret = *** your client secret here (remove this entire line if a secret is not required) *** [your.email@gmail.com] permission_url = https://accounts.google.com/o/oauth2/auth @@ -240,6 +259,7 @@ token_url = https://login.microsoftonline.com/common/oauth2/v2.0/token oauth2_scope = https://outlook.office.com/IMAP.AccessAsUser.All https://outlook.office.com/POP.AccessAsUser.All https://outlook.office.com/SMTP.Send offline_access oauth2_flow = device client_id = *** your client id here *** +client_secret = *** your client secret here (remove this entire line if a secret is not required) *** [ccg.flow.configured.address@your-tenant.com] documentation = *** note: this is an advanced O365 account example; in most cases you want the version above instead *** @@ -247,7 +267,7 @@ token_url = https://login.microsoftonline.com/*** your tenant id here ***/oauth2 oauth2_scope = https://outlook.office365.com/.default oauth2_flow = client_credentials client_id = *** your client id here *** -client_secret = *** your client secret here *** +client_secret = *** your client secret here (remove this entire line if a secret is not required) *** [ropcg.flow.configured.address@your-tenant.com] documentation = *** note: this is an advanced O365 account example; in most cases you want the version above instead *** @@ -255,7 +275,7 @@ token_url = https://login.microsoftonline.com/*** your tenant id here ***/oauth2 oauth2_scope = https://outlook.office365.com/IMAP.AccessAsUser.All https://outlook.office365.com/POP.AccessAsUser.All https://outlook.office365.com/SMTP.Send offline_access oauth2_flow = password client_id = *** your client id here *** -client_secret = *** your client secret here *** +client_secret = *** your client secret here (remove this entire line if a secret is not required) *** [service.account.accessible.address@your-domain.com] documentation = *** note: this is an advanced Google account example; in most cases you want the version above instead ***