From a06b3d78fa7c5b4a855199b823ca82f1a4caab95 Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Tue, 30 Jul 2024 09:42:51 -0400 Subject: [PATCH 01/29] remove config file, use env vars --- vtt/engine.py | 92 ++++++++++++++++++++----------------------- vtt/utils/path_api.py | 3 -- 2 files changed, 42 insertions(+), 53 deletions(-) diff --git a/vtt/engine.py b/vtt/engine.py index d124bb3..53a073b 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -56,11 +56,11 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): # webserver stuff self.listen = '0.0.0.0' self.hosting = { - 'domain' : 'localhost', - 'port' : 8080, - 'socket' : '', - 'ssl' : False, - 'reverse' : False + "domain" : os.getenv('VTT_DOMAIN', 'localhost'), + "port" : os.getenv('VTT_PORT', 8080), + "socket" : os.getenv('VTT_SOCKET', ""), + "ssl" : os.getenv('VTT_SSL', False), + "reverse" : os.getenv('VTT_REVERSE_PROXY', False) } self.shards = list() @@ -72,27 +72,39 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): # maximum file sizes for uploads (in MB) self.file_limit = { - "token" : 2, - "background" : 10, - "game" : 30, - "music" : 10, + "token" : os.getenv('VTT_LIMIT_TOKEN', 2), + "background" : os.gentenv('VTT_LIMIT_BG', 10), + "game" : os.getenv('VTT_LIMIT_GAME', 20), + "music" : os.getenv('VTT_LIMIT_MUSIC', 10) "num_music" : 5 } - self.playercolors = ['#FF0000', '#00FF00', '#0000FF', '#FFFF00', '#FF00FF'] + self.playercolors = ['#FF0000', '#00FF00', '#0000FF', '#FFFF00', '#FF00FF', "#C52828", "#13AA4F", "#ECBC15", "#7F99C7", "#9251B7", "#797A90", "#80533F", "#21A0B7"] - self.title = appname + self.title = os.getenv('VTT_TITLE', appname) self.links = list() self.cleanup = { - 'expire': 3600 * 24 * 30, # default: 30d - 'daytime': '03:00' + 'expire': os.getenv('VTT_CLEANUP_EXPIRE', 2592000), + 'daytime': os.getenv('VTT_CLEANUP_TIME', '03:00') } self.login = dict() # login settings - self.login['type'] = '' + self.login['type'] = os.getenv('VTT_LOGIN_TYPE', '') self.login['providers']: dict[str, str] = {} self.login_api = None # login api instance self.notify = dict() # crash notify settings - self.notify['type'] = '' + self.notify['type'] = os.getenv('VTT_NOTIFY_TYPE', '') self.notify_api = None # notify api instance + # email notification configuration + self.notify['host'] = os.gentenv('VTT_NOTIFY_HOST'), + self.notify['port'] = os.gentenv('VTT_NOTIFY_PORT'), + self.notify['sender'] = os.getenv('VTT_NOTIFY_SENDER'), + self.notify['user'] = os.getenv('VTT_NOTIFY_USER'), + self.notify['password'] = os.getenv('VTT_NOTIFY_PASS'), + # Discord webhook notification configuration + self.notify['provider'] = os.getenv('VTT_NOTIFY_PROVIDER'), + self.notify['alias'] = os.getenv('VTT_NOTIFY_ALIAS'), + self.notify['url'] = os.getenv('VTT_NOTIFY_URL'), + self.notify['roles'] = os.getenv('VTT_NOTIFY_ROLES'), + self.notify['users'] = os.getenv('VTT_NOTIFY_USERS') self.cache = None # later engine cache @@ -120,41 +132,21 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.url_generator = utils.FancyUrlApi(self.paths) # handle settings - settings_path = self.paths.get_settings_path() - if not os.path.exists(settings_path): - # create default settings - settings = { - 'title' : self.title, - 'cleanup' : self.cleanup, - 'links' : self.links, - 'file_limit' : self.file_limit, - 'playercolors' : self.playercolors, - 'shards' : self.shards, - 'hosting' : self.hosting, - 'login' : self.login, - 'notify' : self.notify - } - with open(settings_path, 'w') as h: - json.dump(settings, h, indent=4) - self.logging.info('Created default settings file') - else: - # load settings - with open(settings_path, 'r') as h: - settings = json.load(h) - self.title = settings['title'] - self.cleanup = settings['cleanup'] - self.links = settings['links'] - self.file_limit = settings['file_limit'] - self.playercolors = settings['playercolors'] - self.shards = settings['shards'] - self.hosting = settings['hosting'] - self.login = settings['login'] - self.notify = settings['notify'] - self.logging.info('Settings loaded') - - # add this server to the shards list - self.shards.append(self.get_url()) - + settings = { + 'login' : { + "type" : "auth0", + "domain" : os.getenv('VTT_AUTH0_DOMAIN', "YOUR_APP.us.auth0.com"), + "client_id" : os.getenv('VTT_AUTH0_ID', "YOUR_APP_ID"), + "client_secret" : os.getenv('VTT_AUTH0_SECRET', "YOUR_APP_SECRET"), + "icons": { + "google": "https://www.google.com/favicon.ico", + "discord": "https://assets-global.website-files.com/6257adef93867e50d84d30e2/6266bc493fb42d4e27bb8393_847541504914fd33810e70a0ea73177e.ico", + "patreon": "https://c5.patreon.com/external/favicon/favicon.ico", + "auth0": "https://cdn.auth0.com/website/new-homepage/dark-favicon.png" + } + }, + } + # show argv help if '--help' in argv: print('Commandline options:') diff --git a/vtt/utils/path_api.py b/vtt/utils/path_api.py index add0471..a56ea23 100644 --- a/vtt/utils/path_api.py +++ b/vtt/utils/path_api.py @@ -58,9 +58,6 @@ def get_client_code_path(self) -> pathlib.Path: def get_log_path(self, filename: str) -> pathlib.Path: return self.pref_root / '{0}.log'.format(filename) - def get_settings_path(self) -> pathlib.Path: - return self.pref_root / 'settings.json' - def get_main_database_path(self) -> pathlib.Path: return self.pref_root / 'main.db' From cdfb8e6197b932ef4f5ed8ae822c1c1050ef0b23 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 30 Jul 2024 17:27:21 +0200 Subject: [PATCH 02/29] fixing oauth setup detection --- .gitignore | 5 +++ vtt/engine.py | 76 ++++++++++++++++++++++---------------- vtt/utils/auth/__init__.py | 2 +- vtt/utils/auth/factory.py | 27 ++++++++++++++ 4 files changed, 77 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index 9a3e03f..fa4b621 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,13 @@ sha.txt htmlcov/ +run.sh + +# VSCode +pyvtt.code-workspace # PyCharm .idea +.code-workspace # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/vtt/engine.py b/vtt/engine.py index 53a073b..35cf056 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -57,10 +57,10 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.listen = '0.0.0.0' self.hosting = { "domain" : os.getenv('VTT_DOMAIN', 'localhost'), - "port" : os.getenv('VTT_PORT', 8080), + "port" : int(os.getenv('VTT_PORT', 8080)), "socket" : os.getenv('VTT_SOCKET', ""), - "ssl" : os.getenv('VTT_SSL', False), - "reverse" : os.getenv('VTT_REVERSE_PROXY', False) + "ssl" : bool(os.getenv('VTT_SSL', False)), + "reverse" : bool(os.getenv('VTT_REVERSE_PROXY', False)) } self.shards = list() @@ -72,39 +72,58 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): # maximum file sizes for uploads (in MB) self.file_limit = { - "token" : os.getenv('VTT_LIMIT_TOKEN', 2), - "background" : os.gentenv('VTT_LIMIT_BG', 10), - "game" : os.getenv('VTT_LIMIT_GAME', 20), - "music" : os.getenv('VTT_LIMIT_MUSIC', 10) - "num_music" : 5 + "token" : int(os.getenv('VTT_LIMIT_TOKEN', 2)), + "background" : int(os.getenv('VTT_LIMIT_BG', 10)), + "game" : int(os.getenv('VTT_LIMIT_GAME', 20)), + "music" : int(os.getenv('VTT_LIMIT_MUSIC', 10)), + "num_music" : int(os.getenv('VTT_NUM_NUSIC', 5)) } self.playercolors = ['#FF0000', '#00FF00', '#0000FF', '#FFFF00', '#FF00FF', "#C52828", "#13AA4F", "#ECBC15", "#7F99C7", "#9251B7", "#797A90", "#80533F", "#21A0B7"] self.title = os.getenv('VTT_TITLE', appname) self.links = list() self.cleanup = { - 'expire': os.getenv('VTT_CLEANUP_EXPIRE', 2592000), + 'expire': int(os.getenv('VTT_CLEANUP_EXPIRE', 2592000)), 'daytime': os.getenv('VTT_CLEANUP_TIME', '03:00') } + + self.login_api = None # login api instance + """ + # FIXME: deprecated self.login = dict() # login settings self.login['type'] = os.getenv('VTT_LOGIN_TYPE', '') self.login['providers']: dict[str, str] = {} - self.login_api = None # login api instance + + if self.login['type'] == 'auth0': + self.login['domain'] = os.environ['VTT_AUTH0_DOMAIN'] + self.login['client_id'] = os.environ['VTT_AUTH0_ID'] + self.login['client_secret'] = os.environ['VTT_AUTH0_SECRET'] + self.login['icons'] = { + "google": "https://www.google.com/favicon.ico", + "discord": "https://assets-global.website-files.com/6257adef93867e50d84d30e2/6266bc493fb42d4e27bb8393_847541504914fd33810e70a0ea73177e.ico", + "patreon": "https://c5.patreon.com/external/favicon/favicon.ico", + "auth0": "https://cdn.auth0.com/website/new-homepage/dark-favicon.png" + } + """ + self.notify = dict() # crash notify settings self.notify['type'] = os.getenv('VTT_NOTIFY_TYPE', '') self.notify_api = None # notify api instance + """ + # FIXME: deprecated # email notification configuration - self.notify['host'] = os.gentenv('VTT_NOTIFY_HOST'), - self.notify['port'] = os.gentenv('VTT_NOTIFY_PORT'), + self.notify['host'] = os.getenv('VTT_NOTIFY_HOST'), + self.notify['port'] = int(os.getenv('VTT_NOTIFY_PORT')), self.notify['sender'] = os.getenv('VTT_NOTIFY_SENDER'), self.notify['user'] = os.getenv('VTT_NOTIFY_USER'), self.notify['password'] = os.getenv('VTT_NOTIFY_PASS'), + """ # Discord webhook notification configuration self.notify['provider'] = os.getenv('VTT_NOTIFY_PROVIDER'), self.notify['alias'] = os.getenv('VTT_NOTIFY_ALIAS'), self.notify['url'] = os.getenv('VTT_NOTIFY_URL'), - self.notify['roles'] = os.getenv('VTT_NOTIFY_ROLES'), - self.notify['users'] = os.getenv('VTT_NOTIFY_USERS') + self.notify['roles'] = os.getenv('VTT_NOTIFY_ROLES', []), + self.notify['users'] = [os.getenv('VTT_NOTIFY_USERS')] self.cache = None # later engine cache @@ -131,22 +150,6 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): # load fancy url generator api ... lol self.url_generator = utils.FancyUrlApi(self.paths) - # handle settings - settings = { - 'login' : { - "type" : "auth0", - "domain" : os.getenv('VTT_AUTH0_DOMAIN', "YOUR_APP.us.auth0.com"), - "client_id" : os.getenv('VTT_AUTH0_ID', "YOUR_APP_ID"), - "client_secret" : os.getenv('VTT_AUTH0_SECRET', "YOUR_APP_SECRET"), - "icons": { - "google": "https://www.google.com/favicon.ico", - "discord": "https://assets-global.website-files.com/6257adef93867e50d84d30e2/6266bc493fb42d4e27bb8393_847541504914fd33810e70a0ea73177e.ico", - "patreon": "https://c5.patreon.com/external/favicon/favicon.ico", - "auth0": "https://cdn.auth0.com/website/new-homepage/dark-favicon.png" - } - }, - } - # show argv help if '--help' in argv: print('Commandline options:') @@ -193,9 +196,18 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): # create email notify API self.notify_api = utils.EmailApi(self, appname=appname, **self.notify) - if self.login is not None and self.login['type'] == 'oauth': + # collect provider data from environ vars + providers = {} + for api_name in utils.auth.SUPPORTED_LOGIN_APIS: + data = utils.parse_provider_data(api_name, os.environ) + if data is not None: + providers[api_name] = data + + self.logging.info(f'Found oauth setup for: {[api_name for api_name in providers]}') + + if len(providers) > 1: self.login_api = utils.OAuthClient(on_auth=self.logging.auth, callback_url=self.get_auth_callback_url(), - providers=self.login['providers']) + providers=providers) self.logging.info('Loading main database...') # create main database diff --git a/vtt/utils/auth/__init__.py b/vtt/utils/auth/__init__.py index 7f2b21a..4226b12 100644 --- a/vtt/utils/auth/__init__.py +++ b/vtt/utils/auth/__init__.py @@ -2,4 +2,4 @@ from .discord import DiscordLogin from .oauth_client import OAuthClient from .common import BaseLoginApi -from .factory import SUPPORTED_LOGIN_APIS +from .factory import SUPPORTED_LOGIN_APIS, parse_provider_data diff --git a/vtt/utils/auth/factory.py b/vtt/utils/auth/factory.py index e91d491..f0e9309 100644 --- a/vtt/utils/auth/factory.py +++ b/vtt/utils/auth/factory.py @@ -21,3 +21,30 @@ def create_login_api(client: LoginClient, on_auth: AuthHandle, api_name: str, ca """Figure out Login API implementation and create an instance.""" class_ = SUPPORTED_LOGIN_APIS[api_name] return class_(client=client, on_auth=on_auth, callback_url=callback_url, **data) + + +def get_icon_url(api_name: str) -> str: + """Return url to the service's icon.""" + if api_name == 'google': + return 'https://www.google.com/favicon.ico' + + if api_name == 'discord': + return 'https://assets-global.website-files.com/6257adef93867e50d84d30e2/6266bc493fb42d4e27bb8393_847541504914fd33810e70a0ea73177e.ico' + + raise NotImplementedError(f'{api_name} is not supported as OAuth mechanism') + + +def parse_provider_data(api_name: str, data: dict) -> dict | None: + """Try to parse provider data from a dict (e.g. os.environ).""" + id_key = f'VTT_OAUTH_{api_name.upper()}_ID' + secret_key = f'VTT_OAUTH_{api_name.upper()}_SECRET' + + if id_key not in data or secret_key not in data: + # API not provided + return None + + return { + 'client_id': data[id_key], + 'client_secret': data[secret_key], + 'icon_url': get_icon_url(api_name) + } From 63cd3d7726c8d52489dad51c45ba32684bf4fdc3 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 30 Jul 2024 17:37:58 +0200 Subject: [PATCH 03/29] fix: now working with at least one oauth setup --- vtt/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtt/engine.py b/vtt/engine.py index 35cf056..d5e4701 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -205,7 +205,7 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.logging.info(f'Found oauth setup for: {[api_name for api_name in providers]}') - if len(providers) > 1: + if len(providers) >= 1: self.login_api = utils.OAuthClient(on_auth=self.logging.auth, callback_url=self.get_auth_callback_url(), providers=providers) From d0168ee9b472227bd0cd637a6c8b04a1c48b34a6 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 30 Jul 2024 17:45:13 +0200 Subject: [PATCH 04/29] added support for VTT_LINKS_DISCORD and _FAQ --- vtt/engine.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/vtt/engine.py b/vtt/engine.py index d5e4701..f809c66 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -80,8 +80,33 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): } self.playercolors = ['#FF0000', '#00FF00', '#0000FF', '#FFFF00', '#FF00FF', "#C52828", "#13AA4F", "#ECBC15", "#7F99C7", "#9251B7", "#797A90", "#80533F", "#21A0B7"] - self.title = os.getenv('VTT_TITLE', appname) - self.links = list() + self.title = os.getenv('VTT_TITLE', appname) + + self.links = [ + { + "label" : "HOME", + "url" : "/" + }, { + "label" : "STATUS", + "url" : "/vtt/shard" + }, { + "label" : "ROADMAP", + "url" : "/static/roadmap.html" + }, { + "label" : "TERMS", + "url" : "/static/terms.html" + } + ] + + for category in ['discord', 'faq']: + key = f'VTT_LINKS_{category.upper()}' + value = os.getenv(key) + if value is not None: + self.links.append({ + 'label': category.upper(), + 'url': value + }) + self.cleanup = { 'expire': int(os.getenv('VTT_CLEANUP_EXPIRE', 2592000)), 'daytime': os.getenv('VTT_CLEANUP_TIME', '03:00') From a30ee71cad75b6d435660c73d834ff67d6d58355 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 30 Jul 2024 17:54:57 +0200 Subject: [PATCH 05/29] wip: made links with shards optional but work... somehow hacky --- vtt/engine.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/vtt/engine.py b/vtt/engine.py index f809c66..093ecad 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -86,9 +86,6 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): { "label" : "HOME", "url" : "/" - }, { - "label" : "STATUS", - "url" : "/vtt/shard" }, { "label" : "ROADMAP", "url" : "/static/roadmap.html" @@ -107,6 +104,16 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): 'url': value }) + num_shards = int(os.getenv('VTT_SHARDS_NUM', 0)) + self.shards = [os.getenv(f'VTT_SHARDS_{n}') for n in range(num_shards)] + print('SHARDS', self.shards) + if len(self.shards) > 0: + + self.links.append({ + "label" : "STATUS", + "url" : "/vtt/shard" + }) + self.cleanup = { 'expire': int(os.getenv('VTT_CLEANUP_EXPIRE', 2592000)), 'daytime': os.getenv('VTT_CLEANUP_TIME', '03:00') From e69e0a43b4e70d27aeeeb4ef559327e1800352ab Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 30 Jul 2024 18:09:03 +0200 Subject: [PATCH 06/29] fix: wehbooks added back in (but only one atm) --- vtt/engine.py | 35 ++++++++++++++++++++++++---------- vtt/utils/notifier/__init__.py | 1 + vtt/utils/notifier/common.py | 20 +++++++++++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/vtt/engine.py b/vtt/engine.py index 093ecad..9172e0e 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -138,10 +138,13 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): } """ + """ + # FIXME: deprecated self.notify = dict() # crash notify settings self.notify['type'] = os.getenv('VTT_NOTIFY_TYPE', '') self.notify_api = None # notify api instance """ + """ # FIXME: deprecated # email notification configuration self.notify['host'] = os.getenv('VTT_NOTIFY_HOST'), @@ -150,14 +153,18 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.notify['user'] = os.getenv('VTT_NOTIFY_USER'), self.notify['password'] = os.getenv('VTT_NOTIFY_PASS'), """ + + """ + # FIXME: deprecated # Discord webhook notification configuration self.notify['provider'] = os.getenv('VTT_NOTIFY_PROVIDER'), self.notify['alias'] = os.getenv('VTT_NOTIFY_ALIAS'), self.notify['url'] = os.getenv('VTT_NOTIFY_URL'), self.notify['roles'] = os.getenv('VTT_NOTIFY_ROLES', []), self.notify['users'] = [os.getenv('VTT_NOTIFY_USERS')] + """ - self.cache = None # later engine cache + self.cache = None # later engine cache # handle commandline arguments self.localhost = '--localhost' in argv @@ -215,19 +222,27 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.hosting['domain'] = ip self.logging.info(f'Using Public IP {ip} as Domain') - # FIXME: use factory pattern - if self.notify is not None and not self.debug: - if self.notify['type'] == 'webhook': - if self.notify['provider'] == 'discord': - app_title = f'{self.title} on {self.get_domain()}' - self.notify_api = utils.DiscordWebhook(app_title=app_title, alias=self.notify['alias'], - url=self.notify['url'], roles=self.notify['roles'], - users=self.notify['users']) + webhooks = {} + for api_name in utils.notifier.SUPPORTED_APIS: + data = utils.parse_webhook_data(api_name, os.environ) + if data is not None: + webhooks[api_name] = data + self.logging.info(f'Found webhook setup for: {[api_name for api_name in webhooks]}') + + if len(webhooks) >= 1: + # FIXME: using the next best webhook API does not allow multiple webhooks + app_title = f'{self.title} on {self.get_domain()}' + first_hook = webhooks[list(webhooks.keys())[0]] + self.notify_api = utils.DiscordWebhook(app_title=app_title, alias=app_title, url=first_hook['url'], roles=[], users=first_hook['users']) + + """ + # FIXME: deprecated if self.notify['type'] == 'email': # create email notify API self.notify_api = utils.EmailApi(self, appname=appname, **self.notify) - + """ + # collect provider data from environ vars providers = {} for api_name in utils.auth.SUPPORTED_LOGIN_APIS: diff --git a/vtt/utils/notifier/__init__.py b/vtt/utils/notifier/__init__.py index da4daef..4fc7a37 100644 --- a/vtt/utils/notifier/__init__.py +++ b/vtt/utils/notifier/__init__.py @@ -1,2 +1,3 @@ from .discord import DiscordWebhook from .email import EmailApi +from .common import SUPPORTED_APIS, parse_webhook_data diff --git a/vtt/utils/notifier/common.py b/vtt/utils/notifier/common.py index 32e3bae..fa70184 100644 --- a/vtt/utils/notifier/common.py +++ b/vtt/utils/notifier/common.py @@ -11,8 +11,28 @@ import typing +SUPPORTED_APIS: list[str] = { + 'discord' +} + + class Notifier(typing.Protocol): def login(self) -> None: ... def on_start(self) -> None: ... def on_cleanup(self, report: any) -> None: ... def on_error(self, error_id: str, message: str) -> None: ... + + +def parse_webhook_data(api_name: str, data: dict) -> dict | None: + """Try to parse provider data from a dict (e.g. os.environ).""" + url_key = f'VTT_WEBHOOK_{api_name.upper()}_URL' + user_key = f'VTT_WEBHOOK_{api_name.upper()}_USER' + + if url_key not in data or user_key not in data: + # API not provided + return None + + return { + 'url': data[url_key], + 'users': [data[user_key]] + } From 496ba2159ed5d02ecd8d820249da7eabe97ee20e Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 30 Jul 2024 18:18:51 +0200 Subject: [PATCH 07/29] minor cleanup --- vtt/engine.py | 133 +++++++++++++++++--------------------------------- 1 file changed, 44 insertions(+), 89 deletions(-) diff --git a/vtt/engine.py b/vtt/engine.py index 9172e0e..925c59b 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -106,7 +106,6 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): num_shards = int(os.getenv('VTT_SHARDS_NUM', 0)) self.shards = [os.getenv(f'VTT_SHARDS_{n}') for n in range(num_shards)] - print('SHARDS', self.shards) if len(self.shards) > 0: self.links.append({ @@ -120,50 +119,6 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): } self.login_api = None # login api instance - """ - # FIXME: deprecated - self.login = dict() # login settings - self.login['type'] = os.getenv('VTT_LOGIN_TYPE', '') - self.login['providers']: dict[str, str] = {} - - if self.login['type'] == 'auth0': - self.login['domain'] = os.environ['VTT_AUTH0_DOMAIN'] - self.login['client_id'] = os.environ['VTT_AUTH0_ID'] - self.login['client_secret'] = os.environ['VTT_AUTH0_SECRET'] - self.login['icons'] = { - "google": "https://www.google.com/favicon.ico", - "discord": "https://assets-global.website-files.com/6257adef93867e50d84d30e2/6266bc493fb42d4e27bb8393_847541504914fd33810e70a0ea73177e.ico", - "patreon": "https://c5.patreon.com/external/favicon/favicon.ico", - "auth0": "https://cdn.auth0.com/website/new-homepage/dark-favicon.png" - } - """ - - """ - # FIXME: deprecated - self.notify = dict() # crash notify settings - self.notify['type'] = os.getenv('VTT_NOTIFY_TYPE', '') - self.notify_api = None # notify api instance - """ - """ - # FIXME: deprecated - # email notification configuration - self.notify['host'] = os.getenv('VTT_NOTIFY_HOST'), - self.notify['port'] = int(os.getenv('VTT_NOTIFY_PORT')), - self.notify['sender'] = os.getenv('VTT_NOTIFY_SENDER'), - self.notify['user'] = os.getenv('VTT_NOTIFY_USER'), - self.notify['password'] = os.getenv('VTT_NOTIFY_PASS'), - """ - - """ - # FIXME: deprecated - # Discord webhook notification configuration - self.notify['provider'] = os.getenv('VTT_NOTIFY_PROVIDER'), - self.notify['alias'] = os.getenv('VTT_NOTIFY_ALIAS'), - self.notify['url'] = os.getenv('VTT_NOTIFY_URL'), - self.notify['roles'] = os.getenv('VTT_NOTIFY_ROLES', []), - self.notify['users'] = [os.getenv('VTT_NOTIFY_USERS')] - """ - self.cache = None # later engine cache # handle commandline arguments @@ -206,7 +161,6 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): print(' --loglevel=') print(' Use as logging level') print('') - print('See {0} for custom settings.'.format(settings_path)) sys.exit(0) if self.localhost: @@ -222,39 +176,8 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.hosting['domain'] = ip self.logging.info(f'Using Public IP {ip} as Domain') - webhooks = {} - for api_name in utils.notifier.SUPPORTED_APIS: - data = utils.parse_webhook_data(api_name, os.environ) - if data is not None: - webhooks[api_name] = data - - self.logging.info(f'Found webhook setup for: {[api_name for api_name in webhooks]}') - - if len(webhooks) >= 1: - # FIXME: using the next best webhook API does not allow multiple webhooks - app_title = f'{self.title} on {self.get_domain()}' - first_hook = webhooks[list(webhooks.keys())[0]] - self.notify_api = utils.DiscordWebhook(app_title=app_title, alias=app_title, url=first_hook['url'], roles=[], users=first_hook['users']) - - """ - # FIXME: deprecated - if self.notify['type'] == 'email': - # create email notify API - self.notify_api = utils.EmailApi(self, appname=appname, **self.notify) - """ - - # collect provider data from environ vars - providers = {} - for api_name in utils.auth.SUPPORTED_LOGIN_APIS: - data = utils.parse_provider_data(api_name, os.environ) - if data is not None: - providers[api_name] = data - - self.logging.info(f'Found oauth setup for: {[api_name for api_name in providers]}') - - if len(providers) >= 1: - self.login_api = utils.OAuthClient(on_auth=self.logging.auth, callback_url=self.get_auth_callback_url(), - providers=providers) + self.init_webhooks() + self.init_oauth() self.logging.info('Loading main database...') # create main database @@ -277,13 +200,54 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.recent_rolls = 30 # rolls within past 30s are recent self.latest_rolls = 60 * 10 # rolls within the past 10min are up-to-date + self.git_hash = None + self.debug_hash = None + self.load_version_number() + + # export server constants to javascript-file + self.constants = utils.ConstantExport() + self.constants.load_from_engine(self) + self.constants.save_to_file(self.paths.get_constants_path()) + + # game cache + self.cache = EngineCache(self) + + def init_webhooks(self) -> None: + webhooks = {} + for api_name in utils.notifier.SUPPORTED_APIS: + data = utils.parse_webhook_data(api_name, os.environ) + if data is not None: + webhooks[api_name] = data + + self.logging.info(f'Found webhook setup for: {[api_name for api_name in webhooks]}') + + if len(webhooks) >= 1: + # FIXME: using the next best webhook API does not allow multiple webhooks + app_title = f'{self.title} on {self.get_domain()}' + first_hook = webhooks[list(webhooks.keys())[0]] + self.notify_api = utils.DiscordWebhook(app_title=app_title, alias=app_title, url=first_hook['url'], roles=[], users=first_hook['users']) + + def init_oauth(self) -> None: + # collect provider data from environ vars + providers = {} + for api_name in utils.auth.SUPPORTED_LOGIN_APIS: + data = utils.parse_provider_data(api_name, os.environ) + if data is not None: + providers[api_name] = data + + self.logging.info(f'Found oauth setup for: {[api_name for api_name in providers]}') + + if len(providers) >= 1: + self.login_api = utils.OAuthClient(on_auth=self.logging.auth, callback_url=self.get_auth_callback_url(), + providers=providers) + + def load_version_number(self) -> None: # load version number bn = BuildNumber() bn.load_from_file(self.paths.get_static_path(default=True) / 'client' / 'version.js') self.version = str(bn) # query latest git hash - self.git_hash = None try: with open('sha.txt') as h: self.git_hash = h.read().split('\n')[0] @@ -298,18 +262,9 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.git_hash = p.stdout.decode('utf-8').split('\n')[0] # generate debug hash - self.debug_hash = None if self.debug: self.debug_hash = uuid.uuid4().hex - # export server constants to javascript-file - self.constants = utils.ConstantExport() - self.constants.load_from_engine(self) - self.constants.save_to_file(self.paths.get_constants_path()) - - # game cache - self.cache = EngineCache(self) - def run(self): certfile = '' keyfile = '' From 4decdae5aa57a990d96e48ed173556fed405fa3b Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Sun, 19 Jan 2025 17:37:07 -0500 Subject: [PATCH 08/29] use cwd() instead of home() to support more OSes --- .github/workflows/python-app.yml | 4 ++-- .gitignore | 1 + requirements.txt | 1 - vtt/engine.py | 6 ++++-- vtt/utils/path_api.py | 11 +++++------ 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index f60c970..e84862a 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -20,9 +20,9 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.gitignore b/.gitignore index fa4b621..9019b39 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +pyvtt sha.txt htmlcov/ run.sh diff --git a/requirements.txt b/requirements.txt index a1398ad..da72277 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,6 @@ cryptography emoji-country-flag==1.3.1 gevent==23.9.0 gevent-websocket==0.10.1 -greenlet==2.0.1 httpagentparser==1.9.5 httplib2==0.21.0 idna==3.4 diff --git a/vtt/engine.py b/vtt/engine.py index 925c59b..7638f03 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -32,8 +32,9 @@ class Engine(object): def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): - appname = 'pyvtt' - self.log_level = 'INFO' + appname = os.getenv('VTT_APPNAME', 'pyvtt') + pref_dir = os.getenv('VTT_PREFDIR', None) + self.log_level = os.getenv('VTT_LOG_LEVEL', 'INFO') for arg in argv: if arg.startswith('--appname='): appname = arg.split('--appname=')[1] @@ -118,6 +119,7 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): 'daytime': os.getenv('VTT_CLEANUP_TIME', '03:00') } + self.notify_api = None # notify api instance self.login_api = None # login api instance self.cache = None # later engine cache diff --git a/vtt/utils/path_api.py b/vtt/utils/path_api.py index a56ea23..43eed35 100644 --- a/vtt/utils/path_api.py +++ b/vtt/utils/path_api.py @@ -20,12 +20,11 @@ def __init__(self, appname: str, pref_root: pathlib.Path | None = None, app_root """ Uses given root or pick standard preference directory. """ self.app_root = app_root if pref_root is None: - # get preference dir - pref_root = pathlib.Path.home() - if sys.platform.startswith('linux'): - pref_root = pref_root / ".local" / "share" - else: - raise NotImplementedError('only linux supported yet') + # use current working directory + pref_root = pathlib.Path.cwd() + else: + # we need to convert the string path value to a pathlib object + pref_root = pathlib.Path(pref_root) self.pref_root = pref_root / appname From 549a379bbb70287bae9624a6913f207ea1e8b416 Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Sun, 19 Jan 2025 18:58:35 -0500 Subject: [PATCH 09/29] bump version; WIP test fixes --- static/client/version.js | 2 +- test/test_engine.py | 129 +++++++++++++++------------------------ 2 files changed, 51 insertions(+), 80 deletions(-) diff --git a/static/client/version.js b/static/client/version.js index a688282..f99401e 100644 --- a/static/client/version.js +++ b/static/client/version.js @@ -1 +1 @@ -const version = "1.5.2"; \ No newline at end of file +const version = "1.6.0"; diff --git a/test/test_engine.py b/test/test_engine.py index 4ee197c..e0f9fc5 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -23,54 +23,24 @@ class EngineTest(EngineBaseTest): def tearDown(self): - fname = self.engine.paths.get_settings_path() - if os.path.exists(fname): - os.remove(fname) - super().tearDown() @staticmethod - def defaultSettings(): - return { - 'title' : 'unittest', - 'links' : [ - { 'label': 'FAQ', 'url': '/static/faq.html' }, - { 'label': 'LOGIN', 'url': '/vtt/join' } - ], - 'file_limit' : { - 'token' : 2, - 'background' : 10, - 'game' : 15, - 'music' : 10, - 'num_music' : 5 - }, - 'playercolors' : ['#FF0000', '#00FF00'], - 'shards' : list(), - 'cleanup': { - 'expire' : 3600, - 'daytime' : '03:00' - }, - 'hosting' : { - 'domain' : 'vtt.example.com', - 'port' : 80, - 'ssl' : False, - 'reverse' : True - }, - 'login' : { - 'type' : None - }, - 'notify' : { - 'type' : None - } - } - - def reloadEngine(self, argv=list(), settings=None): - if settings is not None: - # change settings - fname = self.engine.paths.get_settings_path() - with open(fname, 'w') as h: - json.dump(settings, h) - + def defaultEnviron(): + os.environ['VTT_TITLE'] = 'unittest' + os.environ['VTT_LIMIT_TOKEN'] =' 2' + os.environ['VTT_LIMIT_BG'] = '10' + os.environ['VTT_LIMIT_GAME'] = '5' + os.environ['VTT_LIMIT_MUSIC'] = '10' + os.environ['VTT_NUM_MUSIC'] = '5' + os.environ['VTT_CLEANUP_EXPIRE'] = '3600' + os.environ['VTT_CLEANUP_TIME'] = '03:00' + os.environ['VTT_DOMAIN'] = 'vtt.example.com' + os.environ['VTT_PORT'] = '8080' + os.environ['VTT_SSL'] = 'False' + os.environ['VTT_REVERSE_PROXY'] = 'True' + + def reloadEngine(self, argv=list()): # reload engine (without cleanup thread) argv.append('--quiet') self.engine = engine.Engine(argv=argv, pref_dir=self.root) @@ -97,76 +67,77 @@ def test_run(self): requests.get('http://localhost:8080') def test_getDomain(self): - settings = EngineTest.defaultSettings() - settings['hosting']['domain'] = 'example.com' - self.reloadEngine(settings=settings) + EngineTest.defaultEnviron() + os.environ['VTT_DOMAIN'] = 'example.com' + self.reloadEngine() domain = self.engine.get_domain() self.assertEqual(domain, 'example.com') # reload with --localhost - self.reloadEngine(argv=['--localhost'], settings=settings) + self.reloadEngine(argv=['--localhost']) domain = self.engine.get_domain() self.assertEqual(domain, 'localhost') def test_getPort(self): + EngineTest.defaultEnviron() p = self.engine.get_port() self.assertEqual(p, 8080) # reload with custom port - settings = EngineTest.defaultSettings() - settings['hosting']['port'] = 80 - self.reloadEngine(settings=settings) + os.environ['VTT_PORT'] = '80' + self.reloadEngine() p = self.engine.get_port() self.assertEqual(p, 80) def test_hasSsl(self): + EngineTest.defaultEnviron() self.assertFalse(self.engine.has_ssl()) # reload with ssl - settings = EngineTest.defaultSettings() - settings['hosting']['ssl'] = True - self.reloadEngine(settings=settings) + os.environ['VTT_SSL'] = 'True' + self.reloadEngine() self.assertTrue(self.engine.has_ssl()) def test_getUrl(self): - settings = EngineTest.defaultSettings() - self.reloadEngine(settings=settings) + EngineTest.defaultEnviron() + self.reloadEngine() self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') # internal SSL does not effect it - settings['hosting']['ssl'] = False - self.reloadEngine(settings=settings) + os.environ['VTT_SSL'] = 'False' + self.reloadEngine() self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') # internal port does not effect it - settings['hosting']['port'] = 443 - self.reloadEngine(settings=settings) + os.environ['VTT_PORT'] = '443' + self.reloadEngine() self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') # reload without reverse proxy will take effect - settings['hosting']['reverse'] = False - self.reloadEngine(settings=settings) + os.environ['VTT_REVERSE_PROXY'] = 'False' + self.reloadEngine() self.assertEqual(self.engine.get_url(), 'http://vtt.example.com:443') def test_getWebsocketUrl(self): - settings = EngineTest.defaultSettings() - self.reloadEngine(settings=settings) + EngineTest.defaultEnviron() + self.reloadEngine() self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') # internal SSL does not effect it - settings['hosting']['ssl'] = False - self.reloadEngine(settings=settings) + os.environ['VTT_SSL'] = 'False' + self.reloadEngine() self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') # internal port does not effect it - settings['hosting']['port'] = 443 - self.reloadEngine(settings=settings) + os.environ['VTT_PORT'] = '443' + self.reloadEngine() self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') # reload without reverse proxy will take effect - settings['hosting']['reverse'] = False - self.reloadEngine(settings=settings) + os.environ['VTT_REVERSE_PROXY'] = 'False' + os.environ['VTT_PORT'] = '443' + self.reloadEngine() self.assertEqual(self.engine.get_websocket_url(), 'ws://vtt.example.com:443/vtt/websocket') def test_getBuildSha(self): @@ -188,23 +159,23 @@ def test_getBuildSha(self): self.assertEqual(v, self.engine.debug_hash) def test_getAuthCallbackUrl(self): - settings = EngineTest.defaultSettings() - self.reloadEngine(settings=settings) + EngineTest.defaultEnviron() + self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') # internal SSL does not effect it - settings['hosting']['ssl'] = False - self.reloadEngine(settings=settings) + os.environ['VTT_SSL'] = 'False' + self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') # internal port does not effect it - settings['hosting']['port'] = 443 - self.reloadEngine(settings=settings) + os.environ['VTT_PORT'] = '443' + self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') # reload without reverse proxy will take effect - settings['hosting']['reverse'] = False - self.reloadEngine(settings=settings) + os.environ['VTT_REVERSE_PROXY'] = 'False' + self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'http://vtt.example.com:443/vtt/callback') def test_verifyUrlSection(self): From 4215d630f3507c7dfe184d436b0b94fa55aa358e Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Sun, 19 Jan 2025 19:36:17 -0500 Subject: [PATCH 10/29] use empty string for boolean false --- test/test_engine.py | 13 +++++++------ vtt/engine.py | 5 ++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/test/test_engine.py b/test/test_engine.py index e0f9fc5..b6ca92d 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -37,8 +37,8 @@ def defaultEnviron(): os.environ['VTT_CLEANUP_TIME'] = '03:00' os.environ['VTT_DOMAIN'] = 'vtt.example.com' os.environ['VTT_PORT'] = '8080' - os.environ['VTT_SSL'] = 'False' - os.environ['VTT_REVERSE_PROXY'] = 'True' + os.environ['VTT_SSL'] = '' + os.environ['VTT_REVERSE_PROXY'] = '' def reloadEngine(self, argv=list()): # reload engine (without cleanup thread) @@ -49,6 +49,7 @@ def reloadEngine(self, argv=list()): def test_run_engine_with_custom_prefdir(self): engine.Engine(argv=['--prefdir=/tmp']) + def test_run(self): # confirm server is offline @@ -105,7 +106,7 @@ def test_getUrl(self): self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') # internal SSL does not effect it - os.environ['VTT_SSL'] = 'False' + os.environ['VTT_SSL'] = '' self.reloadEngine() self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') @@ -125,7 +126,7 @@ def test_getWebsocketUrl(self): self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') # internal SSL does not effect it - os.environ['VTT_SSL'] = 'False' + os.environ['VTT_SSL'] = '' self.reloadEngine() self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') @@ -164,7 +165,7 @@ def test_getAuthCallbackUrl(self): self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') # internal SSL does not effect it - os.environ['VTT_SSL'] = 'False' + os.environ['VTT_SSL'] = '' self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') @@ -174,7 +175,7 @@ def test_getAuthCallbackUrl(self): self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') # reload without reverse proxy will take effect - os.environ['VTT_REVERSE_PROXY'] = 'False' + os.environ['VTT_REVERSE_PROXY'] = '' self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'http://vtt.example.com:443/vtt/callback') diff --git a/vtt/engine.py b/vtt/engine.py index 7638f03..2893b6c 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -60,11 +60,10 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): "domain" : os.getenv('VTT_DOMAIN', 'localhost'), "port" : int(os.getenv('VTT_PORT', 8080)), "socket" : os.getenv('VTT_SOCKET', ""), - "ssl" : bool(os.getenv('VTT_SSL', False)), - "reverse" : bool(os.getenv('VTT_REVERSE_PROXY', False)) + "ssl" : bool(os.getenv('VTT_SSL', "")), + "reverse" : bool(os.getenv('VTT_REVERSE_PROXY', "")) } self.shards = list() - self.main_db = None # blacklist for GM names and game URLs From 97a06194174b3e45b40478c7b5dce61796cfa4c2 Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 08:09:12 -0500 Subject: [PATCH 11/29] default pref_root to CWD/data --- vtt/utils/path_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vtt/utils/path_api.py b/vtt/utils/path_api.py index 43eed35..e9d8a0b 100644 --- a/vtt/utils/path_api.py +++ b/vtt/utils/path_api.py @@ -22,6 +22,7 @@ def __init__(self, appname: str, pref_root: pathlib.Path | None = None, app_root if pref_root is None: # use current working directory pref_root = pathlib.Path.cwd() + pref_root = pref_root / 'data' else: # we need to convert the string path value to a pathlib object pref_root = pathlib.Path(pref_root) From 893cfa7ebe2c79d18d7673e5075b58ca9c3ed158 Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 08:09:32 -0500 Subject: [PATCH 12/29] comment out unix socket test --- test/test_engine.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/test_engine.py b/test/test_engine.py index b6ca92d..bd63ef7 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -200,11 +200,12 @@ def get(self, s): dummy_request = FakeRequest() self.assertEqual(self.engine.get_client_ip(dummy_request), '1.2.3.4') - # reload engine with unix socket - settings = EngineTest.defaultSettings() - settings['hosting']['socket'] = '/path/to/socket' - self.reloadEngine(settings=settings) - self.assertEqual(self.engine.get_client_ip(dummy_request), '5.6.7.8') + # 2025-01-20 - test disabled, is unix socket worth keeping? + ## reload engine with unix socket + #EngineTest.defaultEnviron() + #settings['hosting']['socket'] = '/path/to/socket' + #self.reloadEngine(settings=settings) + #self.assertEqual(self.engine.get_client_ip(dummy_request), '5.6.7.8') def test_getClientAgent(self): class FakeRequest(object): From 52e21e88e3d19471079f9542ea38904d3f921926 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Mon, 20 Jan 2025 16:41:31 +0100 Subject: [PATCH 13/29] fix: use given pref_dir as default when getenv fails - not just None --- vtt/engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vtt/engine.py b/vtt/engine.py index 2893b6c..939a252 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -33,7 +33,7 @@ class Engine(object): def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): appname = os.getenv('VTT_APPNAME', 'pyvtt') - pref_dir = os.getenv('VTT_PREFDIR', None) + pref_dir = os.getenv('VTT_PREFDIR', pref_dir) self.log_level = os.getenv('VTT_LOG_LEVEL', 'INFO') for arg in argv: if arg.startswith('--appname='): From a955e6a9f684a0ff41cd4223a72c11a41cc445af Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 12:37:32 -0500 Subject: [PATCH 14/29] remove reverse proxy and socket support --- test/test_engine.py | 49 ++++++++++++--------------------------- vtt/engine.py | 56 +++++++++++++++++++++++---------------------- 2 files changed, 43 insertions(+), 62 deletions(-) diff --git a/test/test_engine.py b/test/test_engine.py index bd63ef7..7c64140 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -37,8 +37,8 @@ def defaultEnviron(): os.environ['VTT_CLEANUP_TIME'] = '03:00' os.environ['VTT_DOMAIN'] = 'vtt.example.com' os.environ['VTT_PORT'] = '8080' - os.environ['VTT_SSL'] = '' - os.environ['VTT_REVERSE_PROXY'] = '' + os.environ.pop('VTT_SSL', None) + os.environ.pop('VTT_REVERSE_PROXY', None) def reloadEngine(self, argv=list()): # reload engine (without cleanup thread) @@ -93,7 +93,8 @@ def test_getPort(self): def test_hasSsl(self): EngineTest.defaultEnviron() - self.assertFalse(self.engine.has_ssl()) + self.reloadEngine() + self.assertFalse(self.engine.has_ssl() # reload with ssl os.environ['VTT_SSL'] = 'True' @@ -103,44 +104,34 @@ def test_hasSsl(self): def test_getUrl(self): EngineTest.defaultEnviron() self.reloadEngine() - self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') + self.assertEqual(self.engine.get_url(), 'http://vtt.example.com:8080') # internal SSL does not effect it - os.environ['VTT_SSL'] = '' + os.environ['VTT_SSL'] = 'True' self.reloadEngine() - self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') + self.assertEqual(self.engine.get_url(), 'https://vtt.example.com:8080') # internal port does not effect it os.environ['VTT_PORT'] = '443' self.reloadEngine() self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') - # reload without reverse proxy will take effect - os.environ['VTT_REVERSE_PROXY'] = 'False' - self.reloadEngine() - self.assertEqual(self.engine.get_url(), 'http://vtt.example.com:443') - def test_getWebsocketUrl(self): EngineTest.defaultEnviron() self.reloadEngine() - self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') + self.assertEqual(self.engine.get_websocket_url(), 'ws://vtt.example.com:8080/vtt/websocket') # internal SSL does not effect it - os.environ['VTT_SSL'] = '' + os.environ['VTT_SSL'] = 'True' self.reloadEngine() - self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') + self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com:8080/vtt/websocket') # internal port does not effect it + os.environ['VTT_SSL'] = 'True' os.environ['VTT_PORT'] = '443' self.reloadEngine() self.assertEqual(self.engine.get_websocket_url(), 'wss://vtt.example.com/vtt/websocket') - # reload without reverse proxy will take effect - os.environ['VTT_REVERSE_PROXY'] = 'False' - os.environ['VTT_PORT'] = '443' - self.reloadEngine() - self.assertEqual(self.engine.get_websocket_url(), 'ws://vtt.example.com:443/vtt/websocket') - def test_getBuildSha(self): self.engine.git_hash = None self.engine.debug_hash = None @@ -162,23 +153,18 @@ def test_getBuildSha(self): def test_getAuthCallbackUrl(self): EngineTest.defaultEnviron() self.reloadEngine() - self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') + self.assertEqual(self.engine.get_auth_callback_url(), 'http://vtt.example.com:8080/vtt/callback') # internal SSL does not effect it - os.environ['VTT_SSL'] = '' + os.environ['VTT_SSL'] = 'True' self.reloadEngine() - self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') + self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com:8080/vtt/callback') # internal port does not effect it os.environ['VTT_PORT'] = '443' self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'https://vtt.example.com/vtt/callback') - # reload without reverse proxy will take effect - os.environ['VTT_REVERSE_PROXY'] = '' - self.reloadEngine() - self.assertEqual(self.engine.get_auth_callback_url(), 'http://vtt.example.com:443/vtt/callback') - def test_verifyUrlSection(self): self.assertTrue(self.engine.verify_url_section('foo-bar.lol_test')) self.assertFalse(self.engine.verify_url_section('url-with-speciöl-char')) @@ -200,13 +186,6 @@ def get(self, s): dummy_request = FakeRequest() self.assertEqual(self.engine.get_client_ip(dummy_request), '1.2.3.4') - # 2025-01-20 - test disabled, is unix socket worth keeping? - ## reload engine with unix socket - #EngineTest.defaultEnviron() - #settings['hosting']['socket'] = '/path/to/socket' - #self.reloadEngine(settings=settings) - #self.assertEqual(self.engine.get_client_ip(dummy_request), '5.6.7.8') - def test_getClientAgent(self): class FakeRequest(object): def __init__(self): diff --git a/vtt/engine.py b/vtt/engine.py index 939a252..68329ec 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -59,9 +59,7 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.hosting = { "domain" : os.getenv('VTT_DOMAIN', 'localhost'), "port" : int(os.getenv('VTT_PORT', 8080)), - "socket" : os.getenv('VTT_SOCKET', ""), - "ssl" : bool(os.getenv('VTT_SSL', "")), - "reverse" : bool(os.getenv('VTT_REVERSE_PROXY', "")) + "ssl" : bool(os.getenv('VTT_SSL', False)) } self.shards = list() self.main_db = None @@ -267,18 +265,6 @@ def load_version_number(self) -> None: self.debug_hash = uuid.uuid4().hex def run(self): - certfile = '' - keyfile = '' - if self.has_ssl(): - # enable SSL - ssl_dir = self.paths.get_ssl_path() - certfile = ssl_dir / 'cacert.pem' - keyfile = ssl_dir / 'privkey.pem' - assert(os.path.exists(certfile)) - assert(os.path.exists(keyfile)) - - ssl_args = {'certfile': certfile, 'keyfile': keyfile} if self.has_ssl() else {} - if self.notify_api is not None: self.notify_api.on_start() @@ -288,10 +274,6 @@ def run(self): debug = self.debug, quiet = self.quiet, server = VttServer, - # VttServer-specific - unixsocket = self.hosting['socket'], - # SSL-specific - **ssl_args ) def get_domain(self): @@ -306,18 +288,39 @@ def get_port(self): return self.hosting['port'] def get_url(self): - suffix = 's' if self.has_reverse_proxy() or self.has_ssl() else '' - port = '' if self.has_reverse_proxy() else f':{self.get_port()}' + port = '' + suffix = '' + if self.has_ssl(): + suffix = 's' + if self.get_port() != 443: + port = f':{self.get_port()}' + else: + if self.get_port != 80: + port = f':{self.get_port()}' return f'http{suffix}://{self.get_domain()}{port}' def get_websocket_url(self): - protocol = 'wss' if self.has_reverse_proxy() or self.has_ssl() else 'ws' - port = '' if self.has_reverse_proxy() else f':{self.get_port()}' + port = '' + protocol = 'ws' + if self.has_ssl(): + protocol = 'wss' + if self.get_port() != 443: + port = f':{self.get_port()}' + else: + if self.get_port() != 80: + port = f':{self.get_port()}' return f'{protocol}://{self.get_domain()}{port}/vtt/websocket' def get_auth_callback_url(self): - protocol = 'https' if self.has_reverse_proxy() or self.has_ssl() else 'http' - port = '' if self.has_reverse_proxy() else f':{self.get_port()}' + port = '' + protocol = 'http' + if self.has_ssl(): + protocol = 'https' + if self.get_port() != 443: + port = f':{self.get_port()}' + else: + if self.get_port() != 80: + port = f':{self.get_port()}' return f'{protocol}://{self.get_domain()}{port}/vtt/callback' def get_build_sha(self): @@ -340,8 +343,7 @@ def verify_url_section(self, s): return bool(re.match(self.url_regex, s)) def get_client_ip(self, request): - # use different header if through unix socket or reverse proxy - if self.hosting['socket'] != '' or self.has_reverse_proxy(): + if request.environ.get('HTTP_X_FORWARDED_FOR'): return request.environ.get('HTTP_X_FORWARDED_FOR') else: return request.environ.get('REMOTE_ADDR') From 73c2368badae99f070e73d0868444913a5b9f3ef Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 12:42:27 -0500 Subject: [PATCH 15/29] remove shard support --- static/client/shard.js | 35 --------------------- test/routes/test_api.py | 54 --------------------------------- views/header.tpl | 2 +- views/shard.tpl | 47 ----------------------------- vtt/engine.py | 10 ------ vtt/routes/api/register.py | 16 ---------- vtt/routes/api/shards.py | 62 -------------------------------------- 7 files changed, 1 insertion(+), 225 deletions(-) delete mode 100644 static/client/shard.js delete mode 100644 views/shard.tpl delete mode 100644 vtt/routes/api/register.py delete mode 100644 vtt/routes/api/shards.py diff --git a/static/client/shard.js b/static/client/shard.js deleted file mode 100644 index 4131367..0000000 --- a/static/client/shard.js +++ /dev/null @@ -1,35 +0,0 @@ -/** -https://github.com/cgloeckner/pyvtt/ - -Copyright (c) 2020-2022 Christian Glöckner -License: MIT (see LICENSE for details) -*/ - -function queryShard(index, host) { - $.ajax({ - url: '/vtt/query/' + index, - type: 'GET', - success: function(response) { - console.log(response) - if (response.flag != null) { - $(`#flag${index}`)[0].innerHTML = response.flag - } - - if (response.build.version != null) { - let target = $(`#title${index}`) - let title = `${response.build.title} v${response.build.version}` - let build_hash = `git sha ${response.build.git_hash}` - - if (response.build.debug_hash) { - build_hash += ' (debug)' - } - target[0].title = build_hash - target[0].innerHTML = title - } - - if (response.games != null) { - $(`#games${index}`)[0].innerHTML = response.games - } - } - }) -} diff --git a/test/routes/test_api.py b/test/routes/test_api.py index 61ebe6f..a6d393f 100644 --- a/test/routes/test_api.py +++ b/test/routes/test_api.py @@ -71,57 +71,3 @@ def test_can_query_games_and_assets_as_gm(self): ret = self.app.get('/vtt/api/assets-list/arthur/test-game-1', expect_errors=True) self.assertEqual(ret.status_int, 200) - def test_query_this_server_if_no_shards_specified(self): - ret = self.app.get('/vtt/query/0', expect_errors=True) - self.assertEqual(ret.status_int, 200) - - def test_query_servers_within_shard(self): - # setup server shards - test_ports = [8081, 8000] - self.engine.shards = ['http://localhost:{0}'.format(p) for p in test_ports] - self.engine.shards.append('http://localhost:80') # this server - greenlets = list() - for port in test_ports: - # confirm port to be free - with self.assertRaises(requests.exceptions.ConnectionError): - requests.get('http://localhost:{0}'.format(port)) - # setup server instance - e = EngineBaseTest() - e.setUp() - e.engine.hosting['port'] = port - e.engine.shards = self.engine.shards - # run in thread - g = gevent.Greenlet(run=e.engine.run) - g.start() - greenlets.append(g) - # confirm server is online - requests.get('http://localhost:{0}'.format(port)) - - # can query all servers - for i, url in enumerate(self.engine.shards): - ret = self.app.get('/vtt/query/{0}'.format(i)) - self.assertEqual(ret.status_int, 200) - # @NOTE: cannot test countryCode due to localhost and status - # because this may fail on the GitHub workflow test - - # stop server shard instances - for g in greenlets: - gevent.kill(g) - - def test_cannot_query_unknown_server(self): - ret = self.app.get('/vtt/query/245245', expect_errors=True) - self.assertEqual(ret.status_int, 404) - - def test_show_shard_page_if_no_shards_specified(self): - ret = self.app.get('/vtt/shard', expect_errors=True) - self.assertEqual(ret.status_int, 200) - - def test_show_shard_page_if_only_one_server_specified(self): - self.engine.shards = ['http://localhost:80'] - ret = self.app.get('/vtt/shard') - self.assertEqual(ret.status_int, 200) - - def test_show_shard_page_if_multiple_servers_specified(self): - self.engine.shards = ['https://{0}'.format(h) for h in ['example.com', 'foo.bar', 'test.org']] - ret = self.app.get('/vtt/shard') - self.assertEqual(ret.status_int, 200) diff --git a/views/header.tpl b/views/header.tpl index 7730966..3089c71 100644 --- a/views/header.tpl +++ b/views/header.tpl @@ -12,7 +12,7 @@ License: MIT (see LICENSE for details) %version = engine.get_build_sha() -%for js in ['jquery-3.3.1.min', 'md5', 'version', 'constants', 'errors', 'dropdown', 'render', 'ui', 'socket', 'gm', 'music', 'utils', 'webcam', 'drawing', 'assets', 'shard']: +%for js in ['jquery-3.3.1.min', 'md5', 'version', 'constants', 'errors', 'dropdown', 'render', 'ui', 'socket', 'gm', 'music', 'utils', 'webcam', 'drawing', 'assets']: %end diff --git a/views/shard.tpl b/views/shard.tpl deleted file mode 100644 index 8760388..0000000 --- a/views/shard.tpl +++ /dev/null @@ -1,47 +0,0 @@ -%include("header", title="Servers") - -
- -
- -

SERVERS

- - - - - - - - - -%i = 0 -%for host in engine.shards: - - - - - %if host == own: - - %else: - - %end - - - %i += 1 -%end -
LinkACTIVE GAMES
{{host}}{{host}}
- -
- - -
- -
- -%include("footer") diff --git a/vtt/engine.py b/vtt/engine.py index 68329ec..11c2411 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -61,7 +61,6 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): "port" : int(os.getenv('VTT_PORT', 8080)), "ssl" : bool(os.getenv('VTT_SSL', False)) } - self.shards = list() self.main_db = None # blacklist for GM names and game URLs @@ -102,15 +101,6 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): 'url': value }) - num_shards = int(os.getenv('VTT_SHARDS_NUM', 0)) - self.shards = [os.getenv(f'VTT_SHARDS_{n}') for n in range(num_shards)] - if len(self.shards) > 0: - - self.links.append({ - "label" : "STATUS", - "url" : "/vtt/shard" - }) - self.cleanup = { 'expire': int(os.getenv('VTT_CLEANUP_EXPIRE', 2592000)), 'daytime': os.getenv('VTT_CLEANUP_TIME', '03:00') diff --git a/vtt/routes/api/register.py b/vtt/routes/api/register.py deleted file mode 100644 index 4cf7b80..0000000 --- a/vtt/routes/api/register.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -https://github.com/cgloeckner/pyvtt/ - -Copyright (c) 2020-2023 Christian Glöckner -License: MIT (see LICENSE for details) -""" - -__author__ = 'Christian Glöckner' -__licence__ = 'MIT' - -from . import shards, json_api - - -def register(engine: any): - shards.register(engine) - json_api.register(engine) diff --git a/vtt/routes/api/shards.py b/vtt/routes/api/shards.py deleted file mode 100644 index eb4e2a0..0000000 --- a/vtt/routes/api/shards.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -https://github.com/cgloeckner/pyvtt/ - -Copyright (c) 2020-2023 Christian Glöckner -License: MIT (see LICENSE for details) -""" - -__author__ = 'Christian Glöckner' -__licence__ = 'MIT' - -import flag -import requests - -from bottle import * - - -def register(engine: any): - - @get('/vtt/query/') - def status_query(index: int): - if len(engine.shards) == 0: - abort(404) - - # ask server - game_host = '' - try: - game_host = engine.shards[index] - except IndexError: - abort(404) - - data = dict() - data['games'] = None - data['flag'] = None - data['build'] = {} - try: - json = requests.get(f'{game_host}/vtt/api/build', timeout=3).json() - data['build'] = json - except requests.exceptions.RequestException: - engine.logging.error(f'Server {game_host} seems to be offline') - - # query server location (if possible) - ip = game_host.split('://')[1].split(':')[0] - country = engine.get_country_from_ip(ip) - if country not in ['?', 'unknown']: - data['flag'] = flag.flag(country) - - # query server data - try: - json = requests.get(f'{game_host}/vtt/api/users', timeout=3).json() - data['games'] = json['games']['running'] - except requests.exceptions.RequestException: - engine.logging.error(f'Server {game_host} seems to be offline') - - return data - - @get('/vtt/shard') - @view('shard') - def shard_list(): - if len(engine.shards) == 0: - abort(404) - - return dict(engine=engine, own=engine.get_url()) From 9ea07904aad0885e7b1541b20802e2ed8531d40a Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 12:50:25 -0500 Subject: [PATCH 16/29] remove shard registration functionality --- test/routes/test_api.py | 1 - test/test_engine.py | 2 +- vtt/routes/__init__.py | 1 - vtt/routes/api/__init__.py | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 vtt/routes/api/__init__.py diff --git a/test/routes/test_api.py b/test/routes/test_api.py index a6d393f..ba41350 100644 --- a/test/routes/test_api.py +++ b/test/routes/test_api.py @@ -19,7 +19,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here def test_api_get_queries(self): diff --git a/test/test_engine.py b/test/test_engine.py index 7c64140..8f43a8c 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -94,7 +94,7 @@ def test_getPort(self): def test_hasSsl(self): EngineTest.defaultEnviron() self.reloadEngine() - self.assertFalse(self.engine.has_ssl() + self.assertFalse(self.engine.has_ssl()) # reload with ssl os.environ['VTT_SSL'] = 'True' diff --git a/vtt/routes/__init__.py b/vtt/routes/__init__.py index 88146dc..d0d8521 100644 --- a/vtt/routes/__init__.py +++ b/vtt/routes/__init__.py @@ -1,5 +1,4 @@ from .gm import register as register_gm from .player import register as register_player -from .api import register as register_api from .resources import register as register_resources from .error import register as register_error diff --git a/vtt/routes/api/__init__.py b/vtt/routes/api/__init__.py deleted file mode 100644 index 1d8a5b7..0000000 --- a/vtt/routes/api/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .register import * From ebfc6d6a6cd1a0007e506ae529f40a8ae5157348 Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 13:00:59 -0500 Subject: [PATCH 17/29] remove shard registration from main.py --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 4d01f24..8699900 100644 --- a/main.py +++ b/main.py @@ -14,7 +14,6 @@ engine = Engine(argv=argv) routes.register_gm(engine) routes.register_player(engine) - routes.register_api(engine) routes.register_resources(engine) routes.register_error(engine) From d58289cc2265d6ade3d9534e96be94ff6d968150 Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 17:20:21 -0500 Subject: [PATCH 18/29] remove register_api and get_settings_path from tests --- test/routes/gm/auth_service.py | 1 - test/routes/gm/test_drawer.py | 1 - test/routes/gm/test_export.py | 1 - test/routes/gm/test_fallback_login.py | 1 - test/routes/gm/test_import.py | 1 - test/routes/gm/test_login.py | 1 - test/routes/gm/test_login_callback.py | 1 - test/routes/gm/test_thumbnail.py | 1 - test/routes/players/test_assets.py | 1 - test/routes/players/test_login.py | 1 - test/routes/players/test_upload.py | 1 - test/routes/players/test_websocket.py | 1 - test/routes/test_error.py | 1 - test/routes/test_resources.py | 1 - test/test_engine.py | 2 -- test/utils/test_path_api.py | 2 +- vtt/engine.py | 2 +- 17 files changed, 2 insertions(+), 18 deletions(-) diff --git a/test/routes/gm/auth_service.py b/test/routes/gm/auth_service.py index 471f82d..f5f1408 100644 --- a/test/routes/gm/auth_service.py +++ b/test/routes/gm/auth_service.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # FIXME: callback process via fake auth service via /vtt/callback/fake diff --git a/test/routes/gm/test_drawer.py b/test/routes/gm/test_drawer.py index ee308bb..b2c3aa0 100644 --- a/test/routes/gm/test_drawer.py +++ b/test/routes/gm/test_drawer.py @@ -20,7 +20,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # FIXME: move to EngineBaseTest (create_arthur) -> session_id diff --git a/test/routes/gm/test_export.py b/test/routes/gm/test_export.py index 0dc71f4..c0cdc05 100644 --- a/test/routes/gm/test_export.py +++ b/test/routes/gm/test_export.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # register arthur diff --git a/test/routes/gm/test_fallback_login.py b/test/routes/gm/test_fallback_login.py index 04463b8..73784f5 100644 --- a/test/routes/gm/test_fallback_login.py +++ b/test/routes/gm/test_fallback_login.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here def test_post_vtt_join(self): diff --git a/test/routes/gm/test_import.py b/test/routes/gm/test_import.py index 43355ae..0c94ddb 100644 --- a/test/routes/gm/test_import.py +++ b/test/routes/gm/test_import.py @@ -18,7 +18,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # create some images diff --git a/test/routes/gm/test_login.py b/test/routes/gm/test_login.py index 6bf6055..214b221 100644 --- a/test/routes/gm/test_login.py +++ b/test/routes/gm/test_login.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # register arthur diff --git a/test/routes/gm/test_login_callback.py b/test/routes/gm/test_login_callback.py index 4ba04c8..e7160fd 100644 --- a/test/routes/gm/test_login_callback.py +++ b/test/routes/gm/test_login_callback.py @@ -60,7 +60,6 @@ def setUp(self) -> None: routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here def test_can_login_via_callback_and_create_new_account(self) -> None: diff --git a/test/routes/gm/test_thumbnail.py b/test/routes/gm/test_thumbnail.py index 95e4d1e..3c31633 100644 --- a/test/routes/gm/test_thumbnail.py +++ b/test/routes/gm/test_thumbnail.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # register arthur diff --git a/test/routes/players/test_assets.py b/test/routes/players/test_assets.py index f266c83..bce54c2 100644 --- a/test/routes/players/test_assets.py +++ b/test/routes/players/test_assets.py @@ -19,7 +19,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # register arthur diff --git a/test/routes/players/test_login.py b/test/routes/players/test_login.py index e0febb6..e54ab48 100644 --- a/test/routes/players/test_login.py +++ b/test/routes/players/test_login.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # register arthur diff --git a/test/routes/players/test_upload.py b/test/routes/players/test_upload.py index 47d9c7a..bfe5436 100644 --- a/test/routes/players/test_upload.py +++ b/test/routes/players/test_upload.py @@ -28,7 +28,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # create some images diff --git a/test/routes/players/test_websocket.py b/test/routes/players/test_websocket.py index 91b9846..e6a7443 100644 --- a/test/routes/players/test_websocket.py +++ b/test/routes/players/test_websocket.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # @NOTE establishing a websocket is not tested atm diff --git a/test/routes/test_error.py b/test/routes/test_error.py index 583aff1..bba978f 100644 --- a/test/routes/test_error.py +++ b/test/routes/test_error.py @@ -16,7 +16,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here def test_401(self): diff --git a/test/routes/test_resources.py b/test/routes/test_resources.py index 73f5a26..71d923d 100644 --- a/test/routes/test_resources.py +++ b/test/routes/test_resources.py @@ -19,7 +19,6 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) - routes.register_api(self.engine) # @NOTE: custom error pages are not routed here # register arthur diff --git a/test/test_engine.py b/test/test_engine.py index 8f43a8c..d0092d0 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -388,5 +388,3 @@ def test_loadFromDict(self): all_games = list(gm_cache.db.Game.select()) self.assertEqual(len(all_games), 1) - - diff --git a/test/utils/test_path_api.py b/test/utils/test_path_api.py index 9fdb7f2..a0dadd7 100644 --- a/test/utils/test_path_api.py +++ b/test/utils/test_path_api.py @@ -64,7 +64,7 @@ def test_simple_path_getter(self): self.paths.get_assets_path() self.paths.get_assets_path(default=True) self.paths.get_client_code_path() - self.paths.get_settings_path() + #self.paths.get_settings_path() self.paths.get_main_database_path() self.paths.get_constants_path() self.paths.get_ssl_path() diff --git a/vtt/engine.py b/vtt/engine.py index 11c2411..60e78f6 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -65,7 +65,7 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): # blacklist for GM names and game URLs self.gm_blacklist = ['', 'static', 'asset', 'vtt', 'game'] - self.url_regex = '^[A-Za-z0-9_\-.]+$' + self.url_regex = r'^[A-Za-z0-9_\-.]+$' # maximum file sizes for uploads (in MB) self.file_limit = { From 84f592cb8b5880f2277e17ae346db7d147b6018d Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 18:50:31 -0500 Subject: [PATCH 19/29] try to check proxied IPs --- test/test_engine.py | 12 +++++++++--- vtt/engine.py | 6 ++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/test/test_engine.py b/test/test_engine.py index d0092d0..b636b13 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -174,17 +174,23 @@ def test_verifyUrlSection(self): def test_getClientIp(self): class FakeRequest(object): - def __init__(self): + def __init__(self, proxy = False): class FakeEnviron(object): + def __init__(self, proxy = False): + self.proxy = proxy def get(self, s): - if s == 'REMOTE_ADDR': + if not self.proxy and s == 'REMOTE_ADDR': return '1.2.3.4' + if not self.proxy and s == 'HTTP_X_FORWARDED_FOR': + return None else: return '5.6.7.8' - self.environ = FakeEnviron() + self.environ = FakeEnviron(proxy) dummy_request = FakeRequest() self.assertEqual(self.engine.get_client_ip(dummy_request), '1.2.3.4') + dummy_request = FakeRequest(True) + self.assertEqual(self.engine.get_client_ip(dummy_request), '5.6.7.8') def test_getClientAgent(self): class FakeRequest(object): diff --git a/vtt/engine.py b/vtt/engine.py index 60e78f6..0d038be 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -333,10 +333,8 @@ def verify_url_section(self, s): return bool(re.match(self.url_regex, s)) def get_client_ip(self, request): - if request.environ.get('HTTP_X_FORWARDED_FOR'): - return request.environ.get('HTTP_X_FORWARDED_FOR') - else: - return request.environ.get('REMOTE_ADDR') + client_ip = request.environ.get('HTTP_X_FORWARDED_FOR') or request.environ.get('REMOTE_ADDR') + return client_ip def get_client_agent(self, request): return request.environ.get('HTTP_USER_AGENT') From 89635d0c05ff52a3d04dd467f02a632ae2d7d306 Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Mon, 20 Jan 2025 19:41:46 -0500 Subject: [PATCH 20/29] test no-proxy and proxy; store proxy declaration in engine not test --- test/test_engine.py | 15 +++++++-------- vtt/engine.py | 9 ++++++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/test/test_engine.py b/test/test_engine.py index b636b13..8d277b5 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -174,22 +174,21 @@ def test_verifyUrlSection(self): def test_getClientIp(self): class FakeRequest(object): - def __init__(self, proxy = False): + def __init__(self): class FakeEnviron(object): - def __init__(self, proxy = False): - self.proxy = proxy def get(self, s): - if not self.proxy and s == 'REMOTE_ADDR': + if s == 'REMOTE_ADDR': return '1.2.3.4' - if not self.proxy and s == 'HTTP_X_FORWARDED_FOR': - return None else: return '5.6.7.8' - self.environ = FakeEnviron(proxy) + self.environ = FakeEnviron() + # test no reverse proxy dummy_request = FakeRequest() self.assertEqual(self.engine.get_client_ip(dummy_request), '1.2.3.4') - dummy_request = FakeRequest(True) + # test reverse proxy + os.environ['VTT_REVERSE_PROXY'] = 'True' + self.reloadEngine() self.assertEqual(self.engine.get_client_ip(dummy_request), '5.6.7.8') def test_getClientAgent(self): diff --git a/vtt/engine.py b/vtt/engine.py index 0d038be..acc18a8 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -59,7 +59,8 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): self.hosting = { "domain" : os.getenv('VTT_DOMAIN', 'localhost'), "port" : int(os.getenv('VTT_PORT', 8080)), - "ssl" : bool(os.getenv('VTT_SSL', False)) + "ssl" : bool(os.getenv('VTT_SSL', False)), + "reverse" : bool(os.getenv('VTT_REVERSE_PROXY')) } self.main_db = None @@ -333,8 +334,10 @@ def verify_url_section(self, s): return bool(re.match(self.url_regex, s)) def get_client_ip(self, request): - client_ip = request.environ.get('HTTP_X_FORWARDED_FOR') or request.environ.get('REMOTE_ADDR') - return client_ip + if self.has_reverse_proxy(): + return request.environ.get('HTTP_X_FORWARDED_FOR') + else: + return request.environ.get('REMOTE_ADDR') def get_client_agent(self, request): return request.environ.get('HTTP_USER_AGENT') From b442684f4bad61cc4a1d2e4b45d2d34d8493f4b7 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 15:00:46 +0100 Subject: [PATCH 21/29] added registering api routes for unittesting back in --- vtt/routes/json_api.py | 196 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) create mode 100644 vtt/routes/json_api.py diff --git a/vtt/routes/json_api.py b/vtt/routes/json_api.py new file mode 100644 index 0000000..6fbb8f9 --- /dev/null +++ b/vtt/routes/json_api.py @@ -0,0 +1,196 @@ +""" +https://github.com/cgloeckner/pyvtt/ + +Copyright (c) 2020-2023 Christian Glöckner +License: MIT (see LICENSE for details) +""" + +__author__ = 'Christian Glöckner' +__licence__ = 'MIT' + +import httpagentparser + +from bottle import * + +from vtt.utils.common import add_dict_set, count_dict_set_len + + +def register(engine: any): + + @get('/vtt/api/users') + def api_query_users(): + now = time.time() + + # query gms + total_gms = engine.main_db.GM.select().count() + abandoned_gms = engine.main_db.GM.select(lambda g: g.timeid < now - engine.cleanup['expire']).count() + + # query games + threshold = 10 + total_games = 0 + running_games = 0 + with engine.cache.lock: + for gm in engine.cache.gms: + gm_cache = engine.cache.gms[gm] + with gm_cache.lock: + total_games += gm_cache.db.Game.select().count() + running_games += gm_cache.db.Game.select(lambda g: g.timeid >= now - threshold * 60).count() + done = time.time() + + # return data + return { + 'gms': { + 'total': total_gms, + 'abandoned': abandoned_gms + }, + 'games': { + 'total': total_games, + 'running': running_games + }, + 'query_time': done - now + } + + @get('/vtt/api/games-list/') + def api_games_list(gm_url: str): + start = time.time() + + # load GM from cache + gm_cache = engine.cache.get_from_url(gm_url) + if gm_cache is None: + # @NOTE: not logged because somebody may play around with this + abort(404) + + # query whether user is the hosting GM + session_gm = engine.main_db.GM.load_from_session(request) + if session_gm is None or session_gm.url != gm_url: + abort(404) + + data = { + 'games': [] + } + for game in gm_cache.db.Game.select(): + data['games'].append(game.url) + done = time.time() + + data['query_time'] = done - start + return data + + @get('/vtt/api/assets-list//') + def api_asset_list(gm_url: str, game_url: str): + start = time.time() + + # load GM from cache + gm_cache = engine.cache.get_from_url(gm_url) + if gm_cache is None: + # @NOTE: not logged because somebody may play around with this + abort(404) + + # query whether user is the hosting GM + session_gm = engine.main_db.GM.load_from_session(request) + if session_gm is None or session_gm.url != gm_url: + abort(404) + + # try to load game from GM's database + game = gm_cache.db.Game.select(lambda g: g.url == game_url).first() + root = ['./static/assets', engine.paths.get_assets_path()] + if game is not None: + root = [engine.paths.get_game_path(gm_url, game_url)] + + files = { + 'images': [], + 'audio': [] + } + for sub_root in root: + for filename in os.listdir(sub_root): + if filename.endswith('.png'): + files['images'].append(filename) + if filename.endswith('.mp3'): + files['audio'].append(filename) + if game is not None: + files['images'].sort(key=lambda k: int(k.split('.')[0])) + else: + files['images'].sort() + files['audio'].sort() + done = time.time() + + files['query_time'] = done - start + + return files + + @get('/vtt/api/cleanup') + def api_next_cleanup(): + when, until = engine.cleanup_worker.getNextUpdate() + return { + 'server time': str(when), + 'time left': str(until) + } + + @get('/vtt/api/build') + def api_build(): + return { + 'title': engine.title, + 'version': engine.version, + 'git_hash': engine.git_hash, + 'debug_hash': engine.debug_hash + } + + @get('/vtt/api/logins') + def api_query_logins(): + """Count users locations based on IPs within past 30d.""" + start = time.time() + + logins = engine.parse_login_log() + locations = dict() + platforms = dict() + browsers = dict() + since = start - 30 * 24 * 3600 # past 30d + + # group IPs by country + for record in logins: + if record.timeid < since: + continue + data = httpagentparser.detect(record.agent) + + # save data + add_dict_set(locations, record.country, record.ip) + if 'platform' in data: + add_dict_set(platforms, data['platform']['name'], record.ip) + else: + add_dict_set(platforms, 'unknown', record.ip) + if 'browser' in data: + add_dict_set(browsers, data['browser']['name'], record.ip) + else: + add_dict_set(browsers, 'unknown', record.ip) + + # count IPs per country, platform and browser + count_dict_set_len(locations) + count_dict_set_len(platforms) + count_dict_set_len(browsers) + + done = time.time() + + return { + 'locations': locations, + 'platforms': platforms, + 'browsers': browsers, + 'query_time': done - start + } + + @get('/vtt/api/auth') + def api_query_auth(): + # query gms and how many accounts do idle + now = time.time() + provider = {} + for gm in engine.main_db.GM.select(): + # fetch provider + p = '-'.join(gm.metadata.split('|')[:-1]) + for k in ['-oauth2', 'oauth2-']: + p = p.replace(k, '') + if p not in provider: + provider[p] = 1 + else: + provider[p] += 1 + done = time.time() + + provider['query_time'] = done - now + return provider From 07281aab1dbf870ba415e9d1ad1896f909834667 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 15:58:58 +0100 Subject: [PATCH 22/29] defaultEnviron to base class --- test/common.py | 16 +++ test/routes/test_api.py | 1 + test/test_engine.py | 21 ---- vtt/routes/__init__.py | 1 + vtt/routes/api/json_api.py | 196 ------------------------------------- 5 files changed, 18 insertions(+), 217 deletions(-) delete mode 100644 vtt/routes/api/json_api.py diff --git a/test/common.py b/test/common.py index 73fa1ca..c2d6182 100644 --- a/test/common.py +++ b/test/common.py @@ -59,6 +59,21 @@ def make_zip(filename: str, data: str, n: int) -> bytes: class EngineBaseTest(unittest.TestCase): + @staticmethod + def defaultEnviron(): + os.environ['VTT_TITLE'] = 'unittest' + os.environ['VTT_LIMIT_TOKEN'] =' 2' + os.environ['VTT_LIMIT_BG'] = '10' + os.environ['VTT_LIMIT_GAME'] = '5' + os.environ['VTT_LIMIT_MUSIC'] = '10' + os.environ['VTT_NUM_MUSIC'] = '5' + os.environ['VTT_CLEANUP_EXPIRE'] = '3600' + os.environ['VTT_CLEANUP_TIME'] = '03:00' + os.environ['VTT_DOMAIN'] = 'vtt.example.com' + os.environ['VTT_PORT'] = '8080' + os.environ.pop('VTT_SSL', None) + os.environ.pop('VTT_REVERSE_PROXY', None) + def setUp(self) -> None: # create temporary directory self.tmpdir = tempfile.TemporaryDirectory() @@ -76,6 +91,7 @@ def setUp(self) -> None: self.app = webtest.TestApp(self.engine.app) self.monkeyPatch() + EngineBaseTest.defaultEnviron() def monkeyPatch(self) -> None: # save methods for later diff --git a/test/routes/test_api.py b/test/routes/test_api.py index ba41350..a6d393f 100644 --- a/test/routes/test_api.py +++ b/test/routes/test_api.py @@ -19,6 +19,7 @@ def setUp(self): routes.register_gm(self.engine) routes.register_player(self.engine) routes.register_resources(self.engine) + routes.register_api(self.engine) # @NOTE: custom error pages are not routed here def test_api_get_queries(self): diff --git a/test/test_engine.py b/test/test_engine.py index 8d277b5..07f8ee7 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -24,21 +24,6 @@ class EngineTest(EngineBaseTest): def tearDown(self): super().tearDown() - - @staticmethod - def defaultEnviron(): - os.environ['VTT_TITLE'] = 'unittest' - os.environ['VTT_LIMIT_TOKEN'] =' 2' - os.environ['VTT_LIMIT_BG'] = '10' - os.environ['VTT_LIMIT_GAME'] = '5' - os.environ['VTT_LIMIT_MUSIC'] = '10' - os.environ['VTT_NUM_MUSIC'] = '5' - os.environ['VTT_CLEANUP_EXPIRE'] = '3600' - os.environ['VTT_CLEANUP_TIME'] = '03:00' - os.environ['VTT_DOMAIN'] = 'vtt.example.com' - os.environ['VTT_PORT'] = '8080' - os.environ.pop('VTT_SSL', None) - os.environ.pop('VTT_REVERSE_PROXY', None) def reloadEngine(self, argv=list()): # reload engine (without cleanup thread) @@ -68,7 +53,6 @@ def test_run(self): requests.get('http://localhost:8080') def test_getDomain(self): - EngineTest.defaultEnviron() os.environ['VTT_DOMAIN'] = 'example.com' self.reloadEngine() @@ -81,7 +65,6 @@ def test_getDomain(self): self.assertEqual(domain, 'localhost') def test_getPort(self): - EngineTest.defaultEnviron() p = self.engine.get_port() self.assertEqual(p, 8080) @@ -92,7 +75,6 @@ def test_getPort(self): self.assertEqual(p, 80) def test_hasSsl(self): - EngineTest.defaultEnviron() self.reloadEngine() self.assertFalse(self.engine.has_ssl()) @@ -102,7 +84,6 @@ def test_hasSsl(self): self.assertTrue(self.engine.has_ssl()) def test_getUrl(self): - EngineTest.defaultEnviron() self.reloadEngine() self.assertEqual(self.engine.get_url(), 'http://vtt.example.com:8080') @@ -117,7 +98,6 @@ def test_getUrl(self): self.assertEqual(self.engine.get_url(), 'https://vtt.example.com') def test_getWebsocketUrl(self): - EngineTest.defaultEnviron() self.reloadEngine() self.assertEqual(self.engine.get_websocket_url(), 'ws://vtt.example.com:8080/vtt/websocket') @@ -151,7 +131,6 @@ def test_getBuildSha(self): self.assertEqual(v, self.engine.debug_hash) def test_getAuthCallbackUrl(self): - EngineTest.defaultEnviron() self.reloadEngine() self.assertEqual(self.engine.get_auth_callback_url(), 'http://vtt.example.com:8080/vtt/callback') diff --git a/vtt/routes/__init__.py b/vtt/routes/__init__.py index d0d8521..f0c03f7 100644 --- a/vtt/routes/__init__.py +++ b/vtt/routes/__init__.py @@ -2,3 +2,4 @@ from .player import register as register_player from .resources import register as register_resources from .error import register as register_error +from .json_api import register as register_api diff --git a/vtt/routes/api/json_api.py b/vtt/routes/api/json_api.py deleted file mode 100644 index 6fbb8f9..0000000 --- a/vtt/routes/api/json_api.py +++ /dev/null @@ -1,196 +0,0 @@ -""" -https://github.com/cgloeckner/pyvtt/ - -Copyright (c) 2020-2023 Christian Glöckner -License: MIT (see LICENSE for details) -""" - -__author__ = 'Christian Glöckner' -__licence__ = 'MIT' - -import httpagentparser - -from bottle import * - -from vtt.utils.common import add_dict_set, count_dict_set_len - - -def register(engine: any): - - @get('/vtt/api/users') - def api_query_users(): - now = time.time() - - # query gms - total_gms = engine.main_db.GM.select().count() - abandoned_gms = engine.main_db.GM.select(lambda g: g.timeid < now - engine.cleanup['expire']).count() - - # query games - threshold = 10 - total_games = 0 - running_games = 0 - with engine.cache.lock: - for gm in engine.cache.gms: - gm_cache = engine.cache.gms[gm] - with gm_cache.lock: - total_games += gm_cache.db.Game.select().count() - running_games += gm_cache.db.Game.select(lambda g: g.timeid >= now - threshold * 60).count() - done = time.time() - - # return data - return { - 'gms': { - 'total': total_gms, - 'abandoned': abandoned_gms - }, - 'games': { - 'total': total_games, - 'running': running_games - }, - 'query_time': done - now - } - - @get('/vtt/api/games-list/') - def api_games_list(gm_url: str): - start = time.time() - - # load GM from cache - gm_cache = engine.cache.get_from_url(gm_url) - if gm_cache is None: - # @NOTE: not logged because somebody may play around with this - abort(404) - - # query whether user is the hosting GM - session_gm = engine.main_db.GM.load_from_session(request) - if session_gm is None or session_gm.url != gm_url: - abort(404) - - data = { - 'games': [] - } - for game in gm_cache.db.Game.select(): - data['games'].append(game.url) - done = time.time() - - data['query_time'] = done - start - return data - - @get('/vtt/api/assets-list//') - def api_asset_list(gm_url: str, game_url: str): - start = time.time() - - # load GM from cache - gm_cache = engine.cache.get_from_url(gm_url) - if gm_cache is None: - # @NOTE: not logged because somebody may play around with this - abort(404) - - # query whether user is the hosting GM - session_gm = engine.main_db.GM.load_from_session(request) - if session_gm is None or session_gm.url != gm_url: - abort(404) - - # try to load game from GM's database - game = gm_cache.db.Game.select(lambda g: g.url == game_url).first() - root = ['./static/assets', engine.paths.get_assets_path()] - if game is not None: - root = [engine.paths.get_game_path(gm_url, game_url)] - - files = { - 'images': [], - 'audio': [] - } - for sub_root in root: - for filename in os.listdir(sub_root): - if filename.endswith('.png'): - files['images'].append(filename) - if filename.endswith('.mp3'): - files['audio'].append(filename) - if game is not None: - files['images'].sort(key=lambda k: int(k.split('.')[0])) - else: - files['images'].sort() - files['audio'].sort() - done = time.time() - - files['query_time'] = done - start - - return files - - @get('/vtt/api/cleanup') - def api_next_cleanup(): - when, until = engine.cleanup_worker.getNextUpdate() - return { - 'server time': str(when), - 'time left': str(until) - } - - @get('/vtt/api/build') - def api_build(): - return { - 'title': engine.title, - 'version': engine.version, - 'git_hash': engine.git_hash, - 'debug_hash': engine.debug_hash - } - - @get('/vtt/api/logins') - def api_query_logins(): - """Count users locations based on IPs within past 30d.""" - start = time.time() - - logins = engine.parse_login_log() - locations = dict() - platforms = dict() - browsers = dict() - since = start - 30 * 24 * 3600 # past 30d - - # group IPs by country - for record in logins: - if record.timeid < since: - continue - data = httpagentparser.detect(record.agent) - - # save data - add_dict_set(locations, record.country, record.ip) - if 'platform' in data: - add_dict_set(platforms, data['platform']['name'], record.ip) - else: - add_dict_set(platforms, 'unknown', record.ip) - if 'browser' in data: - add_dict_set(browsers, data['browser']['name'], record.ip) - else: - add_dict_set(browsers, 'unknown', record.ip) - - # count IPs per country, platform and browser - count_dict_set_len(locations) - count_dict_set_len(platforms) - count_dict_set_len(browsers) - - done = time.time() - - return { - 'locations': locations, - 'platforms': platforms, - 'browsers': browsers, - 'query_time': done - start - } - - @get('/vtt/api/auth') - def api_query_auth(): - # query gms and how many accounts do idle - now = time.time() - provider = {} - for gm in engine.main_db.GM.select(): - # fetch provider - p = '-'.join(gm.metadata.split('|')[:-1]) - for k in ['-oauth2', 'oauth2-']: - p = p.replace(k, '') - if p not in provider: - provider[p] = 1 - else: - provider[p] += 1 - done = time.time() - - provider['query_time'] = done - now - return provider From 382d4b3fbe4b91ce837a5082612c5f48f8f00d97 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 15:59:47 +0100 Subject: [PATCH 23/29] os.environ change before Engine ctor --- test/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common.py b/test/common.py index c2d6182..89e4002 100644 --- a/test/common.py +++ b/test/common.py @@ -86,12 +86,12 @@ def setUp(self) -> None: h.write('demo') # load engine app into webtest + EngineBaseTest.defaultEnviron() self.engine = Engine(argv=['--quiet', '--localhost'], pref_dir=self.root) self.engine.app.catchall = False self.app = webtest.TestApp(self.engine.app) self.monkeyPatch() - EngineBaseTest.defaultEnviron() def monkeyPatch(self) -> None: # save methods for later From f4ccbe9e6ede1807866bfc4b5dbb6e243e92b59e Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 16:33:56 +0100 Subject: [PATCH 24/29] non-linux is going to be supported more and more, hence this testcase is obsolete --- test/utils/test_path_api.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/utils/test_path_api.py b/test/utils/test_path_api.py index a0dadd7..ba2058c 100644 --- a/test/utils/test_path_api.py +++ b/test/utils/test_path_api.py @@ -38,13 +38,6 @@ def test_path_api_default(self): root = utils.PathApi(appname='unittest') self.assertEqual(root.pref_root, fake_path / '.local' / 'share' / 'unittest') - def test_path_api_non_linux(self): - patch = pytest.MonkeyPatch() - patch.setattr(sys, 'platform', 'test-system') - - with self.assertRaises(NotImplementedError): - root = utils.PathApi(appname='unittest') - def test_ensure(self): # @NOTE: ensure() is called by the constructor From 51241150a9a03015f4fa3c81a179fd94810c2d67 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 16:40:50 +0100 Subject: [PATCH 25/29] streamlined PathApi pref_root selection --- vtt/utils/path_api.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/vtt/utils/path_api.py b/vtt/utils/path_api.py index e9d8a0b..aafe39f 100644 --- a/vtt/utils/path_api.py +++ b/vtt/utils/path_api.py @@ -19,15 +19,12 @@ class PathApi: def __init__(self, appname: str, pref_root: pathlib.Path | None = None, app_root=pathlib.Path('..')) -> None: """ Uses given root or pick standard preference directory. """ self.app_root = app_root + self.pref_root = pref_root if pref_root is None: - # use current working directory - pref_root = pathlib.Path.cwd() - pref_root = pref_root / 'data' - else: - # we need to convert the string path value to a pathlib object - pref_root = pathlib.Path(pref_root) - - self.pref_root = pref_root / appname + # use current working directory instead + self.pref_root = pathlib.Path.cwd() / 'data' + + self.pref_root /= appname # make sure paths exists self.ensure(self.pref_root) From 836b5cec9582e2e485e65254059f9eb531c96827 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 16:46:52 +0100 Subject: [PATCH 26/29] fixed test_path_api.py --- test/utils/test_path_api.py | 16 +++++++++------- vtt/utils/path_api.py | 6 +++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/test/utils/test_path_api.py b/test/utils/test_path_api.py index ba2058c..ce24008 100644 --- a/test/utils/test_path_api.py +++ b/test/utils/test_path_api.py @@ -30,13 +30,15 @@ def tearDown(self): def assert_directory(self, p: pathlib.Path) -> None: self.assertTrue(p.exists()) - def test_path_api_default(self): - fake_path = pathlib.Path(self.tmpdir.name) / 'fake_home' - patch = pytest.MonkeyPatch() - patch.setattr(pathlib.Path, 'home', lambda: fake_path) - - root = utils.PathApi(appname='unittest') - self.assertEqual(root.pref_root, fake_path / '.local' / 'share' / 'unittest') + def test_path_api_with_given_prefroot(self): + # as setUp + expected = pathlib.Path(self.tmpdir.name) / 'data' / 'unittest' + self.assertEqual(self.paths.pref_root, expected) + + def test_path_api_without_specific_prefroot(self): + paths = utils.PathApi(appname='unittest') + expected = pathlib.Path.cwd() / 'data' / 'unittest' + self.assertEqual(paths.pref_root, expected) def test_ensure(self): # @NOTE: ensure() is called by the constructor diff --git a/vtt/utils/path_api.py b/vtt/utils/path_api.py index aafe39f..77b729c 100644 --- a/vtt/utils/path_api.py +++ b/vtt/utils/path_api.py @@ -19,12 +19,12 @@ class PathApi: def __init__(self, appname: str, pref_root: pathlib.Path | None = None, app_root=pathlib.Path('..')) -> None: """ Uses given root or pick standard preference directory. """ self.app_root = app_root - self.pref_root = pref_root if pref_root is None: # use current working directory instead - self.pref_root = pathlib.Path.cwd() / 'data' + pref_root = pathlib.Path.cwd() - self.pref_root /= appname + # note: `pref_root` may be passed in as `str` + self.pref_root = pathlib.Path(pref_root) / 'data' / appname # make sure paths exists self.ensure(self.pref_root) From 9d730784436a03302ab2b1371119fe68be6d49b1 Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 16:54:30 +0100 Subject: [PATCH 27/29] fixed unittest zip size to 20MiB --- test/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common.py b/test/common.py index 89e4002..acc42d2 100644 --- a/test/common.py +++ b/test/common.py @@ -64,7 +64,7 @@ def defaultEnviron(): os.environ['VTT_TITLE'] = 'unittest' os.environ['VTT_LIMIT_TOKEN'] =' 2' os.environ['VTT_LIMIT_BG'] = '10' - os.environ['VTT_LIMIT_GAME'] = '5' + os.environ['VTT_LIMIT_GAME'] = '20' os.environ['VTT_LIMIT_MUSIC'] = '10' os.environ['VTT_NUM_MUSIC'] = '5' os.environ['VTT_CLEANUP_EXPIRE'] = '3600' From 76eb8af3009788b0b96c5b094e044d2a8d0057df Mon Sep 17 00:00:00 2001 From: cgloeckner Date: Tue, 21 Jan 2025 18:49:24 +0100 Subject: [PATCH 28/29] register_api added back to app --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 8699900..459d70a 100644 --- a/main.py +++ b/main.py @@ -16,6 +16,7 @@ routes.register_player(engine) routes.register_resources(engine) routes.register_error(engine) + routes.register_api(engine) engine.cleanup_worker = CleanupThread(engine) engine.run() From dd2ceec90953083c9daa66df35cc2f50685b464d Mon Sep 17 00:00:00 2001 From: Scott Merrill Date: Wed, 22 Jan 2025 08:40:01 -0500 Subject: [PATCH 29/29] update content-type tests to use AssertIn; remove unix socket stuff --- test/routes/test_resources.py | 6 ++--- test/test_engine.py | 4 +-- test/test_server.py | 46 +++++++++++++++++------------------ vtt/engine.py | 2 +- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/test/routes/test_resources.py b/test/routes/test_resources.py index 71d923d..08743e5 100644 --- a/test/routes/test_resources.py +++ b/test/routes/test_resources.py @@ -50,7 +50,7 @@ def test_can_load_custom_static_file(self): new_file.touch() ret = self.app.get('/static/malicious.js') self.assertEqual(ret.status_int, 200) - self.assertEqual(ret.content_type, 'application/javascript') + self.assertIn('javascript', ret.content_type) def test_can_load_existing_png(self): ret = self.app.get('/static/d20.png') @@ -65,12 +65,12 @@ def test_can_load_existing_jpg(self): def test_can_load_favicon(self): ret = self.app.get('/static/favicon.ico') self.assertEqual(ret.status_int, 200) - self.assertEqual(ret.content_type, 'image/vnd.microsoft.icon') + self.assertIn('icon', ret.content_type) def test_can_load_existing_javascript_file(self): ret = self.app.get('/static/client/render.js') self.assertEqual(ret.status_int, 200) - self.assertEqual(ret.content_type, 'application/javascript') + self.assertIn('javascript', ret.content_type) def test_can_load_existing_css_file(self): ret = self.app.get('/static/client/layout.css') diff --git a/test/test_engine.py b/test/test_engine.py index 07f8ee7..f4ca732 100644 --- a/test/test_engine.py +++ b/test/test_engine.py @@ -69,10 +69,10 @@ def test_getPort(self): self.assertEqual(p, 8080) # reload with custom port - os.environ['VTT_PORT'] = '80' + os.environ['VTT_PORT'] = '8080' self.reloadEngine() p = self.engine.get_port() - self.assertEqual(p, 80) + self.assertEqual(p, 8080) def test_hasSsl(self): self.reloadEngine() diff --git a/test/test_server.py b/test/test_server.py index e81ff19..e574251 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -15,31 +15,31 @@ class ServerTest(unittest.TestCase): - def test_get_unix_socket_listener_creates_socket_file(self) -> None: - with tempfile.TemporaryDirectory() as name: - path = pathlib.Path(name) / 'test.sock' - self.assertFalse(path.exists()) - - listener = server.get_unix_socket_listener(path) - self.assertTrue(path.exists()) - self.assertIsNotNone(listener) - - def test_get_unix_socket_listener_replaces(self) -> None: - with tempfile.TemporaryDirectory() as name: - path = pathlib.Path(name) / 'test.sock' - path.touch() - self.assertTrue(path.exists()) - - listener = server.get_unix_socket_listener(path) - self.assertTrue(path.exists()) - self.assertIsNotNone(listener) +# def test_get_unix_socket_listener_creates_socket_file(self) -> None: +# with tempfile.TemporaryDirectory() as name: +# path = pathlib.Path(name) / 'test.sock' +# self.assertFalse(path.exists()) +# +# listener = server.get_unix_socket_listener(path) +# self.assertTrue(path.exists()) +# self.assertIsNotNone(listener) +# +# def test_get_unix_socket_listener_replaces(self) -> None: +# with tempfile.TemporaryDirectory() as name: +# path = pathlib.Path(name) / 'test.sock' +# path.touch() +# self.assertTrue(path.exists()) +# +# listener = server.get_unix_socket_listener(path) +# self.assertTrue(path.exists()) +# self.assertIsNotNone(listener) def test_can_create_Server_based_on_host_and_port(self) -> None: s = server.VttServer(host='example.com', port=1234) self.assertIsInstance(s.listener, tuple) - def test_can_create_Server_via_unix_socket(self) -> None: - with tempfile.TemporaryDirectory() as name: - path = pathlib.Path(name) / 'test.sock' - s = server.VttServer(host='example.com', port=1234, unixsocket=path) - self.assertIsInstance(s.listener, socket.socket) +# def test_can_create_Server_via_unix_socket(self) -> None: +# with tempfile.TemporaryDirectory() as name: +# path = pathlib.Path(name) / 'test.sock' +# s = server.VttServer(host='example.com', port=1234, unixsocket=path) +# self.assertIsInstance(s.listener, socket.socket) diff --git a/vtt/engine.py b/vtt/engine.py index acc18a8..16ad3a3 100644 --- a/vtt/engine.py +++ b/vtt/engine.py @@ -60,7 +60,7 @@ def __init__(self, app_root=pathlib.Path('.'), argv=list(), pref_dir=None): "domain" : os.getenv('VTT_DOMAIN', 'localhost'), "port" : int(os.getenv('VTT_PORT', 8080)), "ssl" : bool(os.getenv('VTT_SSL', False)), - "reverse" : bool(os.getenv('VTT_REVERSE_PROXY')) + "reverse" : bool(os.getenv('VTT_REVERSE_PROXY', False)) } self.main_db = None