Introduce support for jinja2-based templates.

.gitignore

@@ -1,4 +1,5 @@
 # Byte-compiled / optimized / DLL files
+__jinja__/
 __pycache__/
 *.py[cod]
 *$py.class

mig/shared/objecttypes.py

@@ -26,11 +26,14 @@
 # -- END_HEADER ---
 #
 
+from mig.shared.templates import init_global_templates
+
 """ Defines valid objecttypes and provides a method to verify if an object is correct """
 
 start = {'object_type': 'start', 'required': [], 'optional': ['headers'
                                                               ]}
 end = {'object_type': 'end', 'required': [], 'optional': []}
+template = {'object_type': 'template'}
 timing_info = {'object_type': 'timing_info', 'required': [],
                'optional': []}
 title = {'object_type': 'title', 'required': ['text'],
@@ -396,6 +399,7 @@
 valid_types_list = [
     start,
     end,
+    template,
     timing_info,
     title,
     text,
@@ -499,6 +503,8 @@
     image_settings_list,
 ]
 
+base_template_required = set(('template_name', 'template_group', 'template_args,'))
+
 # valid_types_dict = {"title":title, "link":link, "header":header}
 
 # autogenerate dict based on list. Dictionary access is prefered to allow
@@ -539,8 +545,8 @@ def get_object_type_info(object_type_list):
     return out
 
 
-def validate(input_object):
-    """ validate input_object """
+def validate(input_object, configuration=None):
+    """ validate presented objects against their definitions """
 
     if not type(input_object) == type([]):
         return (False, 'validate object must be a list' % ())
@@ -560,6 +566,19 @@ def validate(input_object):
 
             this_object_type = obj['object_type']
             valid_object_type = valid_types_dict[this_object_type]
+
+            if this_object_type == 'template':
+                # the required keys stuff below is not applicable to templates
+                # because templates know what they need in terms of data thus
+                # are self-documenting - use this fact to perform validation
+                #template_ref = "%s_%s.html" % (obj['template_group'], )
+                store = init_global_templates(configuration)
+                template = store.grab_template(obj['template_name'], obj['template_group'], 'html')
+                valid_object_type = {
+                    'required': store.extract_variables(template)
+                }
+                obj = obj.get('template_args', None)
+
             if 'required' in valid_object_type:
                 for req in valid_object_type['required']:
                     if req not in obj:

mig/shared/output.py

@@ -57,6 +57,7 @@
 from mig.shared.prettyprinttable import pprint_table
 from mig.shared.pwcrypto import sorted_hash_algos
 from mig.shared.safeinput import html_escape
+from mig.shared.templates import init_global_templates
 
 
 row_name = ('even', 'odd')
@@ -746,6 +747,15 @@ def html_format(configuration, ret_val, ret_msg, out_obj):
     for i in out_obj:
         if i['object_type'] == 'start':
             pass
+        elif i['object_type'] == 'template':
+            store = init_global_templates(configuration)
+            template = store.grab_template(
+                i['template_name'],
+                i['template_group'],
+                'html',
+                store.context.extend(**i['template_args'])
+            )
+            lines.append(template.render())
         elif i['object_type'] == 'error_text':
             msg = "%(text)s" % i
             if i.get('exc', False):

mig/shared/templates/__init__.py

@@ -0,0 +1,137 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+from collections import ChainMap
+import errno
+from jinja2 import meta as jinja2_meta, select_autoescape, Environment, \
+    FileSystemLoader, FileSystemBytecodeCache
+import os
+import weakref
+
+from mig.shared.defaults import MIG_BASE
+
+TEMPLATES_DIR = os.path.abspath(os.path.dirname(__file__))
+TEMPLATES_CACHE_DIR = os.path.join(TEMPLATES_DIR, '__jinja__')
+
+_all_template_dirs = [
+    os.path.join(TEMPLATES_DIR, 'pages'),
+    os.path.join(TEMPLATES_DIR, 'partials')
+]
+_global_store = None
+
+
+def cache_dir():
+    return TEMPLATES_CACHE_DIR
+
+
+def template_dirs():
+    return _all_template_dirs
+
+
+class _FormatContext:
+    def __init__(self, configuration):
+        self.output_format = None
+        self.configuration = configuration
+        self.conf_map = ChainMap(configuration)
+        self.script_map = {}
+        self.style_map = {}
+
+    def __getitem__(self, key):
+        return self.__dict__[key]
+
+    def __iter__(self):
+        return iter(self.__dict__)
+
+    def extend(self, **kwargs):
+        return ChainMap(kwargs, self)
+
+
+class TemplateStore:
+    def __init__(self, template_dirs, cache_dir=None, extra_globals=None):
+        assert cache_dir is not None
+
+        self._template_globals = extra_globals
+        self._template_environment = Environment(
+            loader=FileSystemLoader(template_dirs),
+            bytecode_cache=FileSystemBytecodeCache(cache_dir, '%s'),
+            autoescape=select_autoescape()
+        )
+
+    @property
+    def context(self):
+        return self._template_globals
+
+    def _get_template(self, template_fqname):
+        return self._template_environment.get_template(template_fqname)
+
+    def grab_template(self, template_name, template_group, output_format, template_globals=None, **kwargs):
+        template_fqname = "%s_%s.%s.jinja" % (
+            template_group, template_name, output_format)
+        return self._template_environment.get_template(template_fqname, globals=template_globals)
+
+    def list_templates(self):
+        return self._template_environment.list_templates()
+
+    def extract_variables(self, template_fqname):
+        template = self._template_environment.get_template(template_fqname)
+        with open(template.filename) as f:
+            template_source = f.read()
+        ast = self._template_environment.parse(template_source)
+        return jinja2_meta.find_undeclared_variables(ast)
+
+    @staticmethod
+    def populated(template_dirs, cache_dir=None, context=None):
+        assert cache_dir is not None
+
+        try:
+            os.mkdir(cache_dir)
+        except OSError as direxc:
+            if direxc.errno != errno.EEXIST:  # FileExistsError
+                raise
+
+        store = TemplateStore(
+            template_dirs, cache_dir=cache_dir, extra_globals=context)
+
+        for template_fqname in store.list_templates():
+            store._get_template(template_fqname)
+
+        return store
+
+
+def init_global_templates(configuration):
+    global _global_store
+
+    if _global_store is not None:
+        return _global_store
+
+    _global_store = TemplateStore.populated(
+        template_dirs(),
+        cache_dir=cache_dir(),
+        context=_FormatContext(configuration)
+    )
+
+    return _global_store

mig/shared/templates/__main__.py

@@ -0,0 +1,77 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# --- BEGIN_HEADER ---
+#
+# base - shared base helper functions
+# Copyright (C) 2003-2024  The MiG Project by the Science HPC Center at UCPH
+#
+# This file is part of MiG.
+#
+# MiG is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# MiG is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# -- END_HEADER ---
+#
+
+from types import SimpleNamespace
+import os
+import sys
+
+from mig.shared.conf import get_configuration_object
+from mig.shared.templates import init_global_templates
+
+
+def warn(message):
+    print(message, file=sys.stderr, flush=True)
+
+
+def main(args, _print=print):
+    configuration = get_configuration_object(config_file=args.config_file)
+    _template_environment = init_global_templates(configuration)
+
+    command = args.command
+    if command == 'show':
+        print(_template_environment.list_templates())
+    elif command == 'prime':
+        try:
+            os.mkdir(cache_dir())
+        except FileExistsError:
+            pass
+
+        for template_name in _list_templates():
+            _get_template(template_name)
+    elif command == 'vars':
+        for template_ref in _template_environment.list_templates():
+            _print("<%s>" % (template_ref,))
+            for var in _grab_template_vars(template_ref):
+                _print("  %s" % (var,))
+    else:
+        raise RuntimeError("unknown command: %s" % (command,))
+
+
+if __name__ == '__main__':
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-c', dest='config_file', default=None)
+    parser.add_argument('command')
+    args = parser.parse_args()
+
+    try:
+        main(args)
+        sys.exit(0)
+    except Exception as exc:
+        warn(str(exc))
+        sys.exit(1)

mig/shared/templates/partials/partial_other.html.jinja

@@ -0,0 +1 @@
+<q>{{ other }}</q>

mig/shared/templates/partials/partial_something.html.jinja

@@ -0,0 +1 @@
+<aside>{{ content }}</aside>

mig/wsgi-bin/migwsgi.py

@@ -130,7 +130,7 @@ def stub(configuration, client_id, import_path, backend, user_arguments_dict,
         crash_helper(configuration, backend, output_objects)
         return (output_objects, returnvalues.ERROR)
 
-    (val_ret, val_msg) = validate(output_objects)
+    (val_ret, val_msg) = validate(output_objects, configuration=configuration)
     if not val_ret:
         (ret_code, ret_msg) = returnvalues.OUTPUT_VALIDATION_ERROR
         bailout_helper(configuration, backend, output_objects,

requirements.txt

@@ -26,6 +26,8 @@ email-validator;python_version >= "3.7"
 email-validator<2.0;python_version >= "3" and python_version < "3.7"
 email-validator<1.3;python_version < "3"
 
+jinja2
+
 # NOTE: additional optional dependencies depending on site conf are listed
 #       in recommended.txt and can be installed in the same manner by pointing
 #       pip there.

tests/snapshots/test_objects_with_type_template.html

@@ -0,0 +1 @@
+<aside>here!!</aside>

tests/test_mig_shared_templates.py

@@ -0,0 +1,32 @@
+from __future__ import print_function
+
+from tests.support import MigTestCase, testmain
+
+from mig.shared.templates import TemplateStore, cache_dir, template_dirs
+
+
+class TestMigSharedTemplates(MigTestCase):
+    def _provide_configuration(self):
+        return 'testconfig'
+
+    def test_the_creation_of_a_template_store(self):
+        store = TemplateStore.populated(template_dirs(), cache_dir=cache_dir())
+        self.assertIsInstance(store, TemplateStore)
+
+    def test_a_listing_all_templates(self):
+        store = TemplateStore.populated(template_dirs(), cache_dir=cache_dir())
+        self.assertEqual(len(store.list_templates()), 2)
+
+    def test_grab_template(self):
+        store = TemplateStore.populated(template_dirs(), cache_dir=cache_dir())
+        template = store.grab_template('other', 'partial', 'html')
+        pass
+
+    def test_variables_for_remplate_ref(self):
+        store = TemplateStore.populated(template_dirs(), cache_dir=cache_dir())
+        template_vars = store.extract_variables('partial_something.html.jinja')
+        self.assertEqual(template_vars, set(['content']))
+
+
+if __name__ == '__main__':
+    testmain()

tests/test_mig_wsgibin.py

@@ -303,6 +303,32 @@ def test_objects_with_type_text(self):
         output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200)
         self.assertSnapshotOfHtmlContent(output)
 
+    def test_objects_with_type_template(self):
+        output_objects = [
+            # workaround invalid HTML being generated with no title object
+            {
+                'object_type': 'title',
+                'text': 'TEST'
+            },
+            {
+                'object_type': 'template',
+                'template_name': 'something',
+                'template_group': 'partial',
+                'template_args': {
+                    'content': 'here!!'
+                }
+            }
+        ]
+        self.fake_backend.set_response(output_objects, returnvalues.OK)
+
+        wsgi_result = migwsgi.application(
+            *self.application_args,
+            **self.application_kwargs
+        )
+
+        output, _ = self.assertWsgiResponse(wsgi_result, self.fake_wsgi, 200)
+        self.assertSnapshotOfHtmlContent(output)
+
 
 if __name__ == '__main__':
     testmain()