Implement config for actors

* Add actor config to the leapp.conf config file
* Add an attribute for configuration schema to LEAPP Actors
* Create API to load config for actors and validate it against the schemas stored in actors.
* Create a function to retrieve the configuration that an actor has specified.

etc/leapp/leapp.conf

@@ -4,3 +4,5 @@ repo_path=/etc/leapp/repos.d/
 [database]
 path=/var/lib/leapp/leapp.db
 
+[actor_config]
+path=/etc/leapp/actor_conf.d/

leapp/actors/__init__.py

@@ -2,6 +2,7 @@
 import os
 import sys
 
+from leapp.actors.config import retrieve_config
 from leapp.compat import string_types
 from leapp.dialogs import Dialog
 from leapp.exceptions import (MissingActorAttributeError, RequestStopAfterPhase, StopActorExecution,
@@ -41,6 +42,11 @@ class Actor(object):
        Write the actor's description as a docstring.
     """
 
+    config_schemas = ()
+    """
+    Defines the structure of the configuration that the actor uses.
+    """
+
     consumes = ()
     """
     Tuple of :py:class:`leapp.models.Model` derived classes defined in the :ref:`repositories <terminology:repository>`
@@ -86,6 +92,7 @@ def serialize(self):
             'path': os.path.dirname(sys.modules[type(self).__module__].__file__),
             'class_name': type(self).__name__,
             'description': self.description or type(self).__doc__,
+            'config_schemas': self.config_schemas,
             'consumes': [c.__name__ for c in self.consumes],
             'produces': [p.__name__ for p in self.produces],
             'tags': [t.__name__ for t in self.tags],
@@ -100,15 +107,20 @@ def __init__(self, messaging=None, logger=None, config_model=None, skip_dialogs=
         This depends on the definition of such a configuration model being defined by the workflow
         and an actor that provides such a message.
         """
+
         Actor.current_instance = self
         install_translation_for_actor(type(self))
         self._messaging = messaging
         self.log = (logger or logging.getLogger('leapp.actors')).getChild(self.name)
         self.skip_dialogs = skip_dialogs
         """ A configured logger instance for the current actor. """
 
+        # self._configuration is the workflow configuration.
+        # self.config_schemas is the actor defined configuration.
+        # self.config is the actual actor configuration
         if config_model:
             self._configuration = next(self.consume(config_model), None)
+        self.config = retrieve_config(self.config_schemas)
 
         self._path = path
 
@@ -470,6 +482,8 @@ def get_actor_metadata(actor):
         _get_attribute(actor, 'dialogs', _is_dialog_tuple, required=False, default_value=()),
         _get_attribute(actor, 'description', _is_type(string_types), required=False,
                        default_value=actor.__doc__ or 'There has been no description provided for this actor.'),
+        _get_attribute(actor, 'config_schemas', _is_type(string_types), required=False,
+                       default_value=actor.__doc__ or 'Description of the configuration used by this actor.')
         _get_attribute(actor, 'apis', _is_api_tuple, required=False, default_value=())
     ])
 

leapp/actors/config.py

@@ -0,0 +1,330 @@
+"""
+Config file format:
+    yaml file like this:
+
+---
+# Note: have to add a fields.Map type before we can use yaml mappings.
+section_name:
+    field1_name: value
+    field2_name:
+        - listitem1
+        - listitem2
+section2_name:
+    field3_name: value
+
+Config files are any yaml files in /etc/leapp/actor_config.d/
+(This is settable in /etc/leapp/leapp.conf)
+
+"""
+__metaclass__ = type
+
+import abc
+import glob
+import importlib
+import logging
+import os.path
+import pkgutil
+from collections import defaultdict
+
+import six
+import yaml
+
+try:
+    # Compiled versions if available, for speed
+    from yaml import CSafeLoader as SafeLoader, CSafeDumper as SafeDumper
+except ImportError:
+    from yaml import SafeLoader, SafeDumper
+
+
+_ACTOR_CONFIG = None
+_ACTOR_CONFIG_VALIDATED = False
+
+log = logging.getLogger('leapp.actors.config')
+
+
+class SchemaError(Exception):
+    """Raised when a schema fails validation."""
+
+
+class ValidationError(Exception):
+    """
+    Raised when a config file fails to validate against any of the available schemas.
+    """
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Config:
+    """
+    An Actor config schema looks like this.
+
+    ::
+        class RHUIConfig(Config):
+            section = "rhui"
+            name = "file_map"
+            type_ = fields.Map(fields.String())
+            description = 'Description here'
+            default = {"repo": "url"}
+    """
+    @abc.abstractproperty
+    def section(self):
+        pass
+
+    @abc.abstractproperty
+    def name(self):
+        pass
+
+    @abc.abstractproperty
+    def type_(self):
+        pass
+
+    @abc.abstractproperty
+    def description(self):
+        pass
+
+    @abc.abstractproperty
+    def default(self):
+        pass
+
+    @classmethod
+    def to_dict(cls):
+        """
+        Return a dictionary representation of the config item that would be suitable for putting
+        into a config file.
+        """
+        representation = {
+            cls.section: {
+                '{0}_description__'.format(cls.name): cls.description
+            }
+        }
+        ### TODO: Retrieve the default values from the type field.
+        # representation[cls.section][cls.name] = cls.type_.get_default()
+
+        return representation
+
+
+def _merge_config(configuration, new_config):
+    """
+    Merge two dictionaries representing configuration.  fields in new_config overwrite
+    any existing fields of the same name in the same section in configuration.
+    """
+    for section_name, section in new_config.items():
+        if section_name not in configuration:
+            configuration[section_name] = section
+        else:
+            for field_name, field in section:
+                configuration[section_name][field_name] = field
+
+
+def _get_config(config_dir='/etc/leapp/actor_conf.d'):
+    """
+    Read all configuration files from the config_dir and return a dict with their values.
+    """
+    config_files = glob.glob(os.path.join(config_dir, '*'), recursive=True)
+    config_files = [f for f in config_files if f.endswith('.yml') or f.endswith('.yaml')]
+    config_files.sort()
+
+    configuration = {}
+    for config_file in config_files:
+        with open(config_file) as f:
+            raw_cfg = f.read()
+
+        try:
+            parsed_config = yaml.load(raw_cfg, SafeLoader)
+        except Exception as e:
+            log.warning("Warning: unparsable yaml file %s in the config directory."
+                    " Error: %s", filename, str(e))
+            raise
+
+        _merge_config(configuration, parsed_config)
+
+    return configuration
+
+
+def _normalize_schemas(schemas):
+    """
+    Merge all schemas into a single dictionary and validate them for errors we can detect.
+    """
+    added_fields = set()
+    normalized_schema = {}
+    for schema in schemas:
+        for field in schema:
+            unique_name = (field.section, field.name)
+
+            # Error if the field has been added by another schema
+            if unique_name in added_fields and added_fields[unique_name] != field:
+                # TODO: Also have information on what Actor contains the conflicting fields
+                message = "Two actors added incompatible values for {name}".format(name=unique_name))
+                log.error(message)
+                raise SchemaError(message)
+
+            # TODO: More validation here.
+
+            # Store the fields from the schema in a way that we can easily look
+            # up while validating
+            added_fields.add(unique_name)
+            normalized_schema[field.section][field.name] = field
+
+    return normalized_schema
+
+
+def _validate_field_type(field_type, field_value):
+    """
+    Return False if the field is not of the proper type.
+    """
+    # TODO: I took a quick look at the Model code and this is what I came up
+    # with.  This might not work right or there might be a much better way.
+    try:
+        field_type.create(field_value)
+    except Exception:
+        return False
+    return True
+
+
+def _normalize_config(actor_config, schema):
+    for section_name, section in actor_config.items():
+        for field_name in actor_config:
+            if (section_name, field_name) not in added_fields:
+                # TODO: Also have information about which config file contains the unknown field.
+                message = "A config file contained an unknown field: {name}".format(name=(section_name, field_name))
+                log.warning(message)
+
+    normalized_actor_config = {}
+
+    for section_name, section in schema.items():
+        for field_name, field in section.items():
+            # For every item in the schema, either retrieve the value from the
+            # config files or set it to the default.
+            try:
+                value = actor_config[section_name][field_name]
+            except KeyError:
+                # Either section_name or field_name doesn't exist
+                section = actor_config[section_name] = actor_config.get(section_name, {})
+                # May need to deepcopy default if these values are modified.
+                # However, it's probably an error if they are modified and we
+                # should possibly look into disallowing that.
+                value = section[field_name] = schema[section_name, field_name].default
+
+            if not _validate_field(schema[section_name][field_name].type_, value):
+                raise ValidationError("Config value for {name} is not of the correct type".format(name=(section_name, field_name)))
+
+            normalized_section = normalized_actor_config.get(section_name, {})
+            normalized_section[field_name] = value
+            # If the section already exists, this is a no-op.  Otherwise, it
+            # sets it to the newly created dict.
+            normalized_actor_config[section_name] = normalized_section
+
+    return normalized_actor_config
+
+
+def load(config_dir, schemas):
+    """
+    Return Actor Configuration.
+
+    :returns: a dict representing the configuration.
+    :raises ValueError: if the actor configuration does not match the schema.
+
+    This function reads the config, validates it, and adds any default values.
+    """
+    global _ACTOR_CONFIG
+    if _ACTOR_CONFIG:
+        return _ACTOR_CONFIG
+
+    # TODO: Move this to the caller
+    schema = _normalize_schemas(schemas)
+    # End TODO
+    config = _get_config(config_dir)
+    config = _normalize_config(config, schema)
+
+    _ACTOR_CONFIG = config
+    return _ACTOR_CONFIG
+
+
+def retrieve_config(schema):
+    """Called by the actor to retrieve the actor configuration specific to this actor."""
+    # TODO: This isn't good API. Since this function is called by the Actors,
+    # we *know* that this is okay to do (as the configuration will have already
+    # been loaded.) However, there's nothing in the API that ensures that this
+    # is the case.  Need to redesign this.  Can't think of how it should look
+    # right now because loading requires information that the Actor doesn't
+    # know.
+    global _ACTOR_CONFIG
+    all_actor_config = _ACTOR_CONFIG
+
+    configuration = defaultdict(dict)
+    for field in schema:
+        configuration[field.section][field.name] = all_actor_config[field.section][field.name]
+
+    return configuration
+
+#
+# From this point down hasn't been re-evaluated to see if it's needed or how it
+# fits into the bigger picture.  Some of it definitely has been implemented in
+# a different way above but not all of it.
+#
+
+def parse_repo_config_files():
+    repo_config = {}
+    for config in all_repository_config_schemas():
+        section_name = config.section
+
+        if section_name not in repo_config:
+            repo_config.update(config.to_dict())
+        else:
+            if '{0}_description__'.format(config_item.name) in repo_config[config.section]:
+                raise Exception("Error: Two configuration items are declared with the same name Section: {0}, Key: {1}".format(config.section, config.name))
+
+            repo_config[config.section].update(config.to_dict()[config.section])
+
+    return repo_config
+
+
+def parse_config_files(config_dir):
+    """
+    Parse all configuration and return a dict with those values.
+    """
+    config = parse_repo_config_files()
+    system_config = parse_system_config_files(config_dir)
+
+    for section, config_items in system_config.items():
+        if section not in config:
+            print('WARNING: config file contains an unused section: Section: {0}'.format(section))
+            config.update[section] = config_items
+        else:
+            for key, value in config_items:
+                if '{0}_description__'.format(key) not in config[section]:
+                    print('WARNING: config file contains an unused config entry: Section: {0}, Key{1}'.format(section, key))
+
+                config[section][key] = value
+
+    return config
+
+
+def format_config():
+    """
+    Read the configuration definitions from all of the known repositories and return a string that
+    can be used as an example config file.
+
+    Example config file:
+        transaction:
+            to_install_description__: |
+                List of packages to be added to the upgrade transaction.
+                Signed packages which are already installed will be skipped.
+            to_remove_description__: |
+                List of packages to be removed from the upgrade transaction
+                initial-setup should be removed to avoid it asking for EULA acceptance during upgrade
+            to_remove:
+                - initial-setup
+            to_keep_description__: |
+                List of packages to be kept in the upgrade transaction
+            to_keep:
+                - leapp
+                - python2-leapp
+                - python3-leapp
+                - leapp-repository
+                - snactor
+    """
+    return SafeDumper(yaml.dump(parse_config_files(), dumper=SafeDumper))
+    if schemas != actor_config:
+        raise Exception("Invalid entries in the actor config files")
+
+    global_ _ACTOR_CONFIG_VALIDATED = True