diff options
| -rw-r--r-- | R-overlay.conf | 9 | ||||
| -rw-r--r-- | roverlay/__init__.py | 4 | ||||
| -rw-r--r-- | roverlay/config.py | 705 | ||||
| -rw-r--r-- | roverlay/config/__init__.py | 39 | ||||
| -rw-r--r-- | roverlay/config/const.py (renamed from roverlay/const.py) | 0 | ||||
| -rw-r--r-- | roverlay/config/entrymap.py | 82 | ||||
| -rw-r--r-- | roverlay/config/fielddef.py | 102 | ||||
| -rw-r--r-- | roverlay/config/loader.py | 363 | ||||
| -rw-r--r-- | roverlay/config/tree.py | 215 | ||||
| -rw-r--r-- | roverlay/config/util.py | 25 | ||||
| -rwxr-xr-x | run_config.py | 2 |
11 files changed, 838 insertions, 708 deletions
diff --git a/R-overlay.conf b/R-overlay.conf index 3d99153..938fd32 100644 --- a/R-overlay.conf +++ b/R-overlay.conf @@ -5,3 +5,12 @@ LOG_FILE = log/roverlay.log LOG_FILE_RESOLVED = log/dep_resolved.log LOG_FILE_UNRESOLVABLE = log/dep_unresolvable.log + +SIMPLE_RULES_FILE = "simple-deprules.conf" + +OVERLAY_NAME = ROverlay +OVERLAY_DIR = /tmp/ROverlay_1 +OVERLAY_CATEGORY = sci-R +ECLASS = eclass/R-packages.eclass + +#REPO = "x;/somewhere;rsync://?;http://? a;;rsync://?;ftp://?" diff --git a/roverlay/__init__.py b/roverlay/__init__.py index b0478ec..e33811d 100644 --- a/roverlay/__init__.py +++ b/roverlay/__init__.py @@ -6,8 +6,8 @@ import logging from roverlay import config -config.access().load_config ( 'R-overlay.conf' ) -config.access().load_field_definition ( 'description_fields.conf' ) +config.loader().load_config ( 'R-overlay.conf' ) +config.loader().load_field_definition ( 'description_fields.conf' ) logging.basicConfig ( level=logging.DEBUG, diff --git a/roverlay/config.py b/roverlay/config.py deleted file mode 100644 index d09e430..0000000 --- a/roverlay/config.py +++ /dev/null @@ -1,705 +0,0 @@ -# R overlay -- config module -# Copyright 2006-2012 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import os.path -import re -import sys -import shlex - -try: - import configparser -except ImportError as running_python2: - # configparser is named ConfigParser in python2 - import ConfigParser as configparser - - -from roverlay import const -from roverlay.rpackage import descriptionfields - - -CONFIG_INJECTION_IS_BAD = True - -def access(): - """Returns the ConfigTree.""" - return ConfigTree() if ConfigTree.instance is None else ConfigTree.instance -# --- end of access (...) --- - -def get_config_path ( key ): - """Creates a config path for key.""" - _path = key.split ( '.' ) if isinstance ( key, str ) else key - if isinstance ( _path, ( list, tuple ) ): - # config paths are [ CAPSLOCK, CAPSLOCK,.... , lowercase item ] - return [ x.lower() if x == _path [-1] else x.upper() for x in _path ] - else: - return _path -# --- end of get_config_path (...) --- - -def get ( key, fallback_value=None, fail_if_unset=False ): - """Searches for key in the ConfigTree and returns its value if possible, - else fallback_value. - 'key' is a config path [<section>[.<subsection>*]]<option name>. - - arguments: - * key -- - * fallback_value -- - """ - if not fallback_value is None: - return access().get ( - key, fallback_value=fallback_value, fail_if_unset=fail_if_unset - ) - else: - return access().get ( - key, fallback_value=None, fail_if_unset=fail_if_unset - ) -# --- end of get (...) --- - -def get_or_fail ( key ): - return access().get ( key, fail_if_unset=True ) -# --- end of get_or_fail (...) --- - -class InitialLogger: - - def __init__ ( self ): - """Initializes an InitialLogger. - It implements the debug/info/warning/error/critical/exception methods - known from the logging module and its output goes directly to sys.stderr. - This can be used until the real logging has been configured. - """ - # @return None - self.debug = lambda x : sys.stdout.write ( "DBG %s\n" % x ) - self.info = lambda x : sys.stdout.write ( "INFO %s\n" % x ) - self.warning = lambda x : sys.stderr.write ( "WARN %s\n" % x ) - self.error = lambda x : sys.stderr.write ( "ERR %s\n" % x ) - self.critical = lambda x : sys.stderr.write ( "CRIT %s\n" % x ) - self.exception = lambda x : sys.stderr.write ( "EXC! %s\n" % x ) - - # --- end of __init__ (...) --- - -class ConfigTree ( object ): - # static access to the first created ConfigTree - instance = None - - # the map of config entries (keep keys in lowercase) - # format is config_entry = None|''|str|dict(...), where - # None means that config_entry is known but ignored, - # str means that config_entry is an alias for another config entry, - # '' means that config_entry uses defaults, - # dict() means that config_entry has options / diverts from defaults. - # - # known dict keys are: - # * path = str | list of str -- path of this entry in the config tree - # - # * value_type, you can specify: - # ** slist -- value is a whitespace-separated list - # ** list -- value is a list, see DEFAULT_LIST_REGEX below - # ** int -- integer - # ** yesno -- value must evaluate to 'yes' or 'no' (on,off,y,n,1,0...) - # ** fs_path -- ~ will be expanded - # ** fs_abs -- fs_path and path will be converted into an absolute one - # (pwd + path) - # ** fs_dir -- fs_abs and value must be a dir if it exists - # ** fs_file -- fs_abs and value must be a file if it exists - # TODO** fs_prog -- fs_file (and fs_path) and value must be executable (TODO) - # ** regex -- value is a regex and will be compiled (re.compile(..)) - # - # multiple types are generally not supported ('this is an int or a str'), - # but subtypes are ('list of yesno'), which can be specified by either - # using a list of types ['list', 'yesno'] or by separating the types - # with a colon 'list:yesno', which is parsed in a left-to-right order. - # Nested subtypes such as list:slist:int:fs_file:list may lead to errors. - # - CONFIG_ENTRY_MAP = dict ( - log_level = '', - log_console = dict ( - value_type = 'yesno', - ), - log_file = dict ( - # setting path to LOG.FILE.main to avoid collision with LOG.FILE.* - path = [ 'LOG', 'FILE', 'Main' ], - value_type = 'fs_file', - ), - log_file_resolved = dict ( - value_type = 'fs_file', - ), - log_file_unresolvable = dict ( - value_type = 'fs_file', - ), - ebuild_header = dict ( - value_type = 'fs_file', - ), - overlay_dir = dict ( - value_type = 'fs_dir', - ), - distfiles_dir = dict ( - value_type = 'fs_dir', - ), - ebuild_prog = dict ( - path = [ 'TOOLS', 'ebuild_prog' ], - value_type = 'fs_path', - ), - - ) - - # often used regexes - DEFAULT_LIST_REGEX = re.compile ( '\s*[,;]{1}\s*' ) - WHITESPACE = re.compile ( '\s+' ) - - - def __init__ ( self, import_const=True ): - """Initializes an ConfigTree, which is a container for options/values. - Values can be stored directly (such as the field_definitions) or - in a tree-like { section -> subsection[s] -> option = value } structure. - Config keys cannot contain dots because they're used as config path - separator. - - arguments: - * import_const -- whether to deepcopy constants into the config tree or - not. Copying allows faster lookups. - """ - if ConfigTree.instance is None: - ConfigTree.instance = self - - self.logger = InitialLogger() - - self.parser = dict() - - self._config = const.clone() if import_const else dict () - self._const_imported = import_const - self._field_definitions = None - - # --- end of __init__ (...) --- - - - def _findpath ( self, path, - root=None, create=False, value=None, forcepath=False, forceval=False - ): - """All-in-one method that searches for a config path. - It is able to create the path if non-existent and to assign a - value to it. - - arguments: - * path -- config path as path list ([a,b,c]) or as path str (a.b.c) - * root -- config root (dict expected). - Uses self._config if None (the default) - * create -- create path if nonexistent - * value -- assign value to the last path element - an empty dict will be created if this is None and - create is True - * forcepath -- if set and True: do not 'normalize' path if path is a list - * forceval -- if set and True: accept None as value - """ - if path is None: - return root - elif isinstance ( path, ( list, tuple ) ) and forcepath: - pass - else: - path = get_config_path ( path ) - - - config_position = self._config if root is None else root - - if config_position is None: return None - - for k in path: - if len (k) == 0: - continue - # FIXME use enumerate and check index - config keys like 'a.a.a'! - if k == path [-1] and ( forceval or not value is None ): - # overwrite entry - config_position [k] = value - elif not k in config_position: - if create: - config_position [k] = dict() - else: - return None - - config_position = config_position [k] - - return config_position - - # --- end of _findpath (...) --- - - def inject ( self, key, value, suppress_log=False, **kw_extra ): - """This method offer direct write access to the ConfigTree. No checks - will be performed, so make sure you know what you're doing. - - arguments: - * key -- config path of the entry to-be-created/overwritten - the whole path will be created, this operation does not fail - if a path component is missing ('<root>.<new>.<entry> creates - root, new and entry if required) - * value -- value to be assigned - * **kw_extra -- extra keywords for _findpath, e.g. forceval=True - - returns: None (implicit) - """ - if not suppress_log: - msg = 'config injection: value %s will '\ - 'be assigned to config key %s ...' % ( value, key ) - - if CONFIG_INJECTION_IS_BAD: - self.logger.warning ( msg ) - else: - self.logger.debug ( msg ) - - self._findpath ( key, create=True, value=value, **kw_extra ) - # --- end of inject (...) --- - - def get ( self, key, fallback_value=None, fail_if_unset=False ): - """Searches for key in the ConfigTree and returns its value. - Searches in const if ConfigTree does not contain the requested key and - returns the fallback_value if key not found. - - arguments: - * key -- - * fallback_value -- - * fail_if_unset -- fail if key is neither in config nor const - """ - - config_value = self._findpath ( key ) - - if config_value is None: - fallback = None if fail_if_unset else fallback_value - if not self._const_imported: - config_value = const.lookup ( key, fallback ) - else: - config_value = fallback - - if config_value is None and fail_if_unset: - raise Exception ( "config key '%s' not found but required." % key ) - - return config_value - - # --- end of get (...) --- - - def _add_entry ( self, option, value=None, config_root=None ): - """Adds an option to the config. - - arguments: - * option -- name of the option as it appears in the config file - * value -- value to assign, defaults to None - * config_root -- root of the config (a dict), defaults to None which is - later understood as self._config - """ - - def make_and_verify_value ( value_type, value, entryconfig_ref ): - """Prepares the value of a config option so that it can be used - in the ConfigTree. - - arguments: - * value_type -- type of the value, - look above for explanation concerning this - * value -- value to verify and transform - * entryconfig_ref -- reference to the config entry config - """ - - def to_int ( val, fallback_value=-1 ): - """Tries to convert val to an int, returning a fallback value - on any error. - - arguments: - * val -- - * fallback_value -- - - catches: ValueError in case of an unsuccesful int conversion - raises: nothing - """ - try: - ret = int ( val ) - return ret - except ValueError as verr: - return fallback_value - # --- end of to_int (...) --- - - def yesno ( val ): - """Tries to canonize an yes or no value to its integer - representation. Returns 1 if val means 'yes', 0 if 'no' and - -1 otherwise. - - arguments: - * val -- - """ - if not val is None: - to_check = str ( val ) . lower () - if to_check in [ 'y', 'yes', '1', 'true', 'enabled', 'on' ]: - return 1 - elif to_check in [ 'n', 'no', '0', 'false', 'disabled', 'off' ]: - return 0 - - self.logger.warning ( to_check + " is not a valid yesno value." ) - return -1 - # --- end of yesno (...) --- - - def fs_path ( val ): - """val is a filesystem path - returns expanded path (~ -> HOME). - - arguments: - * val -- - """ - return os.path.expanduser ( val ) if val else None - # --- end of fs_path (...) --- - - def fs_abs ( val ): - """val is a filesystem path - returns absolute + expanded path.""" - if val: - return os.path.abspath ( os.path.expanduser ( val ) ) - else: - return None - - - def fs_file ( val ): - """"val is a file - returns expanded path if it is - an existent file or it does not exist. - - arguments: - * val -- - """ - retval = fs_abs ( val ) - if retval: - if os.path.isfile ( retval ) or not os.path.exists ( retval ): - return retval - - return None - # --- end of fs_file (...) --- - - def fs_dir ( val ): - """val is a directory -- returns expanded path if it is - an existent dir or it does not exist. - - arguments: - * val -- - """ - retval = fs_abs ( val ) - if retval: - if os.path.isdir ( retval ) or not os.path.exists ( retval ): - return retval - - return None - # --- end of fs_dir (...) --- - - def _regex ( val ): - """val is a regex -- compile it if possible - - arguments: - * val -- - """ - return re.compile ( val ) if not val is None else None - # --- end of _regex (...) --- - - # replace whitespace with a single ' ' - value = ConfigTree.WHITESPACE.sub ( ' ', value ) - - # convert value_type into a list of value types - if not value_type: - return value - elif isinstance ( value_type, list ): - vtypes = value_type - elif isinstance ( value_type, str ): - vtypes = value_type.split ( ':' ) - else: - self.logger.error ( "Unknown data type for value type." ) - return value - - # value_type -> function where function accepts one parameter - funcmap = { - 'list' : ConfigTree.DEFAULT_LIST_REGEX.split, - 'slist' : ConfigTree.WHITESPACE.split, - 'yesno' : yesno, - 'int' : to_int, - 'fs_path' : fs_path, - 'fs_file' : fs_file, - 'regex' : _regex, - } - - # dofunc ( function f, <list or str> v) calls f(x) for every str in v - dofunc = lambda f, v : [ f(x) for x in v ] \ - if isinstance ( v, list ) else f(v) - - retval = value - - for vtype in vtypes: - if vtype in funcmap: - retval = dofunc ( funcmap [vtype], retval ) - else: - self.logger.warning ( "unknown value type '" + vtype + "'." ) - - return retval - # --- end of make_and_verify_value (...) --- - - - real_option = option - low_option = option.lower() - - # known option? - if option and low_option in ConfigTree.CONFIG_ENTRY_MAP: - - original_cref = cref = ConfigTree.CONFIG_ENTRY_MAP [low_option] - cref_level = 0 - - # check if cref is a link to another entry in CONFIG_ENTRY_MAP - while isinstance ( cref, str ) and cref != '': - if cref == original_cref and cref_level: - self.logger.critical ( - "CONFIG_ENTRY_MAP is invalid! circular cref detected." - ) - raise Exception ( "CONFIG_ENTRY_MAP is invalid!" ) - - elif cref in ConfigTree.CONFIG_ENTRY_MAP: - option = low_option = cref - cref = ConfigTree.CONFIG_ENTRY_MAP [cref] - cref_level += 1 - else: - self.logger.critical ( - 'CONFIG_ENTRY_MAP is invalid! ' - 'last cref = %s, current cref = %s.' % ( option, cref ) - ) - raise Exception ( "CONFIG_ENTRY_MAP is invalid!" ) - - # check if config entry is disabled - if cref is None: - # deftly ignored - return True - - - # determine the config path - path = None - if 'path' in cref: - path = cref ['path'] - else: - path = option.split ( '_' ) - - path = get_config_path ( path ) - - # need a valid path - if path: - - # verify and convert value if value_type is set - if 'value_type' in cref: - value = make_and_verify_value ( - cref ['value_type'], value, cref - ) - - # need a valid value - if value: - - self.logger.debug ( - "New config entry %s with path %s and value %s." % - ( option, path, value ) - ) - - # add option/value to the config - self._findpath ( path, config_root, True, value ) - - return True - else: - self.logger.error ( - "Option '%s' has an unusable value '%s'." % - ( real_option, value ) - ) - return False - # --- - - self.logger.error ( "Option '%s' is unusable..." % real_option ) - return False - # --- - - self.logger.warning ( "Option '%s' is unknown." % real_option ) - return False - - # --- end of _add_entry (...) --- - - def load_config ( self, config_file, start_section='' ): - """Loads a config file and integrates its content into the config tree. - Older config entries may be overwritten. - - arguments: - config_file -- path to the file that should be read - start_section -- relative root in the config tree as str - """ - - config_root = None - if start_section: - if isinstance ( start_section, str ): - config_root = self._findpath ( start_section, None, True ) - else: - raise Exception ("bad usage") - - # load file - - try: - fh = open ( config_file, 'r' ) - reader = shlex.shlex ( fh ) - reader.wordchars += ' ./$()[]:+-@*~' - reader.whitespace_split = False - - - - nextline = lambda : ( reader.get_token() for n in range (3) ) - - option, equal, value = nextline () - - while equal == '=' or not ( option == value == reader.eof ): - if equal == '=': - self._add_entry ( option, value, config_root ) - else: - - self.logger.warning ( - "In '%s', cannot parse this line: '%s%s%s'." % - ( config_file, option, equal, value ) - ) - - option, equal, value = nextline () - - if fh: - fh.close () - - except IOError as ioerr: - raise - - # --- end of load_config (...) --- - - def load_field_definition ( self, def_file, lenient=False ): - """Loads a field definition file. - Please see the example file for format details. - - arguments: - * def_file -- file (str) to read, - this can be a list of str if lenient is True - * lenient -- if True: do not fail if a file cannot be read; - defaults to False - """ - if not 'field_def' in self.parser: - self.parser ['field_def'] = \ - configparser.SafeConfigParser ( allow_no_value=True ) - - try: - self.logger.debug ( - "Reading description field definition file %s." % def_file - ) - if lenient: - self.parser ['field_def'] . read ( def_file ) - else: - fh = open ( def_file, 'r' ) - self.parser ['field_def'] . readfp ( fh ) - - if fh: fh.close() - except IOError as err: - self.logger.exception ( err ) - raise - except configparser.MissingSectionHeaderError as mshe: - self.logger.exception ( mshe ) - raise - - # --- end of load_field_definition (...) --- - - - def get_field_definition ( self, force_update=False ): - """Gets the field definition stored in this ConfigTree. - - arguments: - * force_update -- enforces recreation of the field definition data. - """ - if force_update or not self._field_definitions: - self._field_definitions = self._make_field_definition () - - return self._field_definitions - - # --- end of get_field_definition (...) --- - - - def _make_field_definition ( self ): - """Creates and returns field definition data. Please see the example - field definition config file for details. - """ - - def get_list ( value_str ): - if value_str is None: - return [] - else: - l = value_str.split ( ', ' ) - return [ e for e in l if e.strip() ] - - if not 'field_def' in self.parser: return None - - fdef = descriptionfields.DescriptionFields () - - for field_name in self.parser ['field_def'].sections(): - field = descriptionfields.DescriptionField ( field_name ) - for option, value in self.parser ['field_def'].items( field_name, 1 ): - - if option == 'alias' or option == 'alias_withcase': - for alias in get_list ( value ): - field.add_simple_alias ( alias, True ) - - elif option == 'alias_nocase': - for alias in get_list ( value ): - field.add_simple_alias ( alias, False ) - - elif option == 'default_value': - field.set_default_value ( value ) - - elif option == 'allowed_value': - field.add_allowed_value ( value ) - - elif option == 'allowed_values': - for item in get_list ( value ): - field.add_allowed_value ( item ) - - elif option == 'flags': - for flag in get_list ( value ): - field.add_flag ( flag ) - else: - # treat option as flag - field.add_flag ( option ) - - fdef.add ( field ) - - return fdef - - # --- end of _make_field_definition (...) --- - - - def _tree_to_str ( self, root, name, level=0 ): - """Returns string representation of a config tree rooted at root. - Uses recursion (DFS). - - arguments: - * root -- config 'root', is a value (config 'leaf') or a dict ('tree') - * name -- - * level -- - - returns: string representation of the given root - """ - - indent = level * ' ' - var_indent = indent + '* ' - if root is None: - return "%s%s is unset\n" % ( var_indent, name ) - elif len ( root ) == 0: - return "%s%s is empty\n" % ( var_indent, name ) - elif isinstance ( root, dict ): - extra = ''.join ( [ - self._tree_to_str ( n, r, level+1 ) for r, n in root.items() - ] ) - return "%s%s {\n%s%s}\n" % ( indent, name, extra, indent ) - else: - return "%s%s = '%s'\n" % ( var_indent, name, root ) - # --- end of _tree_to_str (...) --- - - def visualize ( self, into=None ): - """Visualizes the ConfigTree, - either into a file-like object or as return value. - - arguments: - * into -- if not None: write into file - - returns: string if into is None, else None (implicit) - """ - _vis = self._tree_to_str ( self._config, 'ConfigTree', level=0 ) - if into is None: - return _vis - else: - into.write ( _vis ) - # --- end of visualize (...) --- - diff --git a/roverlay/config/__init__.py b/roverlay/config/__init__.py new file mode 100644 index 0000000..9640c9f --- /dev/null +++ b/roverlay/config/__init__.py @@ -0,0 +1,39 @@ +# R overlay -- config module +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from roverlay.config.tree import ConfigTree + +CONFIG_INJECTION_IS_BAD = True + +def access(): + """Returns the ConfigTree.""" + return ConfigTree() if ConfigTree.instance is None else ConfigTree.instance +# --- end of access (...) --- + +def loader(): + return access().get_loader() +# --- end of get_loader (...) --- + +def get ( key, fallback_value=None, fail_if_unset=False ): + """Searches for key in the ConfigTree and returns its value if possible, + else fallback_value. + 'key' is a config path [<section>[.<subsection>*]]<option name>. + + arguments: + * key -- + * fallback_value -- + """ + if not fallback_value is None: + return access().get ( + key, fallback_value=fallback_value, fail_if_unset=fail_if_unset + ) + else: + return access().get ( + key, fallback_value=None, fail_if_unset=fail_if_unset + ) +# --- end of get (...) --- + +def get_or_fail ( key ): + return access().get ( key, fail_if_unset=True ) +# --- end of get_or_fail (...) --- diff --git a/roverlay/const.py b/roverlay/config/const.py index a69a33a..a69a33a 100644 --- a/roverlay/const.py +++ b/roverlay/config/const.py diff --git a/roverlay/config/entrymap.py b/roverlay/config/entrymap.py new file mode 100644 index 0000000..6e206e5 --- /dev/null +++ b/roverlay/config/entrymap.py @@ -0,0 +1,82 @@ +# <header!!> + +# TODO/FIXME comments + +# the map of config entries (keep keys in lowercase) +# format is config_entry = None|''|str|dict(...), where +# None means that config_entry is known but ignored, +# str means that config_entry is an alias for another config entry, +# '' means that config_entry uses defaults, +# dict() means that config_entry has options / diverts from defaults. +# +# known dict keys are: +# * path = str | list of str -- path of this entry in the config tree +# +# * value_type, you can specify: +# ** slist -- value is a whitespace-separated list +# ** list -- value is a list, see DEFAULT_LIST_REGEX below +# ** int -- integer +# ** str -- [explicit string conversion] +# ** yesno -- value must evaluate to 'yes' or 'no' (on,off,y,n,1,0...) +# ** fs_path -- ~ will be expanded +# ** fs_abs -- fs_path and path will be converted into an absolute one +# (pwd + path) +# ** fs_dir -- fs_abs and value must be a dir if it exists +# ** fs_file -- fs_abs and value must be a file if it exists +# TODO** fs_prog -- fs_file (and fs_path) and value must be executable (TODO) +# ** regex -- value is a regex and will be compiled (re.compile(..)) +# +# multiple types are generally not supported ('this is an int or a str'), +# but subtypes are ('list of yesno'), which can be specified by either +# using a list of types ['list', 'yesno'] or by separating the types +# with a colon 'list:yesno', which is parsed in a left-to-right order. +# Nested subtypes such as list:slist:int:fs_file:list may lead to errors. +# +CONFIG_ENTRY_MAP = dict ( + log_level = '', + log_console = dict ( + value_type = 'yesno', + ), + log_file = dict ( + # setting path to LOG.FILE.main to avoid collision with LOG.FILE.* + path = [ 'LOG', 'FILE', 'Main' ], + value_type = 'fs_file', + ), + log_file_resolved = dict ( + value_type = 'fs_file', + ), + log_file_unresolvable = dict ( + value_type = 'fs_file', + ), + ebuild_header = dict ( + value_type = 'fs_file', + ), + overlay_category = dict ( + value_type = 'str', + ), + overlay_eclass = dict ( + path = [ 'OVERLAY', 'eclass_files' ], + value_type = 'list:fs_abs:fs_file', + ), + eclass = 'overlay_eclass', + overlay_dir = dict ( + value_type = 'fs_abs:fs_dir', + ), + overlay_name = dict ( + value_type = 'str', + ), + distfiles_dir = dict ( + value_type = 'fs_dir', + ), + ebuild_prog = dict ( + path = [ 'TOOLS', 'ebuild_prog' ], + value_type = 'fs_path', + ), + simple_rules_files = dict ( + path = [ 'DEPRES', 'SIMPLE_RULES', 'files' ], + value_type = 'list:fs_abs', + ), + simple_rules_file = 'simple_rules_files', + remote = 'repo', + repo = '', +) diff --git a/roverlay/config/fielddef.py b/roverlay/config/fielddef.py new file mode 100644 index 0000000..cb3f2c3 --- /dev/null +++ b/roverlay/config/fielddef.py @@ -0,0 +1,102 @@ +# R overlay -- config, <?> +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +try: + import configparser +except ImportError as running_python2: + # configparser is named ConfigParser in python2 + import ConfigParser as configparser + +from roverlay.rpackage import descriptionfields + + +class DescriptionFieldDefinition ( object ): + + + def __init__ ( self, logger ): + self.logger = logger + self._parser = configparser.SafeConfigParser ( allow_no_value=True ) + + # --- end of __init__ (...) --- + + def load_file ( self, def_file, lenient=False ): + """Loads a field definition file. + Please see the example file for format details. + + arguments: + * def_file -- file (str) to read, + this can be a list of str if lenient is True + * lenient -- if True: do not fail if a file cannot be read; + defaults to False + """ + + try: + self.logger.debug ( + "Reading description field definition file %s." % def_file + ) + if lenient: + self._parser.read ( def_file ) + else: + fh = open ( def_file, 'r' ) + self._parser.readfp ( fh ) + + if fh: fh.close() + except IOError as err: + self.logger.exception ( err ) + raise + except configparser.MissingSectionHeaderError as mshe: + self.logger.exception ( mshe ) + raise + + # --- end of load_file (...) --- + + def get ( self ): + """Creates and returns field definition data. Please see the example + field definition config file for details. + """ + + def get_list ( value_str ): + if value_str is None: + return [] + else: + l = value_str.split ( ', ' ) + return [ e for e in l if e.strip() ] + + + fdef = descriptionfields.DescriptionFields () + + for field_name in self._parser.sections(): + field = descriptionfields.DescriptionField ( field_name ) + for option, value in self._parser.items ( field_name, 1 ): + + if option == 'alias' or option == 'alias_withcase': + for alias in get_list ( value ): + field.add_simple_alias ( alias, True ) + + elif option == 'alias_nocase': + for alias in get_list ( value ): + field.add_simple_alias ( alias, False ) + + elif option == 'default_value': + field.set_default_value ( value ) + + elif option == 'allowed_value': + field.add_allowed_value ( value ) + + elif option == 'allowed_values': + for item in get_list ( value ): + field.add_allowed_value ( item ) + + elif option == 'flags': + for flag in get_list ( value ): + field.add_flag ( flag ) + else: + # treat option as flag + field.add_flag ( option ) + + fdef.add ( field ) + + return fdef + + # --- end of get (...) --- diff --git a/roverlay/config/loader.py b/roverlay/config/loader.py new file mode 100644 index 0000000..62f0980 --- /dev/null +++ b/roverlay/config/loader.py @@ -0,0 +1,363 @@ +# R overlay -- config module +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import re +import shlex +import os.path + +from roverlay.config import fielddef +from roverlay.config.util import get_config_path, unquote +from roverlay.config.entrymap import CONFIG_ENTRY_MAP + +class ConfigLoader ( object ): + + # often used regexes + DEFAULT_LIST_REGEX = re.compile ( '\s*[,;]{1}\s*' ) + WHITESPACE = re.compile ( '\s+' ) + + + def __init__ ( self, config_tree, logger=None ): + self.ctree = config_tree + + self.config_root = None + + self.logger = self.ctree.logger if logger is None else logger + self.fielddef = None + + # --- end of __init__ (...) --- + + def _setval ( self, path, value, allow_empty_value=False ): + self.ctree._findpath ( + path, + value=value, + root=self.config_root, + create=True, + forceval=allow_empty_value, + forcepath=False + ) + # --- end of _setval (...) --- + + def _config_entry ( self, cref, option, value, config_root ): + # determine the config path + path = None + if 'path' in cref: + path = cref ['path'] + else: + path = option.split ( '_' ) + + path = get_config_path ( path ) + + # need a valid path + if path: + + # verify and convert value if value_type is set + if 'value_type' in cref: + value = self._make_and_verify_value ( + cref ['value_type'], value + ) + + # need a valid value + if value: + + self.logger.debug ( + "New config entry %s with path %s and value %s." % + ( option, path, value ) + ) + + # add option/value to the config + self._setval ( path, value ) + + return True + else: + self.logger.error ( + "Option '%s' has an unusable value '%s'." % + ( real_option, value ) + ) + return False + # --- + # --- end of _config_enty (...) --- + + def _add_entry ( self, option, value=None, config_root=None ): + """Adds an option to the config. + + arguments: + * option -- name of the option as it appears in the config file + * value -- value to assign, defaults to None + * config_root -- root of the config (a dict), defaults to None which is + later understood as self._config + """ + + real_option = option + low_option = option.lower() + + # known option? option not empty and its lowercase repr in the entry map + if option and low_option in CONFIG_ENTRY_MAP: + + original_cref = cref = CONFIG_ENTRY_MAP [low_option] + cref_level = 0 + + # check if cref is a link to another entry in CONFIG_ENTRY_MAP + while isinstance ( cref, str ) and cref != '': + if cref == original_cref and cref_level: + self.logger.critical ( + "CONFIG_ENTRY_MAP is invalid! circular cref detected." + ) + raise Exception ( "CONFIG_ENTRY_MAP is invalid!" ) + + elif cref in CONFIG_ENTRY_MAP: + option = low_option = cref + cref = CONFIG_ENTRY_MAP [cref] + cref_level += 1 + else: + self.logger.critical ( + 'CONFIG_ENTRY_MAP is invalid! ' + 'last cref = %s, current cref = %s.' % ( option, cref ) + ) + raise Exception ( "CONFIG_ENTRY_MAP is invalid!" ) + + # check if config entry is disabled + if cref is None: + # deftly ignored + return True + + elif self._config_entry ( cref, option, value, config_root ): + return True + else: + self.logger.error ( "Option '%s' is unusable..." % real_option ) + return False + # --- + + self.logger.warning ( "Option '%s' is unknown." % real_option ) + return False + + # --- end of _add_entry (...) --- + + def load_config ( self, config_file ): + """Loads a config file and integrates its content into the config tree. + Older config entries may be overwritten. + + arguments: + config_file -- path to the file that should be read + """ + + # load file + + try: + fh = open ( config_file, 'r' ) + reader = shlex.shlex ( fh ) + reader.wordchars += ' ,./$()[]:+-@*~' + reader.whitespace_split = False + + + nextline = lambda : ( reader.get_token() for n in range (3) ) + + option, equal, value = nextline () + + while equal == '=' or not ( option == value == reader.eof ): + if equal == '=': + self._add_entry ( option, value ) + else: + + self.logger.warning ( + "In '%s', cannot parse this line: '%s%s%s'." % + ( config_file, option, equal, value ) + ) + + option, equal, value = nextline () + + if fh: + fh.close () + + except IOError as ioerr: + raise + + # --- end of load_config (...) --- + + def load_field_definition ( self, def_file, lenient=False ): + """Loads a field definition file. + Please see the example file for format details. + + arguments: + * def_file -- file (str) to read, + this can be a list of str if lenient is True + * lenient -- if True: do not fail if a file cannot be read; + defaults to False + """ + if self.fielddef is None: + self.fielddef = fielddef.DescriptionFieldDefinition ( + self.logger + ) + + self.fielddef.load_file ( def_file, lenient=lenient ) + + self.ctree._field_definition = self.fielddef.get() + # --- end of load_field_definition (...) --- + + def _make_and_verify_value ( self, value_type, value ): + """Prepares the value of a config option so that it can be used + in the ConfigLoader. + + arguments: + * value_type -- type of the value, + look above for explanation concerning this + * value -- value to verify and transform + """ + + def to_int ( val, fallback_value=-1 ): + """Tries to convert val to an int, returning a fallback value + on any error. + + arguments: + * val -- + * fallback_value -- + + catches: ValueError in case of an unsuccesful int conversion + raises: nothing + """ + try: + ret = int ( val ) + return ret + except ValueError as verr: + return fallback_value + # --- end of to_int (...) --- + + def yesno ( val ): + """Tries to canonize an yes or no value to its integer + representation. Returns 1 if val means 'yes', 0 if 'no' and + -1 otherwise. + + arguments: + * val -- + """ + if not val is None: + to_check = str ( val ) . lower () + if to_check in [ 'y', 'yes', '1', 'true', 'enabled', 'on' ]: + return 1 + elif to_check in [ 'n', 'no', '0', 'false', 'disabled', 'off' ]: + return 0 + + self.logger.warning ( to_check + " is not a valid yesno value." ) + return -1 + # --- end of yesno (...) --- + + def fs_path ( val ): + """val is a filesystem path - returns expanded path (~ -> HOME). + + arguments: + * val -- + """ + return os.path.expanduser ( val ) if val else None + # --- end of fs_path (...) --- + + def fs_abs ( val ): + """val is a filesystem path - returns absolute + expanded path.""" + if val: + return os.path.abspath ( os.path.expanduser ( val ) ) + else: + return None + + + def fs_file ( val ): + """"val is a file - returns expanded path if it is + an existent file or it does not exist. + + arguments: + * val -- + """ + retval = fs_abs ( val ) + if retval: + if os.path.isfile ( retval ) or not os.path.exists ( retval ): + return retval + + return None + # --- end of fs_file (...) --- + + def fs_dir ( val ): + """val is a directory -- returns expanded path if it is + an existent dir or it does not exist. + + arguments: + * val -- + """ + retval = fs_abs ( val ) + if retval: + if os.path.isdir ( retval ) or not os.path.exists ( retval ): + return retval + + return None + # --- end of fs_dir (...) --- + + def repo ( val ): + if not val: return None + + name, sepa, remainder = val.partition ( ':' ) + + if sepa != ':' or not name or not remainder: return None + + if remainder [0] == ':': + # name::url + return ( name, None, remainder [1:] ) + + elif remainder [0] == '/': + # name:dir:url + _dir, sepa, url = remainder.partition ( ':' ) + if sepa != ':' or not url: return None + if not _dir: _dir = None + return ( name, _dir, url ) + else: + return ( name, None, remainder ) + # --- end of repo (...) --- + + def _regex ( val ): + """val is a regex -- compile it if possible + + arguments: + * val -- + """ + return re.compile ( val ) if not val is None else None + # --- end of _regex (...) --- + + # replace whitespace with a single ' ' + value = unquote ( ConfigLoader.WHITESPACE.sub ( ' ', value ) ) + + # convert value_type into a list of value types + if not value_type: + return value + elif isinstance ( value_type, list ): + vtypes = value_type + elif isinstance ( value_type, str ): + vtypes = value_type.split ( ':' ) + else: + self.logger.error ( "Unknown data type for value type." ) + return value + + # value_type -> function where function accepts one parameter + funcmap = { + 'list' : ConfigLoader.DEFAULT_LIST_REGEX.split, + 'slist' : ConfigLoader.WHITESPACE.split, + 'yesno' : yesno, + 'int' : to_int, + 'fs_dir' : fs_dir, + 'fs_path' : fs_path, + 'fs_file' : fs_file, + 'fs_abs' : fs_abs, + 'regex' : _regex, + 'str' : str, + 'repo' : repo, + } + + # dofunc ( function f, <list or str> v) calls f(x) for every str in v + dofunc = lambda f, v : [ f(x) for x in v ] \ + if isinstance ( v, list ) else f(v) + + retval = value + + for vtype in vtypes: + if vtype in funcmap: + retval = dofunc ( funcmap [vtype], retval ) + else: + self.logger.warning ( "unknown value type '" + vtype + "'." ) + + return retval + # --- end of _make_and_verify_value (...) --- diff --git a/roverlay/config/tree.py b/roverlay/config/tree.py new file mode 100644 index 0000000..b38bc2c --- /dev/null +++ b/roverlay/config/tree.py @@ -0,0 +1,215 @@ +# R overlay -- config module +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +import sys + +from roverlay.config import const +from roverlay.config.loader import ConfigLoader +from roverlay.config.util import get_config_path + +CONFIG_INJECTION_IS_BAD = True + +class InitialLogger: + + def __init__ ( self ): + """Initializes an InitialLogger. + It implements the debug/info/warning/error/critical/exception methods + known from the logging module and its output goes directly to sys.stderr. + This can be used until the real logging has been configured. + """ + # @return None + self.debug = lambda x : sys.stdout.write ( "DBG %s\n" % x ) + self.info = lambda x : sys.stdout.write ( "INFO %s\n" % x ) + self.warning = lambda x : sys.stderr.write ( "WARN %s\n" % x ) + self.error = lambda x : sys.stderr.write ( "ERR %s\n" % x ) + self.critical = lambda x : sys.stderr.write ( "CRIT %s\n" % x ) + self.exception = lambda x : sys.stderr.write ( "EXC! %s\n" % x ) + + # --- end of __init__ (...) --- + +class ConfigTree ( object ): + # static access to the first created ConfigTree + instance = None + + def __init__ ( self, import_const=True ): + """Initializes an ConfigTree, which is a container for options/values. + Values can be stored directly (such as the field_definitions) or + in a tree-like { section -> subsection[s] -> option = value } structure. + Config keys cannot contain dots because they're used as config path + separator. + + arguments: + * import_const -- whether to deepcopy constants into the config tree or + not. Copying allows faster lookups. + """ + if ConfigTree.instance is None: + ConfigTree.instance = self + + self.logger = InitialLogger() + + self._config = const.clone() if import_const else dict () + self._const_imported = import_const + self._field_definition = None + + # --- end of __init__ (...) --- + + def get_loader ( self ): + return ConfigLoader ( self ) + # --- end of get_loader (...) --- + + def _findpath ( self, path, + root=None, create=False, value=None, forcepath=False, forceval=False + ): + """All-in-one method that searches for a config path. + It is able to create the path if non-existent and to assign a + value to it. + + arguments: + * path -- config path as path list ([a,b,c]) or as path str (a.b.c) + * root -- config root (dict expected). + Uses self._config if None (the default) + * create -- create path if nonexistent + * value -- assign value to the last path element + an empty dict will be created if this is None and + create is True + * forcepath -- if set and True: do not 'normalize' path if path is a list + * forceval -- if set and True: accept None as value + """ + if path is None: + return root + elif isinstance ( path, ( list, tuple ) ) and forcepath: + pass + else: + path = get_config_path ( path ) + + + config_position = self._config if root is None else root + + if config_position is None: return None + + last = len ( path ) - 1 + + for index, k in enumerate ( path ): + if len (k) == 0: + continue + + if index == last and ( forceval or not value is None ): + # overwrite entry + config_position [k] = value + elif not k in config_position: + if create: + config_position [k] = dict() + else: + return None + + config_position = config_position [k] + + return config_position + + # --- end of _findpath (...) --- + + def inject ( self, key, value, suppress_log=False, **kw_extra ): + """This method offer direct write access to the ConfigTree. No checks + will be performed, so make sure you know what you're doing. + + arguments: + * key -- config path of the entry to-be-created/overwritten + the whole path will be created, this operation does not fail + if a path component is missing ('<root>.<new>.<entry> creates + root, new and entry if required) + * value -- value to be assigned + * **kw_extra -- extra keywords for _findpath, e.g. forceval=True + + returns: None (implicit) + """ + if not suppress_log: + msg = 'config injection: value %s will '\ + 'be assigned to config key %s ...' % ( value, key ) + + if CONFIG_INJECTION_IS_BAD: + self.logger.warning ( msg ) + else: + self.logger.debug ( msg ) + + self._findpath ( key, create=True, value=value, **kw_extra ) + # --- end of inject (...) --- + + def get ( self, key, fallback_value=None, fail_if_unset=False ): + """Searches for key in the ConfigTree and returns its value. + Searches in const if ConfigTree does not contain the requested key and + returns the fallback_value if key not found. + + arguments: + * key -- + * fallback_value -- + * fail_if_unset -- fail if key is neither in config nor const + """ + + config_value = self._findpath ( key ) + + if config_value is None: + fallback = None if fail_if_unset else fallback_value + if not self._const_imported: + config_value = const.lookup ( key, fallback ) + else: + config_value = fallback + + if config_value is None and fail_if_unset: + raise Exception ( "config key '%s' not found but required." % key ) + + return config_value + + # --- end of get (...) --- + + def get_field_definition ( self, force_update=False ): + """Gets the field definition stored in this ConfigTree. + + arguments: + * force_update -- enforces recreation of the field definition data. + """ + return self._field_definition + # --- end of get_field_definition (...) --- + + def _tree_to_str ( self, root, name, level=0 ): + """Returns string representation of a config tree rooted at root. + Uses recursion (DFS). + + arguments: + * root -- config 'root', is a value (config 'leaf') or a dict ('tree') + * name -- + * level -- + + returns: string representation of the given root + """ + + indent = level * ' ' + var_indent = indent + '* ' + if root is None: + return "%s%s is unset\n" % ( var_indent, name ) + elif len ( root ) == 0: + return "%s%s is empty\n" % ( var_indent, name ) + elif isinstance ( root, dict ): + extra = ''.join ( [ + self._tree_to_str ( n, r, level+1 ) for r, n in root.items() + ] ) + return "%s%s {\n%s%s}\n" % ( indent, name, extra, indent ) + else: + return "%s%s = '%s'\n" % ( var_indent, name, root ) + # --- end of _tree_to_str (...) --- + + def visualize ( self, into=None ): + """Visualizes the ConfigTree, + either into a file-like object or as return value. + + arguments: + * into -- if not None: write into file + + returns: string if into is None, else None (implicit) + """ + _vis = self._tree_to_str ( self._config, 'ConfigTree', level=0 ) + if into is None: + return _vis + else: + into.write ( _vis ) + # --- end of visualize (...) --- diff --git a/roverlay/config/util.py b/roverlay/config/util.py new file mode 100644 index 0000000..c1aef66 --- /dev/null +++ b/roverlay/config/util.py @@ -0,0 +1,25 @@ +# R Overlay -- config, <?> +# Copyright 2006-2012 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +def get_config_path ( key ): + """Creates a config path for key.""" + _path = key.split ( '.' ) if isinstance ( key, str ) else key + if isinstance ( _path, ( list, tuple ) ): + # config paths are [ CAPSLOCK, CAPSLOCK,.... , lowercase item ] + return [ x.lower() if x == _path [-1] else x.upper() for x in _path ] + else: + return _path +# --- end of get_config_path (...) --- + +def unquote ( _str ): + if len ( _str ) < 2: return _str + chars = '"' + chars += "'" + + for c in chars: + if _str [0] == c and _str [-1] == c: + return _str[1:-1] + + return _str +# --- end of unquote (...) --- diff --git a/run_config.py b/run_config.py index 84f2740..b6e1cab 100755 --- a/run_config.py +++ b/run_config.py @@ -8,7 +8,7 @@ from roverlay import config for c in ARGV: print ( "<=== " + c + " ===>" ) - config.access().load_config ( c ) + config.loader().load_config ( c ) print ( ">=== " + c + " ===<" ) conf = config.access() |