From: Ondrej Lichtner olichtne@redhat.com
These modules are only used by the controller so they should be placed in the Controller/ directory. We originally placed them in Common/ because we expected external tools to use them and have them dependent on the lnst-common package, however now it seems that having them dependent on the lnst-ctl package will be better. The reason behind it is that if you're working with recipes in an external tool you're probably also running lnst-ctl on the same machine.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Common/XmlParser.py | 185 --------------- lnst/Common/XmlProcessing.py | 205 ---------------- lnst/Common/XmlTemplates.py | 425 ---------------------------------- lnst/Controller/RecipeParser.py | 7 +- lnst/Controller/SlaveMachineParser.py | 5 +- lnst/Controller/SlavePool.py | 2 +- lnst/Controller/XmlParser.py | 185 +++++++++++++++ lnst/Controller/XmlProcessing.py | 205 ++++++++++++++++ lnst/Controller/XmlTemplates.py | 425 ++++++++++++++++++++++++++++++++++ 9 files changed, 823 insertions(+), 821 deletions(-) delete mode 100644 lnst/Common/XmlParser.py delete mode 100644 lnst/Common/XmlProcessing.py delete mode 100644 lnst/Common/XmlTemplates.py create mode 100644 lnst/Controller/XmlParser.py create mode 100644 lnst/Controller/XmlProcessing.py create mode 100644 lnst/Controller/XmlTemplates.py
diff --git a/lnst/Common/XmlParser.py b/lnst/Common/XmlParser.py deleted file mode 100644 index 4cde0b4..0000000 --- a/lnst/Common/XmlParser.py +++ /dev/null @@ -1,185 +0,0 @@ -""" -This module contains the XmlParser and LnstParser classes. - -Copyright 2013 Red Hat, Inc. -Licensed under the GNU General Public License, version 2 as -published by the Free Software Foundation; see COPYING for details. -""" - -__author__ = """ -rpazdera@redhat.com (Radek Pazdera) -""" - -import os -import re -import sys -import logging -import copy -from lxml import etree -from lnst.Common.Config import lnst_config -from lnst.Common.XmlTemplates import XmlTemplates, XmlTemplateError -from lnst.Common.XmlProcessing import XmlProcessingError, XmlData - -class XmlParser(object): - XINCLUDE_RE = r"{http://www.w3.org/[0-9]{4}/XInclude}include" - - def __init__(self, schema_file, xml_path): - # locate the schema file - # try git path - dirname = os.path.dirname(sys.argv[0]) - schema_path = os.path.join(dirname, schema_file) - if not os.path.exists(schema_path): - # try configuration - res_dir = lnst_config.get_option("environment", "resource_dir") - schema_path = os.path.join(res_dir, schema_file) - - if not os.path.exists(schema_path): - raise Exception("The recipe schema file was not found. " + \ - "Your LNST installation is corrupt!") - - self._template_proc = XmlTemplates() - - self._path = xml_path - relaxng_doc = etree.parse(schema_path) - self._schema = etree.RelaxNG(relaxng_doc) - - def parse(self): - doc = self._parse(self._path) - self._remove_comments(doc) - - # Due to a weird implementation of XInclude in lxml, the - # XmlParser resolves included documents on it's own. - # - # To be able to tell later on where each tag was located - # in the XML document, we add a '__file' attribute to - # each element of the tree during the parsing. - # - # However, these special attributes are of course not - # valid according to our schemas. To solve this, a copy of - # the tree is made and the '__file' attributes are removed - # before validation. - # - # XXX This is a *EXTREMELY* dirty hack. Ideas/proposals - # for cleaner solutions are more than welcome! - root_tag = self._init_loc(doc.getroot(), self._path) - self._expand_xinclude(root_tag, os.path.dirname(self._path)) - - self._template_proc.process_aliases(root_tag) - - try: - self._validate(doc) - except: - err = self._schema.error_log[0] - loc = {"file": os.path.basename(err.filename), - "line": err.line, "col": err.column} - exc = XmlProcessingError(err.message) - exc.set_loc(loc) - raise exc - - return self._process(root_tag) - - def _parse(self, path): - try: - doc = etree.parse(path) - except etree.LxmlError as err: - # A workaround for cases when lxml (quite strangely) - # sets the filename to <string>. - if err.error_log[0].filename == "<string>": - filename = self._path - else: - filename = err.error_log[0].filename - loc = {"file": os.path.basename(filename), - "line": err.error_log[0].line, - "col": err.error_log[0].column} - exc = XmlProcessingError(err.error_log[0].message) - exc.set_loc(loc) - raise exc - except Exception as err: - loc = {"file": os.path.basename(self._path), - "line": None, - "col": None} - exc = XmlProcessingError(str(err)) - exc.set_loc(loc) - raise exc - - return doc - - def _process(self, root_tag): - pass - - def set_machines(self, machines): - self._template_proc.set_machines(machines) - - def set_aliases(self, defined, overriden): - self._template_proc.set_aliases(defined, overriden) - - def _has_attribute(self, element, attr): - return attr in element.attrib - - def _get_attribute(self, element, attr): - text = element.attrib[attr].strip() - return self._template_proc.expand_functions(text) - - def _get_content(self, element): - text = etree.tostring(element, method="text").strip() - return self._template_proc.expand_functions(text) - - def _expand_xinclude(self, elem, base_url=""): - for e in elem: - if re.match(self.XINCLUDE_RE, str(e.tag)): - href = os.path.join(base_url, e.get("href")) - filename = os.path.basename(href) - - doc = self._parse(href) - self._remove_comments(doc) - node = doc.getroot() - - node = self._init_loc(node, href) - - if e.tail: - node.tail = (node.tail or "") + e.tail - self._expand_xinclude(node, os.path.dirname(href)) - - parent = e.getparent() - if parent is None: - return node - - parent.replace(e, node) - else: - self._expand_xinclude(e, base_url) - return elem - - def _remove_comments(self, doc): - comments = doc.xpath('//comment()') - for c in comments: - p = c.getparent() - if p is not None: - p.remove(c) - - def _init_loc(self, elem, filename): - """ Remove all coment tags from the tree """ - - elem.attrib["__file"] = filename - for e in elem: - self._init_loc(e, os.path.basename(filename)) - - return elem - - def _validate(self, original): - """ - Make a copy of the tree, remove the '__file' attributes - and validate against the appropriate schema. - - Very unfortunate solution. - """ - doc = copy.deepcopy(original) - root = doc.getroot() - - self._prepare_tree_for_validation(root) - self._schema.assertValid(doc) - - def _prepare_tree_for_validation(self, elem): - if "__file" in elem.attrib: - del elem.attrib["__file"] - for e in elem: - self._prepare_tree_for_validation(e) diff --git a/lnst/Common/XmlProcessing.py b/lnst/Common/XmlProcessing.py deleted file mode 100644 index 771ead5..0000000 --- a/lnst/Common/XmlProcessing.py +++ /dev/null @@ -1,205 +0,0 @@ -""" -This module contains code code for XML parsing and processing. - -Copyright 2012 Red Hat, Inc. -Licensed under the GNU General Public License, version 2 as -published by the Free Software Foundation; see COPYING for details. -""" - -__author__ = """ -rpazdera@redhat.com (Radek Pazdera) -""" - -import os -import logging - -class XmlProcessingError(Exception): - """ Exception thrown on parsing errors """ - - _filename = None - _line = None - _col = None - - def __init__(self, msg, obj=None): - super(XmlProcessingError, self).__init__() - self._msg = msg - - if obj is not None: - if hasattr(obj, "loc"): - self.set_loc(obj.loc) - elif hasattr(obj, "attrib") and "__file" in obj.attrib: - loc = {} - loc["file"] = obj.attrib["__file"] - if hasattr(obj, "sourceline"): - loc["line"] = obj.sourceline - self.set_loc(loc) - elif hasattr(obj, "base") and obj.base != None: - loc = {} - loc["file"] = os.path.basename(obj.base) - if hasattr(obj, "sourceline"): - loc["line"] = obj.sourceline - self.set_loc(loc) - - - def set_loc(self, loc): - self._filename = loc["file"] - self._line = loc["line"] - if "col" in loc: - self._col = loc["col"] - - def __str__(self): - line = "" - col = "" - sep = "" - loc = "" - filename = "<unknown>" - - if self._filename: - filename = self._filename - - if self._line: - line = "%d" % self._line - sep = ":" - - if self._col: - col = "%s%d" % (sep, self._col) - - if self._line or self._col: - loc = "%s%s:" % (line, col) - - return "%s:%s %s" % (filename, loc, self._msg) - -class XmlDataIterator: - def __init__(self, iterator): - self._iterator = iterator - - def __iter__(self): - return self - - def next(self): - n = self._iterator.next() - - # For normal iterators - if type(n) == XmlTemplateString: - return str(n) - - # For iteritems() iterators - if type(n) == tuple and len(n) == 2 and type(n[1]) == XmlTemplateString: - return (n[0], str(n[1])) - - return n - -class XmlCollection(list): - def __init__(self, node=None): - super(XmlCollection, self).__init__() - if node is not None: - if hasattr(node, "loc"): - self.loc = node.loc - elif "__file" in node.attrib: - loc = {} - loc["file"] = node.attrib["__file"] - if hasattr(node, "sourceline"): - loc["line"] = node.sourceline - self.loc = loc - elif hasattr(node, "base") and node.base != None: - loc = {} - loc["file"] = os.path.basename(node.base) - if hasattr(node, "sourceline"): - loc["line"] = node.sourceline - self.loc = loc - - def __getitem__(self, key): - value = super(XmlCollection, self).__getitem__(key) - if type(value) == XmlData or type(value) == XmlCollection: - return value - - return str(value) - - def __iter__(self): - it = super(XmlCollection, self).__iter__() - return XmlDataIterator(it) - -class XmlData(dict): - def __init__(self, node=None): - super(XmlData, self).__init__() - if node is not None: - if hasattr(node, "loc"): - self.loc = node.loc - elif "__file" in node.attrib: - loc = {} - loc["file"] = node.attrib["__file"] - if hasattr(node, "sourceline"): - loc["line"] = node.sourceline - self.loc = loc - elif hasattr(node, "base") and node.base != None: - loc = {} - loc["file"] = os.path.basename(node.base) - if hasattr(node, "sourceline"): - loc["line"] = node.sourceline - self.loc = loc - - def __getitem__(self, key): - value = super(XmlData, self).__getitem__(key) - if type(value) == XmlData or type(value) == XmlCollection: - return value - - return str(value) - - def __iter__(self): - it = super(XmlData, self).__iter__() - return XmlDataIterator(it) - - def iteritems(self): - it = super(XmlData, self).iteritems() - return XmlDataIterator(it) - - def iterkeys(self): - it = super(XmlData, self).iterkeys() - return XmlDataIterator(it) - - def itervalues(self): - it = super(XmlData, self).itervalues() - return XmlDataIterator(it) - -class XmlTemplateString(object): - def __init__(self, param=None, node=None): - if type(param) == str: - self._parts = [param] - elif type(param) == list: - self._parts = param - else: - self._parts = [] - - if node and hasattr(node, "loc"): - self.loc = node.loc - - def __add__(self, other): - if type(other) is str: - self.add_part(other) - elif type(other) is self.__class__: - self._parts += other._parts - else: - raise XmlProcessingError("Cannot concatenate %s and %s" % \ - str(type(self)), str(type(other))) - return self - - def __str__(self): - string = "" - for part in self._parts: - string += str(part) - return string - - def __hash__(self): - return hash(str(self)) - - def __eq__(self, other): - return str(self) == str(other) - - def __ne__(self, other): - return str(self) != str(other) - - def __len__(self): - return len(str(self)) - - def add_part(self, part): - self._parts.append(part) diff --git a/lnst/Common/XmlTemplates.py b/lnst/Common/XmlTemplates.py deleted file mode 100644 index 1d7a9fd..0000000 --- a/lnst/Common/XmlTemplates.py +++ /dev/null @@ -1,425 +0,0 @@ -""" -This module contains code to aid processing templates in XML files/recipes -while they're being parsed. - -Templates are strings enclosed in curly braces {} and can be present -in all text elements of the XML file (this includes tag values or -attribute values). Templates cannot be used as a stubstitution for tag -names, attribute names or any other structural elements of the document. - -There are two supported types of templates: - - * aliases - $alias_name - * functions - function_name(param1, param2) - -Copyright 2012 Red Hat, Inc. -Licensed under the GNU General Public License, version 2 as -published by the Free Software Foundation; see COPYING for details. -""" - -__author__ = """ -rpazdera@redhat.com (Radek Pazdera) -""" - -import re -from lxml import etree -from lnst.Common.XmlProcessing import XmlTemplateString - -class XmlTemplateError(Exception): - pass - -class TemplateFunc(object): - def __init__(self, args, machines): - self._check_args(args) - self._args = args - - self._machines = machines - - def __str__(self): - return self._implementation() - - def _check_args(self, args): - pass - - def _implementation(self): - pass - -class IpFunc(TemplateFunc): - def _check_args(self, args): - if len(args) > 3: - msg = "Function ip() takes at most 3 arguments, %d passed" \ - % len(args) - raise XmlTemplateError(msg) - if len(args) < 2: - msg = "Function ip() must have at least 2 arguments, %d passed" \ - % len(args) - raise XmlTemplateError(msg) - - if len(args) == 3: - try: - int(args[2]) - except ValueError: - msg = "The third argument of ip() function must be an integer" - raise XmlTemplateError(msg) - - def _implementation(self): - m_id = self._args[0] - if_id = self._args[1] - addr = 0 - if len(self._args) == 3: - addr = self._args[2] - - try: - machine = self._machines[m_id] - except KeyError: - msg = "First parameter of function ip() is invalid: " \ - "Machine %s does not exist." % m_id - raise XmlTemplateError(msg) - - try: - iface = machine.get_interface(if_id) - except MachineError: - msg = "Second parameter of function ip() is invalid: "\ - "Interface %s does not exist." % if_id - raise XmlTemplateError(msg) - - try: - return iface.get_address(int(addr)) - except IndexError: - msg = "There is no address with index %s on machine %s, " \ - "interface %s." % (addr, m_id, if_id) - raise XmlTemplateError(msg) - -class DevnameFunc(TemplateFunc): - def _check_args(self, args): - if len(args) != 2: - msg = "Function devname() takes 2 arguments, %d passed." % len(args) - raise XmlTemplateError(msg) - - def _implementation(self): - m_id = self._args[0] - if_id = self._args[1] - - try: - machine = self._machines[m_id] - except KeyError: - msg = "First parameter of function devname() is invalid: " \ - "Machine %s does not exist." % m_id - raise XmlTemplateError(msg) - - try: - iface = machine.get_interface(if_id) - except MachineError: - msg = "Second parameter of function devname() is invalid: "\ - "Interface %s does not exist." % if_id - raise XmlTemplateError(msg) - - try: - return iface.get_devname() - except MachineError: - msg = "Devname not availablefor interface '%s' on machine '%s'." \ - % (m_id, if_id) - raise XmlTemplateError(msg) - -class PrefixFunc(TemplateFunc): - def _check_args(self, args): - if len(args) > 3: - msg = "Function prefix() takes at most 3 arguments, %d passed" \ - % len(args) - raise XmlTemplateError(msg) - if len(args) < 2: - msg = "Function prefix() must have at least 2 arguments, %d " \ - "passed" % len(args) - raise XmlTemplateError(msg) - - if len(args) == 3: - try: - int(args[2]) - except ValueError: - msg = "The third argument of prefix() function must be an " \ - "integer" - raise XmlTemplateError(msg) - - def _implementation(self): - m_id = self._args[0] - if_id = self._args[1] - addr = 0 - if len(self._args) == 3: - addr = self._args[2] - - try: - machine = self._machines[m_id] - except KeyError: - msg = "First parameter of function prefix() is invalid: " \ - "Machine %s does not exist." % m_id - raise XmlTemplateError(msg) - - try: - iface = machine.get_interface(if_id) - except MachineError: - msg = "Second parameter of function prefix() is invalid: "\ - "Interface %s does not exist." % if_id - raise XmlTemplateError(msg) - - try: - return iface.get_prefix(int(addr)) - except IndexError: - msg = "There is no address with index %s on machine %s, " \ - "interface %s." % (addr, m_id, if_id) - raise XmlTemplateError(msg) - except PrefixMissingError: - msg = "Address with the index %s for the interface %s on machine" \ - "%s does not contain any prefix" % (addr, m_id, if_id) - -class HwaddrFunc(TemplateFunc): - def _check_args(self, args): - if len(args) != 2: - msg = "Function hwaddr() takes 2 arguments, %d passed." % len(args) - raise XmlTemplateError(msg) - - def _implementation(self): - m_id = self._args[0] - if_id = self._args[1] - - try: - machine = self._machines[m_id] - except KeyError: - msg = "First parameter of function hwaddr() is invalid: " \ - "Machine %s does not exist." % m_id - raise XmlTemplateError(msg) - - try: - iface = machine.get_interface(if_id) - except MachineError: - msg = "Second parameter of function hwaddr() is invalid: "\ - "Interface %s does not exist." % if_id - raise XmlTemplateError(msg) - - try: - return iface.get_hwaddr() - except MachineError: - msg = "Hwaddr not availablefor interface '%s' on machine '%s'." \ - % (m_id, if_id) - raise XmlTemplateError(msg) - -class XmlTemplates: - """ This class serves as template processor """ - - _alias_re = "{$([a-zA-Z0-9_]+)}" - _func_re = "{([a-zA-Z0-9_]+)(([^()]*))}" - - _func_map = {"ip": IpFunc, "hwaddr": HwaddrFunc, "devname": DevnameFunc, \ - "prefix": PrefixFunc } - - def __init__(self, definitions=None): - if definitions: - self._definitions = [definitions] - else: - self._definitions = [{}] - - self._machines = {} - self._reserved_aliases = [] - - def set_definitions(self, defs): - """ Set alias definitions - - All existing definitions and namespace levels are - destroyed and replaced with new definitions. - """ - del self._definitions - self._definitions = [defs] - - def get_definitions(self): - """ Return definitions dict - - Definitions are returned as a single dictionary of - all currently defined aliases, regardless the internal - division to namespace levels. - """ - defs = {} - for level in self._definitions: - for name, val in level.iteritems(): - defs[name] = val - - return defs - - def set_machines(self, machines): - """ Assign machine information - - XmlTemplates use these information about the machines - to resolve template functions within the recipe. - """ - self._machines = machines - - def set_aliases(self, defined, overriden): - """ Set aliases defined or overriden from CLI """ - - for name, value in defined.iteritems(): - self.define_alias(name, value) - - self._overriden_aliases = overriden - - def define_alias(self, name, value, skip_reserved_check=False): - """ Associate an alias name with some value - - The value can be of an atomic type or an array. The - definition is added to the current namespace level. - """ - - if not name in self._reserved_aliases \ - or skip_reserved_check == True: - self._definitions[-1][name] = value - else: - raise XmlTemplateError("Alias name '%s' is reserved" % name) - - def add_namespace_level(self): - """ Create new namespace level - - This method will create a new level for definitions on - the stack. All aliases, that will be defined after this - call will be dropped as soon as `drop_namespace_level' - is called. - """ - self._definitions.append({}) - - def drop_namespace_level(self): - """ Remove one namespace level - - This method will erease all defined aliases since the - last call of `add_namespace_level' method. All aliases, - that were defined beforehand will be kept. - """ - self._definitions.pop() - - def _find_definition(self, name): - if name in self._overriden_aliases: - return self._overriden_aliases[name] - - for level in reversed(self._definitions): - if name in level: - return level[name] - - err = "Alias '%s' is not defined here" % name - raise XmlTemplateError(err) - - def process_aliases(self, element): - """ Expand aliases within an element and its children - - This method will iterate through the element tree that is - passed and expand aliases in all the text content and - attributes. - """ - if element.text != None: - element.text = self.expand_aliases(element.text) - - if element.tail != None: - element.tail = self.expand_aliases(element.tail) - - for name, value in element.attrib.iteritems(): - element.set(name, self.expand_aliases(value)) - - if element.tag == "define": - for alias in element.getchildren(): - name = alias.attrib["name"].strip() - if "value" in alias.attrib: - value = alias.attrib["value"].strip() - else: - value = etree.tostring(element, method="text").strip() - self.define_alias(name, value) - parent = element.getparent() - parent.remove(element) - return - - self.add_namespace_level() - - for child in element.getchildren(): - self.process_aliases(child) - - self.drop_namespace_level() - - def expand_aliases(self, string): - while True: - alias_match = re.search(self._alias_re, string) - - if alias_match: - template = alias_match.group(0) - result = self._process_alias_template(template) - string = string.replace(template, result) - else: - break - - return string - - def _process_alias_template(self, string): - result = None - - alias_match = re.match(self._alias_re, string) - if alias_match: - alias_name = alias_match.group(1) - result = self._find_definition(alias_name) - - return result - - def expand_functions(self, string, node=None): - """ Process a string and expand it into a XmlTemplateString """ - - parts = self._partition_string(string) - value = XmlTemplateString(node=node) - - for part in parts: - value.add_part(part) - - return value - - def _partition_string(self, string): - """ Process templates in a string - - This method will process and expand all template functions - in a string. - - The function returns an array of string partitions and - unresolved template functions for further processing. - """ - - result = None - - func_match = re.search(self._func_re, string) - if func_match: - prefix = string[0:func_match.start(0)] - suffix = string[func_match.end(0):] - - template = func_match.group(0) - func = self._process_func_template(template) - - return self._partition_string(prefix) + [func] + \ - self._partition_string(suffix) - - return [string] - - def _process_func_template(self, string): - func_match = re.match(self._func_re, string) - if func_match: - func_name = func_match.group(1) - func_args = func_match.group(2) - - if func_args == None: - func_args = [] - else: - func_args = func_args.split(",") - - param_values = [] - for param in func_args: - param = param.strip() - if re.match(self._alias_re, param): - param = self._process_alias_template(param) - param_values.append(param) - - if func_name not in self._func_map: - msg = "Unknown template function '%s'." % func_name - raise XmlTemplateError(msg) - - func = self._func_map[func_name](param_values, self._machines) - return func - else: - msg = "The passed string is not a template function." - raise XmlTemplateError(msg) diff --git a/lnst/Controller/RecipeParser.py b/lnst/Controller/RecipeParser.py index 7833e7f..2e178e8 100644 --- a/lnst/Controller/RecipeParser.py +++ b/lnst/Controller/RecipeParser.py @@ -19,9 +19,10 @@ from lnst.Common.Config import lnst_config from lnst.Common.NetUtils import normalize_hwaddr from lnst.Common.Utils import bool_it from lnst.Common.RecipePath import RecipePath -from lnst.Common.XmlParser import XmlParser -from lnst.Common.XmlProcessing import XmlProcessingError, XmlData, XmlCollection -from lnst.Common.XmlTemplates import XmlTemplates, XmlTemplateError +from lnst.Controller.XmlParser import XmlParser +from lnst.Controller.XmlProcessing import XmlProcessingError, XmlData +from lnst.Controller.XmlProcessing import XmlCollection +from lnst.Controller.XmlTemplates import XmlTemplates, XmlTemplateError
class RecipeError(XmlProcessingError): pass diff --git a/lnst/Controller/SlaveMachineParser.py b/lnst/Controller/SlaveMachineParser.py index 686d570..1ec7ef7 100644 --- a/lnst/Controller/SlaveMachineParser.py +++ b/lnst/Controller/SlaveMachineParser.py @@ -15,8 +15,9 @@ import logging import os import re from lxml import etree -from lnst.Common.XmlParser import XmlParser -from lnst.Common.XmlProcessing import XmlProcessingError, XmlData, XmlCollection +from lnst.Controller.XmlParser import XmlParser +from lnst.Controller.XmlProcessing import XmlProcessingError, XmlData +from lnst.Controller.XmlProcessing import XmlCollection
class SlaveMachineError(XmlProcessingError): pass diff --git a/lnst/Controller/SlavePool.py b/lnst/Controller/SlavePool.py index 4162e16..e7d4557 100644 --- a/lnst/Controller/SlavePool.py +++ b/lnst/Controller/SlavePool.py @@ -21,7 +21,7 @@ from xml.dom import minidom from lnst.Common.Config import lnst_config from lnst.Common.NetUtils import normalize_hwaddr from lnst.Common.NetUtils import test_tcp_connection -from lnst.Common.XmlProcessing import XmlProcessingError, XmlData +from lnst.Controller.XmlProcessing import XmlProcessingError, XmlData from lnst.Controller.Machine import Machine from lnst.Controller.SlaveMachineParser import SlaveMachineParser from lnst.Controller.SlaveMachineParser import SlaveMachineError diff --git a/lnst/Controller/XmlParser.py b/lnst/Controller/XmlParser.py new file mode 100644 index 0000000..32bbf2a --- /dev/null +++ b/lnst/Controller/XmlParser.py @@ -0,0 +1,185 @@ +""" +This module contains the XmlParser and LnstParser classes. + +Copyright 2013 Red Hat, Inc. +Licensed under the GNU General Public License, version 2 as +published by the Free Software Foundation; see COPYING for details. +""" + +__author__ = """ +rpazdera@redhat.com (Radek Pazdera) +""" + +import os +import re +import sys +import logging +import copy +from lxml import etree +from lnst.Common.Config import lnst_config +from lnst.Controller.XmlTemplates import XmlTemplates, XmlTemplateError +from lnst.Controller.XmlProcessing import XmlProcessingError, XmlData + +class XmlParser(object): + XINCLUDE_RE = r"{http://www.w3.org/[0-9]{4}/XInclude}include" + + def __init__(self, schema_file, xml_path): + # locate the schema file + # try git path + dirname = os.path.dirname(sys.argv[0]) + schema_path = os.path.join(dirname, schema_file) + if not os.path.exists(schema_path): + # try configuration + res_dir = lnst_config.get_option("environment", "resource_dir") + schema_path = os.path.join(res_dir, schema_file) + + if not os.path.exists(schema_path): + raise Exception("The recipe schema file was not found. " + \ + "Your LNST installation is corrupt!") + + self._template_proc = XmlTemplates() + + self._path = xml_path + relaxng_doc = etree.parse(schema_path) + self._schema = etree.RelaxNG(relaxng_doc) + + def parse(self): + doc = self._parse(self._path) + self._remove_comments(doc) + + # Due to a weird implementation of XInclude in lxml, the + # XmlParser resolves included documents on it's own. + # + # To be able to tell later on where each tag was located + # in the XML document, we add a '__file' attribute to + # each element of the tree during the parsing. + # + # However, these special attributes are of course not + # valid according to our schemas. To solve this, a copy of + # the tree is made and the '__file' attributes are removed + # before validation. + # + # XXX This is a *EXTREMELY* dirty hack. Ideas/proposals + # for cleaner solutions are more than welcome! + root_tag = self._init_loc(doc.getroot(), self._path) + self._expand_xinclude(root_tag, os.path.dirname(self._path)) + + self._template_proc.process_aliases(root_tag) + + try: + self._validate(doc) + except: + err = self._schema.error_log[0] + loc = {"file": os.path.basename(err.filename), + "line": err.line, "col": err.column} + exc = XmlProcessingError(err.message) + exc.set_loc(loc) + raise exc + + return self._process(root_tag) + + def _parse(self, path): + try: + doc = etree.parse(path) + except etree.LxmlError as err: + # A workaround for cases when lxml (quite strangely) + # sets the filename to <string>. + if err.error_log[0].filename == "<string>": + filename = self._path + else: + filename = err.error_log[0].filename + loc = {"file": os.path.basename(filename), + "line": err.error_log[0].line, + "col": err.error_log[0].column} + exc = XmlProcessingError(err.error_log[0].message) + exc.set_loc(loc) + raise exc + except Exception as err: + loc = {"file": os.path.basename(self._path), + "line": None, + "col": None} + exc = XmlProcessingError(str(err)) + exc.set_loc(loc) + raise exc + + return doc + + def _process(self, root_tag): + pass + + def set_machines(self, machines): + self._template_proc.set_machines(machines) + + def set_aliases(self, defined, overriden): + self._template_proc.set_aliases(defined, overriden) + + def _has_attribute(self, element, attr): + return attr in element.attrib + + def _get_attribute(self, element, attr): + text = element.attrib[attr].strip() + return self._template_proc.expand_functions(text) + + def _get_content(self, element): + text = etree.tostring(element, method="text").strip() + return self._template_proc.expand_functions(text) + + def _expand_xinclude(self, elem, base_url=""): + for e in elem: + if re.match(self.XINCLUDE_RE, str(e.tag)): + href = os.path.join(base_url, e.get("href")) + filename = os.path.basename(href) + + doc = self._parse(href) + self._remove_comments(doc) + node = doc.getroot() + + node = self._init_loc(node, href) + + if e.tail: + node.tail = (node.tail or "") + e.tail + self._expand_xinclude(node, os.path.dirname(href)) + + parent = e.getparent() + if parent is None: + return node + + parent.replace(e, node) + else: + self._expand_xinclude(e, base_url) + return elem + + def _remove_comments(self, doc): + comments = doc.xpath('//comment()') + for c in comments: + p = c.getparent() + if p is not None: + p.remove(c) + + def _init_loc(self, elem, filename): + """ Remove all coment tags from the tree """ + + elem.attrib["__file"] = filename + for e in elem: + self._init_loc(e, os.path.basename(filename)) + + return elem + + def _validate(self, original): + """ + Make a copy of the tree, remove the '__file' attributes + and validate against the appropriate schema. + + Very unfortunate solution. + """ + doc = copy.deepcopy(original) + root = doc.getroot() + + self._prepare_tree_for_validation(root) + self._schema.assertValid(doc) + + def _prepare_tree_for_validation(self, elem): + if "__file" in elem.attrib: + del elem.attrib["__file"] + for e in elem: + self._prepare_tree_for_validation(e) diff --git a/lnst/Controller/XmlProcessing.py b/lnst/Controller/XmlProcessing.py new file mode 100644 index 0000000..771ead5 --- /dev/null +++ b/lnst/Controller/XmlProcessing.py @@ -0,0 +1,205 @@ +""" +This module contains code code for XML parsing and processing. + +Copyright 2012 Red Hat, Inc. +Licensed under the GNU General Public License, version 2 as +published by the Free Software Foundation; see COPYING for details. +""" + +__author__ = """ +rpazdera@redhat.com (Radek Pazdera) +""" + +import os +import logging + +class XmlProcessingError(Exception): + """ Exception thrown on parsing errors """ + + _filename = None + _line = None + _col = None + + def __init__(self, msg, obj=None): + super(XmlProcessingError, self).__init__() + self._msg = msg + + if obj is not None: + if hasattr(obj, "loc"): + self.set_loc(obj.loc) + elif hasattr(obj, "attrib") and "__file" in obj.attrib: + loc = {} + loc["file"] = obj.attrib["__file"] + if hasattr(obj, "sourceline"): + loc["line"] = obj.sourceline + self.set_loc(loc) + elif hasattr(obj, "base") and obj.base != None: + loc = {} + loc["file"] = os.path.basename(obj.base) + if hasattr(obj, "sourceline"): + loc["line"] = obj.sourceline + self.set_loc(loc) + + + def set_loc(self, loc): + self._filename = loc["file"] + self._line = loc["line"] + if "col" in loc: + self._col = loc["col"] + + def __str__(self): + line = "" + col = "" + sep = "" + loc = "" + filename = "<unknown>" + + if self._filename: + filename = self._filename + + if self._line: + line = "%d" % self._line + sep = ":" + + if self._col: + col = "%s%d" % (sep, self._col) + + if self._line or self._col: + loc = "%s%s:" % (line, col) + + return "%s:%s %s" % (filename, loc, self._msg) + +class XmlDataIterator: + def __init__(self, iterator): + self._iterator = iterator + + def __iter__(self): + return self + + def next(self): + n = self._iterator.next() + + # For normal iterators + if type(n) == XmlTemplateString: + return str(n) + + # For iteritems() iterators + if type(n) == tuple and len(n) == 2 and type(n[1]) == XmlTemplateString: + return (n[0], str(n[1])) + + return n + +class XmlCollection(list): + def __init__(self, node=None): + super(XmlCollection, self).__init__() + if node is not None: + if hasattr(node, "loc"): + self.loc = node.loc + elif "__file" in node.attrib: + loc = {} + loc["file"] = node.attrib["__file"] + if hasattr(node, "sourceline"): + loc["line"] = node.sourceline + self.loc = loc + elif hasattr(node, "base") and node.base != None: + loc = {} + loc["file"] = os.path.basename(node.base) + if hasattr(node, "sourceline"): + loc["line"] = node.sourceline + self.loc = loc + + def __getitem__(self, key): + value = super(XmlCollection, self).__getitem__(key) + if type(value) == XmlData or type(value) == XmlCollection: + return value + + return str(value) + + def __iter__(self): + it = super(XmlCollection, self).__iter__() + return XmlDataIterator(it) + +class XmlData(dict): + def __init__(self, node=None): + super(XmlData, self).__init__() + if node is not None: + if hasattr(node, "loc"): + self.loc = node.loc + elif "__file" in node.attrib: + loc = {} + loc["file"] = node.attrib["__file"] + if hasattr(node, "sourceline"): + loc["line"] = node.sourceline + self.loc = loc + elif hasattr(node, "base") and node.base != None: + loc = {} + loc["file"] = os.path.basename(node.base) + if hasattr(node, "sourceline"): + loc["line"] = node.sourceline + self.loc = loc + + def __getitem__(self, key): + value = super(XmlData, self).__getitem__(key) + if type(value) == XmlData or type(value) == XmlCollection: + return value + + return str(value) + + def __iter__(self): + it = super(XmlData, self).__iter__() + return XmlDataIterator(it) + + def iteritems(self): + it = super(XmlData, self).iteritems() + return XmlDataIterator(it) + + def iterkeys(self): + it = super(XmlData, self).iterkeys() + return XmlDataIterator(it) + + def itervalues(self): + it = super(XmlData, self).itervalues() + return XmlDataIterator(it) + +class XmlTemplateString(object): + def __init__(self, param=None, node=None): + if type(param) == str: + self._parts = [param] + elif type(param) == list: + self._parts = param + else: + self._parts = [] + + if node and hasattr(node, "loc"): + self.loc = node.loc + + def __add__(self, other): + if type(other) is str: + self.add_part(other) + elif type(other) is self.__class__: + self._parts += other._parts + else: + raise XmlProcessingError("Cannot concatenate %s and %s" % \ + str(type(self)), str(type(other))) + return self + + def __str__(self): + string = "" + for part in self._parts: + string += str(part) + return string + + def __hash__(self): + return hash(str(self)) + + def __eq__(self, other): + return str(self) == str(other) + + def __ne__(self, other): + return str(self) != str(other) + + def __len__(self): + return len(str(self)) + + def add_part(self, part): + self._parts.append(part) diff --git a/lnst/Controller/XmlTemplates.py b/lnst/Controller/XmlTemplates.py new file mode 100644 index 0000000..b73b30d --- /dev/null +++ b/lnst/Controller/XmlTemplates.py @@ -0,0 +1,425 @@ +""" +This module contains code to aid processing templates in XML files/recipes +while they're being parsed. + +Templates are strings enclosed in curly braces {} and can be present +in all text elements of the XML file (this includes tag values or +attribute values). Templates cannot be used as a stubstitution for tag +names, attribute names or any other structural elements of the document. + +There are two supported types of templates: + + * aliases - $alias_name + * functions - function_name(param1, param2) + +Copyright 2012 Red Hat, Inc. +Licensed under the GNU General Public License, version 2 as +published by the Free Software Foundation; see COPYING for details. +""" + +__author__ = """ +rpazdera@redhat.com (Radek Pazdera) +""" + +import re +from lxml import etree +from lnst.Controller.XmlProcessing import XmlTemplateString + +class XmlTemplateError(Exception): + pass + +class TemplateFunc(object): + def __init__(self, args, machines): + self._check_args(args) + self._args = args + + self._machines = machines + + def __str__(self): + return self._implementation() + + def _check_args(self, args): + pass + + def _implementation(self): + pass + +class IpFunc(TemplateFunc): + def _check_args(self, args): + if len(args) > 3: + msg = "Function ip() takes at most 3 arguments, %d passed" \ + % len(args) + raise XmlTemplateError(msg) + if len(args) < 2: + msg = "Function ip() must have at least 2 arguments, %d passed" \ + % len(args) + raise XmlTemplateError(msg) + + if len(args) == 3: + try: + int(args[2]) + except ValueError: + msg = "The third argument of ip() function must be an integer" + raise XmlTemplateError(msg) + + def _implementation(self): + m_id = self._args[0] + if_id = self._args[1] + addr = 0 + if len(self._args) == 3: + addr = self._args[2] + + try: + machine = self._machines[m_id] + except KeyError: + msg = "First parameter of function ip() is invalid: " \ + "Machine %s does not exist." % m_id + raise XmlTemplateError(msg) + + try: + iface = machine.get_interface(if_id) + except MachineError: + msg = "Second parameter of function ip() is invalid: "\ + "Interface %s does not exist." % if_id + raise XmlTemplateError(msg) + + try: + return iface.get_address(int(addr)) + except IndexError: + msg = "There is no address with index %s on machine %s, " \ + "interface %s." % (addr, m_id, if_id) + raise XmlTemplateError(msg) + +class DevnameFunc(TemplateFunc): + def _check_args(self, args): + if len(args) != 2: + msg = "Function devname() takes 2 arguments, %d passed." % len(args) + raise XmlTemplateError(msg) + + def _implementation(self): + m_id = self._args[0] + if_id = self._args[1] + + try: + machine = self._machines[m_id] + except KeyError: + msg = "First parameter of function devname() is invalid: " \ + "Machine %s does not exist." % m_id + raise XmlTemplateError(msg) + + try: + iface = machine.get_interface(if_id) + except MachineError: + msg = "Second parameter of function devname() is invalid: "\ + "Interface %s does not exist." % if_id + raise XmlTemplateError(msg) + + try: + return iface.get_devname() + except MachineError: + msg = "Devname not availablefor interface '%s' on machine '%s'." \ + % (m_id, if_id) + raise XmlTemplateError(msg) + +class PrefixFunc(TemplateFunc): + def _check_args(self, args): + if len(args) > 3: + msg = "Function prefix() takes at most 3 arguments, %d passed" \ + % len(args) + raise XmlTemplateError(msg) + if len(args) < 2: + msg = "Function prefix() must have at least 2 arguments, %d " \ + "passed" % len(args) + raise XmlTemplateError(msg) + + if len(args) == 3: + try: + int(args[2]) + except ValueError: + msg = "The third argument of prefix() function must be an " \ + "integer" + raise XmlTemplateError(msg) + + def _implementation(self): + m_id = self._args[0] + if_id = self._args[1] + addr = 0 + if len(self._args) == 3: + addr = self._args[2] + + try: + machine = self._machines[m_id] + except KeyError: + msg = "First parameter of function prefix() is invalid: " \ + "Machine %s does not exist." % m_id + raise XmlTemplateError(msg) + + try: + iface = machine.get_interface(if_id) + except MachineError: + msg = "Second parameter of function prefix() is invalid: "\ + "Interface %s does not exist." % if_id + raise XmlTemplateError(msg) + + try: + return iface.get_prefix(int(addr)) + except IndexError: + msg = "There is no address with index %s on machine %s, " \ + "interface %s." % (addr, m_id, if_id) + raise XmlTemplateError(msg) + except PrefixMissingError: + msg = "Address with the index %s for the interface %s on machine" \ + "%s does not contain any prefix" % (addr, m_id, if_id) + +class HwaddrFunc(TemplateFunc): + def _check_args(self, args): + if len(args) != 2: + msg = "Function hwaddr() takes 2 arguments, %d passed." % len(args) + raise XmlTemplateError(msg) + + def _implementation(self): + m_id = self._args[0] + if_id = self._args[1] + + try: + machine = self._machines[m_id] + except KeyError: + msg = "First parameter of function hwaddr() is invalid: " \ + "Machine %s does not exist." % m_id + raise XmlTemplateError(msg) + + try: + iface = machine.get_interface(if_id) + except MachineError: + msg = "Second parameter of function hwaddr() is invalid: "\ + "Interface %s does not exist." % if_id + raise XmlTemplateError(msg) + + try: + return iface.get_hwaddr() + except MachineError: + msg = "Hwaddr not availablefor interface '%s' on machine '%s'." \ + % (m_id, if_id) + raise XmlTemplateError(msg) + +class XmlTemplates: + """ This class serves as template processor """ + + _alias_re = "{$([a-zA-Z0-9_]+)}" + _func_re = "{([a-zA-Z0-9_]+)(([^()]*))}" + + _func_map = {"ip": IpFunc, "hwaddr": HwaddrFunc, "devname": DevnameFunc, \ + "prefix": PrefixFunc } + + def __init__(self, definitions=None): + if definitions: + self._definitions = [definitions] + else: + self._definitions = [{}] + + self._machines = {} + self._reserved_aliases = [] + + def set_definitions(self, defs): + """ Set alias definitions + + All existing definitions and namespace levels are + destroyed and replaced with new definitions. + """ + del self._definitions + self._definitions = [defs] + + def get_definitions(self): + """ Return definitions dict + + Definitions are returned as a single dictionary of + all currently defined aliases, regardless the internal + division to namespace levels. + """ + defs = {} + for level in self._definitions: + for name, val in level.iteritems(): + defs[name] = val + + return defs + + def set_machines(self, machines): + """ Assign machine information + + XmlTemplates use these information about the machines + to resolve template functions within the recipe. + """ + self._machines = machines + + def set_aliases(self, defined, overriden): + """ Set aliases defined or overriden from CLI """ + + for name, value in defined.iteritems(): + self.define_alias(name, value) + + self._overriden_aliases = overriden + + def define_alias(self, name, value, skip_reserved_check=False): + """ Associate an alias name with some value + + The value can be of an atomic type or an array. The + definition is added to the current namespace level. + """ + + if not name in self._reserved_aliases \ + or skip_reserved_check == True: + self._definitions[-1][name] = value + else: + raise XmlTemplateError("Alias name '%s' is reserved" % name) + + def add_namespace_level(self): + """ Create new namespace level + + This method will create a new level for definitions on + the stack. All aliases, that will be defined after this + call will be dropped as soon as `drop_namespace_level' + is called. + """ + self._definitions.append({}) + + def drop_namespace_level(self): + """ Remove one namespace level + + This method will erease all defined aliases since the + last call of `add_namespace_level' method. All aliases, + that were defined beforehand will be kept. + """ + self._definitions.pop() + + def _find_definition(self, name): + if name in self._overriden_aliases: + return self._overriden_aliases[name] + + for level in reversed(self._definitions): + if name in level: + return level[name] + + err = "Alias '%s' is not defined here" % name + raise XmlTemplateError(err) + + def process_aliases(self, element): + """ Expand aliases within an element and its children + + This method will iterate through the element tree that is + passed and expand aliases in all the text content and + attributes. + """ + if element.text != None: + element.text = self.expand_aliases(element.text) + + if element.tail != None: + element.tail = self.expand_aliases(element.tail) + + for name, value in element.attrib.iteritems(): + element.set(name, self.expand_aliases(value)) + + if element.tag == "define": + for alias in element.getchildren(): + name = alias.attrib["name"].strip() + if "value" in alias.attrib: + value = alias.attrib["value"].strip() + else: + value = etree.tostring(element, method="text").strip() + self.define_alias(name, value) + parent = element.getparent() + parent.remove(element) + return + + self.add_namespace_level() + + for child in element.getchildren(): + self.process_aliases(child) + + self.drop_namespace_level() + + def expand_aliases(self, string): + while True: + alias_match = re.search(self._alias_re, string) + + if alias_match: + template = alias_match.group(0) + result = self._process_alias_template(template) + string = string.replace(template, result) + else: + break + + return string + + def _process_alias_template(self, string): + result = None + + alias_match = re.match(self._alias_re, string) + if alias_match: + alias_name = alias_match.group(1) + result = self._find_definition(alias_name) + + return result + + def expand_functions(self, string, node=None): + """ Process a string and expand it into a XmlTemplateString """ + + parts = self._partition_string(string) + value = XmlTemplateString(node=node) + + for part in parts: + value.add_part(part) + + return value + + def _partition_string(self, string): + """ Process templates in a string + + This method will process and expand all template functions + in a string. + + The function returns an array of string partitions and + unresolved template functions for further processing. + """ + + result = None + + func_match = re.search(self._func_re, string) + if func_match: + prefix = string[0:func_match.start(0)] + suffix = string[func_match.end(0):] + + template = func_match.group(0) + func = self._process_func_template(template) + + return self._partition_string(prefix) + [func] + \ + self._partition_string(suffix) + + return [string] + + def _process_func_template(self, string): + func_match = re.match(self._func_re, string) + if func_match: + func_name = func_match.group(1) + func_args = func_match.group(2) + + if func_args == None: + func_args = [] + else: + func_args = func_args.split(",") + + param_values = [] + for param in func_args: + param = param.strip() + if re.match(self._alias_re, param): + param = self._process_alias_template(param) + param_values.append(param) + + if func_name not in self._func_map: + msg = "Unknown template function '%s'." % func_name + raise XmlTemplateError(msg) + + func = self._func_map[func_name](param_values, self._machines) + return func + else: + msg = "The passed string is not a template function." + raise XmlTemplateError(msg)
lnst-developers@lists.fedorahosted.org