From: Ondrej Lichtner olichtne@redhat.com
v2 changes: * fix match description logging by Controller - split format_match_description into lines * fixed match description log by RunSummaryFormatter - split format_match_description by lines * Fixed filtering out Start/Finish logs for the same job in RunSummaryFormatter * added RunSummaryFormatter example to the example recipe: commit: examples/python_recipe.py: print result summary
Ondrej Lichtner (8): add lnst.Controller.RecipeResults Controller: move print_match_description to MachineMapper Recipe: add class RecipeRun lnst.Controller.Controller: use the Recipe instance for test init lnst.Controller.Machine: generate and store JobResult objects lnst.Controller.Job: passed should reflect what was expected add lnst.Controller.RunSummaryFormatter examples/python_recipe.py: print result summary
lnst/Controller/Controller.py | 29 ++++------ lnst/Controller/Job.py | 5 +- lnst/Controller/Machine.py | 13 ++++- lnst/Controller/MachineMapper.py | 13 +++++ lnst/Controller/Recipe.py | 39 +++++++++++++ lnst/Controller/RecipeResults.py | 98 +++++++++++++++++++++++++++++++ lnst/Controller/RunSummaryFormatter.py | 103 +++++++++++++++++++++++++++++++++ recipes/examples/python_recipe.py | 8 +++ 8 files changed, 286 insertions(+), 22 deletions(-) create mode 100644 lnst/Controller/RecipeResults.py create mode 100644 lnst/Controller/RunSummaryFormatter.py
From: Ondrej Lichtner olichtne@redhat.com
This module defines classes for storing Result data related to a test run. Most are generated automatically by LNST during test execution and a tester also has a Recipe interface available to create Result objects for custom entries.
These will later be integrated with the rest of the LNST Controller so that they're created in the right places and accessible as Recipe data after the test run is finished for optional post-processing.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Controller/RecipeResults.py | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 lnst/Controller/RecipeResults.py
diff --git a/lnst/Controller/RecipeResults.py b/lnst/Controller/RecipeResults.py new file mode 100644 index 0000000..b79516d --- /dev/null +++ b/lnst/Controller/RecipeResults.py @@ -0,0 +1,98 @@ +""" +This module defines classes for storing Result data related to a test run. +Most are generated automatically by LNST during test execution and a tester +also has a Recipe interface available to create Result objects for custom +entries. + +Copyright 2018 Red Hat, Inc. +Licensed under the GNU General Public License, version 2 as +published by the Free Software Foundation; see COPYING for details. +""" + +__author__ = """ +olichtne@redhat.com (Ondrej Lichtner) +""" + +import time + +class BaseResult(object): + """Base class for storing result data + + should not be instantiated directly, only defines the interface""" + def __init__(self, success=True): + self._timestamp = time.time() + self._success = success + + @property + def timestamp(self): + return self._timestamp + + @property + def success(self): + return self._success + + @property + def short_desc(self): + return "Short description of result if relevant" + + @property + def data(self): + return None + +class JobResult(BaseResult): + """Base class for storing result data of Jobs + + should not be instantiated directly, just stores the Job instance""" + def __init__(self, job, success): + super(JobResult, self).__init__(success) + + self._job = job + + @property + def job(self): + return self._job + +class JobStartResult(JobResult): + """Generated automatically when a Job is succesfully started on a slave""" + @BaseResult.short_desc.getter + def short_desc(self): + return "Job started: {}".format(str(self.job)) + +class JobFinishResult(JobResult): + """Generated automatically when a Job is finished on a slave + + success depends on the Job passed value and returns the data returned as + a result of the Job.""" + def __init__(self, job): + super(JobFinishResult, self).__init__(job, True) + + @BaseResult.success.getter + def success(self): + return self._job.passed + + @BaseResult.short_desc.getter + def short_desc(self): + return "Job finished: {}".format(str(self.job)) + + @BaseResult.data.getter + def data(self): + return self.job.result + +class Result(BaseResult): + """Class intended to store aribitrary tester supplied data + + Will be created when the tester calls the Recipe interface for adding + results.""" + def __init__(self, success, short_desc="", data=None): + super(Result, self).__init__(success) + + self._short_desc = short_desc + self._data = data + + @BaseResult.short_desc.getter + def short_desc(self): + return self._short_desc + + @BaseResult.data.getter + def data(self): + return self._data
From: Ondrej Lichtner olichtne@redhat.com
Renamed the method to format_match_description and made it return the formatted string describing a match that gets returned by a MachineMapper.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com
--- v2: fix match description logging by Controller - split format_match_description into lines --- --- lnst/Controller/Controller.py | 15 +++------------ lnst/Controller/MachineMapper.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py index 0d30026..9a5d2dd 100644 --- a/lnst/Controller/Controller.py +++ b/lnst/Controller/Controller.py @@ -27,6 +27,7 @@ from lnst.Controller.Config import CtlConfig from lnst.Controller.MessageDispatcher import MessageDispatcher from lnst.Controller.SlavePoolManager import SlavePoolManager from lnst.Controller.MachineMapper import MachineMapper +from lnst.Controller.MachineMapper import format_match_description from lnst.Controller.Host import Hosts, Host from lnst.Controller.Recipe import BaseRecipe
@@ -118,7 +119,8 @@ class Controller(object): expand="match_%d" % i) i += 1
- self._print_match_description(match) + for line in format_match_description(match).split('\n'): + logging.info(line) try: self._map_match(match, req) recipe.test() @@ -218,14 +220,3 @@ class Controller(object): config.load_config(gitcfg)
return config - - def _print_match_description(self, match): - logging.info("Pool match description:") - if match["virtual"]: - logging.info(" Setup is using virtual machines.") - for m_id, m in sorted(match["machines"].iteritems()): - logging.info(" host "%s" uses "%s"" % (m_id, m["target"])) - for if_id, match in m["interfaces"].iteritems(): - pool_id = match["target"] - logging.info(" interface "%s" matched to "%s"" %\ - (if_id, pool_id)) diff --git a/lnst/Controller/MachineMapper.py b/lnst/Controller/MachineMapper.py index 39b2acc..13488eb 100644 --- a/lnst/Controller/MachineMapper.py +++ b/lnst/Controller/MachineMapper.py @@ -16,6 +16,19 @@ from lnst.Controller.Common import ControllerError class MapperError(ControllerError): pass
+def format_match_description(match): + output = [] + output.append("Pool match description:") + if match["virtual"]: + output.append(" Setup is using virtual machines.") + for m_id, m in sorted(match["machines"].iteritems()): + output.append(" host "{}" uses "{}"".format(m_id, m["target"])) + for if_id, match in m["interfaces"].iteritems(): + pool_id = match["target"] + output.append(" interface "{}" matched to "{}"". + format(if_id, pool_id)) + return "\n".join(output) + class MachineMapper(object): """Implements a matching algorithm that maps requirements to available hosts
From: Ondrej Lichtner olichtne@redhat.com
Instances of the RecipeRun class represent a single test run of a recipe. They store information about Results (executed Jobs or tester defined) and about the matchine match that was used.
A recipe instance stores a list of these RecipeRun objects and all of them can be accessed after testing for optional post-processing.
The Recipe class now also has an interface method for the tester which can be used to add Result objects to the current run, this is intended to be used from within the tester defined "test" method.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Controller/Controller.py | 3 ++- lnst/Controller/Recipe.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py index 9a5d2dd..d044870 100644 --- a/lnst/Controller/Controller.py +++ b/lnst/Controller/Controller.py @@ -29,7 +29,7 @@ from lnst.Controller.SlavePoolManager import SlavePoolManager from lnst.Controller.MachineMapper import MachineMapper from lnst.Controller.MachineMapper import format_match_description from lnst.Controller.Host import Hosts, Host -from lnst.Controller.Recipe import BaseRecipe +from lnst.Controller.Recipe import BaseRecipe, RecipeRun
class Controller(object): """The LNST Controller class @@ -123,6 +123,7 @@ class Controller(object): logging.info(line) try: self._map_match(match, req) + recipe._init_run(RecipeRun(match)) recipe.test() except Exception as exc: logging.error("Recipe execution terminated by unexpected exception") diff --git a/lnst/Controller/Recipe.py b/lnst/Controller/Recipe.py index 9380a35..a10f081 100644 --- a/lnst/Controller/Recipe.py +++ b/lnst/Controller/Recipe.py @@ -14,6 +14,7 @@ import copy from lnst.Common.Parameters import Parameters, Param from lnst.Controller.Requirements import _Requirements, HostReq from lnst.Controller.Common import ControllerError +from lnst.Controller.RecipeResults import BaseResult, Result
class RecipeError(ControllerError): """Exception thrown by the BaseRecipe class""" @@ -81,6 +82,7 @@ class BaseRecipe(object): and checked if mandatory Parameters have values. """ self._ctl = None + self.runs = [] self.req = _Requirements() self.params = Parameters() for attr in dir(self): @@ -121,3 +123,40 @@ class BaseRecipe(object): def test(self): """Method to be implemented by the Tester""" raise NotImplementedError("Method test must be defined by a child class.") + + def _init_run(self, run): + self.runs.append(run) + + @property + def current_run(self): + if len(self.runs) > 0: + return self.runs[-1] + else: + return None + + def add_result(self, success, description="", data=None): + self.current_run.add_result(Result(success, description, data)) + +class RecipeRun(object): + def __init__(self, match, desc=None): + self._match = match + self._desc = desc + self._results = [] + + def add_result(self, result): + if not isinstance(result, BaseResult): + raise RecipeError("result must be a BaseActionResult instance.") + + self._results.append(result) + + @property + def match(self): + return self._match + + @property + def description(self): + return self._desc + + @property + def results(self): + return self._results
From: Ondrej Lichtner olichtne@redhat.com
When a recipe is succesfully matched and the Controller starts setting up the Machine objects we should pass the Recipe instance. This is important since the Machine object needs to be able to generate JobResult objects.
Another minor improvement is that a correct recipe name will be used (the name of the Recipe class) instead of the filename of the currently running executable which was incorrectly used until now.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Controller/Controller.py | 11 +++++------ lnst/Controller/Machine.py | 6 +++++- 2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/lnst/Controller/Controller.py b/lnst/Controller/Controller.py index d044870..f306319 100644 --- a/lnst/Controller/Controller.py +++ b/lnst/Controller/Controller.py @@ -122,7 +122,7 @@ class Controller(object): for line in format_match_description(match).split('\n'): logging.info(line) try: - self._map_match(match, req) + self._map_match(match, req, recipe) recipe._init_run(RecipeRun(match)) recipe.test() except Exception as exc: @@ -143,7 +143,7 @@ class Controller(object):
self._msg_dispatcher.wait_for_condition(condition)
- def _map_match(self, match, requested): + def _map_match(self, match, requested, recipe): self._machines = {} self._hosts = Hosts() pool = self._pools.get_machine_pool(match["pool_name"]) @@ -154,7 +154,7 @@ class Controller(object): host = getattr(self._hosts, m_id)
machine.set_id(m_id) - self._prepare_machine(machine) + self._prepare_machine(machine, recipe)
for if_id, i in m["interfaces"].items(): host._map_device(if_id, i) @@ -168,13 +168,12 @@ class Controller(object): setattr(host, name, new_virt_dev) new_virt_dev._enable()
- def _prepare_machine(self, machine): + def _prepare_machine(self, machine, recipe): self._log_ctl.add_slave(machine.get_id()) machine.set_mac_pool(self._mac_pool) machine.set_network_bridges(self._network_bridges)
- recipe_name = os.path.basename(sys.argv[0]) - machine.set_recipe(recipe_name) + machine.set_recipe(recipe)
def _cleanup_slaves(self): if self._machines == None: diff --git a/lnst/Controller/Machine.py b/lnst/Controller/Machine.py index cd14f78..b429721 100644 --- a/lnst/Controller/Machine.py +++ b/lnst/Controller/Machine.py @@ -70,6 +70,8 @@ class Machine(object): self._port = ctl_config.get_option('environment', 'rpcport')
self._msg_dispatcher = msg_dispatcher + + self._recipe = None self._mac_pool = None
self._interfaces = [] @@ -231,12 +233,14 @@ class Machine(object):
self._slave_desc = slave_desc
- def set_recipe(self, recipe_name): + def set_recipe(self, recipe): """ Reserves the machine for the specified recipe
Also sends Device classes from the controller and initializes the InterfaceManager on the Slave and builds the device database. """ + self._recipe = recipe + recipe_name = recipe.__class__.__name__ self.rpc_call("set_recipe", recipe_name) self._send_device_classes() self.rpc_call("init_if_manager")
From: Ondrej Lichtner olichtne@redhat.com
Generates a JobStartResult object when the "run_job" rpc call returns and stores it in the current RecipeRun.
Generates a JobFinishResult object when a job_finished message is received and stores it in the current RecipeRun.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Controller/Machine.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/lnst/Controller/Machine.py b/lnst/Controller/Machine.py index b429721..09e7540 100644 --- a/lnst/Controller/Machine.py +++ b/lnst/Controller/Machine.py @@ -21,6 +21,7 @@ from lnst.Common.TestModule import BaseTestModule from lnst.Common.Version import lnst_version from lnst.Controller.Common import ControllerError from lnst.Controller.CtlSecSocket import CtlSecSocket +from lnst.Controller.RecipeResults import JobStartResult, JobFinishResult from lnst.Devices import device_classes from lnst.Devices.Device import Device from lnst.Devices.RemoteDevice import RemoteDevice @@ -365,7 +366,10 @@ class Machine(object): if job._desc is not None: logging.info("Job description: %s" % job._desc)
- return self.rpc_call("run_job", job._to_dict(), netns=job.netns) + res = self.rpc_call("run_job", job._to_dict(), netns=job.netns) + + self._recipe.current_run.add_result(JobStartResult(job, res)) + return res
def wait_for_job(self, job, timeout): res = True @@ -426,6 +430,7 @@ class Machine(object): job_id = msg["job_id"] job = self._jobs[job_id] job._res = msg["result"] + self._recipe.current_run.add_result(JobFinishResult(job))
def kill(self, job, signal): if job.id not in self._jobs:
From: Ondrej Lichtner olichtne@redhat.com
In case the Job was expected to fail, the passed attribute should reflect this.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Controller/Job.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/lnst/Controller/Job.py b/lnst/Controller/Job.py index 96cf045..1026938 100644 --- a/lnst/Controller/Job.py +++ b/lnst/Controller/Job.py @@ -88,10 +88,13 @@ class Job(object): def passed(self): """Indicates whether or not the Job passed
+ The return value is True or False based on if the Job was expected to + pass or fail. + Type: Boolean """ try: - return self._res["passed"] + return self._res["passed"] == self._expect except: return False
From: Ondrej Lichtner olichtne@redhat.com
Instances of this class can process RecipeRun objects and output a formatted run summary string.
Signed-off-by: Ondrej Lichtner olichtne@redhat.com
--- v2: * fixed match description log - split format_match_description by lines * Fixed filtering out Start/Finish logs for the same job --- lnst/Controller/RunSummaryFormatter.py | 103 +++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 lnst/Controller/RunSummaryFormatter.py
diff --git a/lnst/Controller/RunSummaryFormatter.py b/lnst/Controller/RunSummaryFormatter.py new file mode 100644 index 0000000..d035717 --- /dev/null +++ b/lnst/Controller/RunSummaryFormatter.py @@ -0,0 +1,103 @@ +""" +Defines the RunSummaryFormatter class which is can be used to process a +RecipeRun object to return a formatted run summary string. + +Copyright 2018 Red Hat, Inc. +Licensed under the GNU General Public License, version 2 as +published by the Free Software Foundation; see COPYING for details. +""" + +__author__ = """ +olichtne@redhat.com (Ondrej Lichtner) +""" + +from lnst.Common.Colours import decorate_with_preset +from lnst.Controller.Common import ControllerError +from lnst.Controller.MachineMapper import format_match_description +from lnst.Controller.Recipe import BaseRecipe, RecipeRun +from lnst.Controller.RecipeResults import BaseResult, JobResult, Result +from lnst.Controller.RecipeResults import JobStartResult, JobFinishResult + +class RunFormatterException(ControllerError): + pass + +class RunSummaryFormatter(object): + def __init__(self): + #TODO changeable format? + self._format = "" + + def _format_success(self, success): + if success: + return decorate_with_preset("PASS", "pass") + else: + return decorate_with_preset("FAIL", "fail") + + def _format_source(self, res): + if isinstance(res, JobResult): + return "Host {} job {}".format(res.job.host.hostid, res.job.id) + elif isinstance(res, Result): + return "TestResult:" + else: + return "" + + def _format_data(self, data, prefix=" ", level=1): + output = [] + if data is not None: + if isinstance(data, dict): + for key, value in data.items(): + output.append("{pref}{key}:".format(pref=level*prefix, + key=key)) + nest_res = self._format_data(value, level=level+1) + if len(nest_res) == 1: + output[-1] += " " + nest_res[0].lstrip() + else: + output.extend(nest_res) + elif isinstance(data, list): + for i, v in enumerate(data): + output.append("{pref}item {i}:".format(pref=level*prefix, + i=i)) + output.extend(self._format_data(v, level=level+1)) + else: + for line in str(data).split('\n'): + output.append("{pref}{val}".format(pref=level*prefix, + val=line)) + return output + + def format_run(self, run): + if not isinstance(run, RecipeRun): + raise RunFormatterException("run must be a RecipeRun instance.") + + output_lines = [] + output_lines.append("RUN SUMMARY") + output_lines.append("Description:") + if run.description: + output_lines.extend(str(run.description).split("\n")) + + output_lines.extend(format_match_description(run.match).split('\n')) + + overall_result = True + for i, res in enumerate(run.results): + overall_result = overall_result and res.success + + try: + next_res = run.results[i+1] + if (isinstance(res, JobStartResult) and + isinstance(next_res, JobFinishResult) and + res.job.host == next_res.job.host and + res.job.id == next_res.job.id and + res.success): + continue + except IndexError: + pass + + output_lines.append("{res}\t{src}\t{desc}".format( + res = self._format_success(res.success), + src = self._format_source(res), + desc = res.short_desc)) + + output_lines.extend(self._format_data(res.data)) + + output_lines.append("Overall result of this Run: {}". + format(self._format_success(overall_result))) + + return "\n".join(output_lines)
From: Ondrej Lichtner olichtne@redhat.com
Adding the Result summary print to the end of the example recipe
Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- recipes/examples/python_recipe.py | 8 ++++++++ 1 file changed, 8 insertions(+)
diff --git a/recipes/examples/python_recipe.py b/recipes/examples/python_recipe.py index 43fa95d..00dd329 100755 --- a/recipes/examples/python_recipe.py +++ b/recipes/examples/python_recipe.py @@ -4,6 +4,7 @@ This is an example python recipe that can be run as an executable script. Performs a simple ping between two hosts. """
+import logging from lnst.Common.Parameters import IpParam from lnst.Common.IpAddress import ipaddress from lnst.Controller import Controller @@ -24,6 +25,9 @@ from lnst.Tests.Netperf import Netperf, Netserver
import signal
+ +from lnst.Controller.RunSummaryFormatter import RunSummaryFormatter + class MyRecipe(BaseRecipe): m1 = HostReq() m1.eth0 = DeviceReq(label="net1") @@ -118,3 +122,7 @@ ctl = Controller(debug=1)
r = MyRecipe() ctl.run(r, allow_virt=True) + +summary_fmt = RunSummaryFormatter() +for run in r.runs: + logging.info(summary_fmt.format_run(run))
On Mon, Mar 26, 2018 at 11:16:49AM +0200, olichtne@redhat.com wrote:
From: Ondrej Lichtner olichtne@redhat.com
v2 changes: * fix match description logging by Controller - split format_match_description into lines * fixed match description log by RunSummaryFormatter - split format_match_description by lines * Fixed filtering out Start/Finish logs for the same job in RunSummaryFormatter * added RunSummaryFormatter example to the example recipe: commit: examples/python_recipe.py: print result summary
Ondrej Lichtner (8): add lnst.Controller.RecipeResults Controller: move print_match_description to MachineMapper Recipe: add class RecipeRun lnst.Controller.Controller: use the Recipe instance for test init lnst.Controller.Machine: generate and store JobResult objects lnst.Controller.Job: passed should reflect what was expected add lnst.Controller.RunSummaryFormatter examples/python_recipe.py: print result summary
lnst/Controller/Controller.py | 29 ++++------ lnst/Controller/Job.py | 5 +- lnst/Controller/Machine.py | 13 ++++- lnst/Controller/MachineMapper.py | 13 +++++ lnst/Controller/Recipe.py | 39 +++++++++++++ lnst/Controller/RecipeResults.py | 98 +++++++++++++++++++++++++++++++ lnst/Controller/RunSummaryFormatter.py | 103 +++++++++++++++++++++++++++++++++ recipes/examples/python_recipe.py | 8 +++ 8 files changed, 286 insertions(+), 22 deletions(-) create mode 100644 lnst/Controller/RecipeResults.py create mode 100644 lnst/Controller/RunSummaryFormatter.py
-- 2.16.1
pushed
-Ondrej
lnst-developers@lists.fedorahosted.org