From: Ondrej Lichtner olichtne@redhat.com

Can be useful for inspecting the measurement objects.

Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/RecipeCommon/Perf/Measurements/BaseFlowMeasurement.py | 4 ++++ lnst/RecipeCommon/Perf/Measurements/IperfFlowMeasurement.py | 4 ++++ lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py | 4 ++++ 3 files changed, 12 insertions(+)

diff --git a/lnst/RecipeCommon/Perf/Measurements/BaseFlowMeasurement.py b/lnst/RecipeCommon/Perf/Measurements/BaseFlowMeasurement.py index 856eeba..f76a25a 100644 --- a/lnst/RecipeCommon/Perf/Measurements/BaseFlowMeasurement.py +++ b/lnst/RecipeCommon/Perf/Measurements/BaseFlowMeasurement.py @@ -195,6 +195,10 @@ def add_results(self, results): raise MeasurementError("Adding incorrect results.")

class BaseFlowMeasurement(BaseMeasurement): + @property + def flows(self): + raise NotImplementedError() + @classmethod def report_results(cls, recipe, results): for flow_results in results: diff --git a/lnst/RecipeCommon/Perf/Measurements/IperfFlowMeasurement.py b/lnst/RecipeCommon/Perf/Measurements/IperfFlowMeasurement.py index c193987..4c110f6 100644 --- a/lnst/RecipeCommon/Perf/Measurements/IperfFlowMeasurement.py +++ b/lnst/RecipeCommon/Perf/Measurements/IperfFlowMeasurement.py @@ -27,6 +27,10 @@ def __init__(self, *args):

self._hosts_versions = {}

+ @property + def flows(self): + return self._conf + @property def version(self): if not self._hosts_versions: diff --git a/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py b/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py index c12e922..e047420 100644 --- a/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py +++ b/lnst/RecipeCommon/Perf/Measurements/TRexFlowMeasurement.py @@ -37,6 +37,10 @@ def __init__(self, flows, trex_dir, server_cpu_cores, recipe_conf=None):

self._hosts_versions = {}

+ @property + def flows(self): + return self._flows + @property def version(self): if not self._hosts_versions:

From: Ondrej Lichtner olichtne@redhat.com

Splitting off the perf measurement generation methods into a separate smaller package.

This should make it easier to skip and define more custom behaviours for the various groups of recipes that we have. To provide more context - many of our recipes are baremetal and share the same characteristics wrt. flow generation, however for ovs+dpdk recipes or for virtual recipes, or for ipsec based recipes we need different groups of behaviours. These still share the same basic interface so some form of class inheritance is valid.

These will replace the BaseEnrt generation mechanism.

For now this commit adds 3 main generator types: * IperfMeasurementGenerator that generates iperf measurements using a flow generation algorithm so that multiple combinations of ip_protocol*transport_protocol flows are created for each provided flow endpoint pair * FlowEndpointsStatCPUMeasurementGenerator that generates a cpu measurement based on the endpoints of any flow measurements generated before this * HypervisorsStatCPUMeasurementGenerator that generates a cpu measurement based on a "hypervisors" property that should return the list of hypervisor hosts in the recipe, this is intended to be used with recipes that deal with virtual machines

Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- .../BaseMeasurementGenerator.py | 3 + ...lowEndpointsStatCPUMeasurementGenerator.py | 26 ++++ .../HypervisorsStatCPUMeasurementGenerator.py | 19 +++ .../IperfMeasurementGenerator.py | 146 ++++++++++++++++++ .../ENRT/MeasurementGenerators/__init__.py | 0 5 files changed, 194 insertions(+) create mode 100644 lnst/Recipes/ENRT/MeasurementGenerators/BaseMeasurementGenerator.py create mode 100644 lnst/Recipes/ENRT/MeasurementGenerators/FlowEndpointsStatCPUMeasurementGenerator.py create mode 100644 lnst/Recipes/ENRT/MeasurementGenerators/HypervisorsStatCPUMeasurementGenerator.py create mode 100644 lnst/Recipes/ENRT/MeasurementGenerators/IperfMeasurementGenerator.py create mode 100644 lnst/Recipes/ENRT/MeasurementGenerators/__init__.py

diff --git a/lnst/Recipes/ENRT/MeasurementGenerators/BaseMeasurementGenerator.py b/lnst/Recipes/ENRT/MeasurementGenerators/BaseMeasurementGenerator.py new file mode 100644 index 0000000..c13bfb7 --- /dev/null +++ b/lnst/Recipes/ENRT/MeasurementGenerators/BaseMeasurementGenerator.py @@ -0,0 +1,3 @@ +class BaseMeasurementGenerator(object): + def generate_perf_measurements_combinations(self, config): + return [] diff --git a/lnst/Recipes/ENRT/MeasurementGenerators/FlowEndpointsStatCPUMeasurementGenerator.py b/lnst/Recipes/ENRT/MeasurementGenerators/FlowEndpointsStatCPUMeasurementGenerator.py new file mode 100644 index 0000000..7cc41bf --- /dev/null +++ b/lnst/Recipes/ENRT/MeasurementGenerators/FlowEndpointsStatCPUMeasurementGenerator.py @@ -0,0 +1,26 @@ +from lnst.Common.Parameters import Param + +from lnst.RecipeCommon.Perf.Measurements.BaseFlowMeasurement import BaseFlowMeasurement +from lnst.RecipeCommon.Perf.Measurements.StatCPUMeasurement import StatCPUMeasurement + +from lnst.Recipes.ENRT.MeasurementGenerators.BaseMeasurementGenerator import BaseMeasurementGenerator + +class FlowEndpointsStatCPUMeasurementGenerator(BaseMeasurementGenerator): + cpu_perf_tool = Param(default=StatCPUMeasurement) + + def generate_perf_measurements_combinations(self, config): + combinations = super().generate_perf_measurements_combinations(config) + for combination in combinations: + cpu_measurement_hosts = self.extract_endpoints(config, combination) + yield [ + self.params.cpu_perf_tool(cpu_measurement_hosts) + ] + combination + + def extract_endpoints(self, config, measurements): + endpoints = set() + for measurement in measurements: + if isinstance(measurement, BaseFlowMeasurement): + for flow in measurement.flows: + endpoints.add(flow.generator) + endpoints.add(flow.receiver) + return endpoints diff --git a/lnst/Recipes/ENRT/MeasurementGenerators/HypervisorsStatCPUMeasurementGenerator.py b/lnst/Recipes/ENRT/MeasurementGenerators/HypervisorsStatCPUMeasurementGenerator.py new file mode 100644 index 0000000..9029637 --- /dev/null +++ b/lnst/Recipes/ENRT/MeasurementGenerators/HypervisorsStatCPUMeasurementGenerator.py @@ -0,0 +1,19 @@ +from lnst.Common.Parameters import Param + +from lnst.RecipeCommon.Perf.Measurements.StatCPUMeasurement import StatCPUMeasurement + +from lnst.Recipes.ENRT.MeasurementGenerators.BaseMeasurementGenerator import BaseMeasurementGenerator + +class HypervisorsStatCPUMeasurementGenerator(BaseMeasurementGenerator): + cpu_perf_tool = Param(default=StatCPUMeasurement) + + def generate_perf_measurements_combinations(self, config): + combinations = super().generate_perf_measurements_combinations(config) + for combination in combinations: + yield [ + self.params.cpu_perf_tool(self.hypervisor_hosts) + ] + combination + + @property + def hypervisor_hosts(self): + return set() diff --git a/lnst/Recipes/ENRT/MeasurementGenerators/IperfMeasurementGenerator.py b/lnst/Recipes/ENRT/MeasurementGenerators/IperfMeasurementGenerator.py new file mode 100644 index 0000000..4ccf922 --- /dev/null +++ b/lnst/Recipes/ENRT/MeasurementGenerators/IperfMeasurementGenerator.py @@ -0,0 +1,146 @@ +from lnst.Common.Parameters import ( + Param, + IntParam, + ListParam, +) +from lnst.Common.IpAddress import AF_INET, AF_INET6 + +from lnst.RecipeCommon.Perf.Measurements import Flow as PerfFlow +from lnst.RecipeCommon.Perf.Measurements import IperfFlowMeasurement + +from lnst.Recipes.ENRT.MeasurementGenerators.BaseMeasurementGenerator import BaseMeasurementGenerator + +class IperfMeasurementGenerator(BaseMeasurementGenerator): + """ + :param perf_tests: + Parameter used by the :any:`generate_flow_combinations` generator. + Tells the generator what types of network flow measurements to generate + perf test configurations for. + :type perf_tests: Tuple[str] (default ("tcp_stream", "udp_stream", + "sctp_stream")) + + :param perf_tool_cpu: + Parameter used by the :any:`generate_flow_combinations` generator. To + indicate that the flow measurement should be pinned to a specific CPU + core. + :type perf_tool_cpu: :any:`IntParam` (optional parameter) + + :param perf_duration: + Parameter used by the :any:`generate_perf_configurations` generator. To + specify the duration of the performance measurements, in seconds. + :type perf_duration: :any:`IntParam` (default 60) + + :param perf_iterations: + Parameter used by the :any:`generate_perf_configurations` generator. To + specify how many times should each performance measurement be repeated + to generate cumulative results which can be statistically analyzed. + :type perf_iterations: :any:`IntParam` (default 5) + + :param perf_parallel_streams: + Parameter used by the :any:`generate_flow_combinations` generator. To + specify how many parallel streams of the same network flow should be + measured at the same time. + :type perf_parallel_streams: :any:`IntParam` (default 1) + + :param perf_msg_sizes: + Parameter used by the :any:`generate_flow_combinations` generator. To + specify what different message sizes (in bytes) used generated for the + network flow should be tested - each message size resulting in a + separate performance measurement. + :type perf_msg_sizes: List[Int] (default [123]) + """ + + # common perf test params + perf_tests = Param(default=("tcp_stream", "udp_stream", "sctp_stream")) + perf_tool_cpu = IntParam(mandatory=False) + perf_duration = IntParam(default=60) + perf_iterations = IntParam(default=5) + perf_parallel_streams = IntParam(default=1) + perf_msg_sizes = ListParam(default=[123]) + + net_perf_tool = Param(default=IperfFlowMeasurement) + + def generate_perf_measurements_combinations(self, config): + combinations = super().generate_perf_measurements_combinations(config) + for flow_combination in self.generate_flow_combinations(config): + combinations.append([self.params.net_perf_tool(flow_combination)]) + return combinations + + def generate_flow_combinations(self, config): + """Base flow combination generator + + The generator loops over all endpoint pairs to test performance between + (generated by the :any:`generate_perf_endpoints` method) then over all + the selected :any:`ip_versions` and uses the first IP address fitting + these criteria. Then the generator loops over the selected performance + tests as selected via :any:`perf_tests`, then message sizes from + :any:`msg_sizes`. + + :return: list of Flow combinations to measure in parallel + :rtype: List[:any:`PerfFlow`] + """ + for client_nic, server_nic in self.generate_perf_endpoints(config): + for ipv in self.params.ip_versions: + ip_filter = {} + if ipv == "ipv4": + ip_filter.update(family=AF_INET) + elif ipv == "ipv6": + ip_filter.update(family=AF_INET6) + ip_filter.update(is_link_local=False) + + client_bind = client_nic.ips_filter(**ip_filter)[0] + server_bind = server_nic.ips_filter(**ip_filter)[0] + + for perf_test in self.params.perf_tests: + for size in self.params.perf_msg_sizes: + yield [ + self._create_perf_flow( + perf_test, + client_nic, + client_bind, + server_nic, + server_bind, + size, + ) + ] + + def generate_perf_endpoints(self, config): + """Generator for perf endpoints + + To be overriden by a derived class. + + :return: list of device pairs + :rtype: List[Tuple[:any:`Device`, :any:`Device`]] + """ + return [] + + def _create_perf_flow( + self, + perf_test, + client_nic, + client_bind, + server_nic, + server_bind, + msg_size, + ) -> PerfFlow: + """ + Wrapper to create a PerfFlow. Mixins that want to change this behavior (for example, to reverse the direction) + can override this method as an alternative to overriding :any:`generate_flow_combinations` + """ + return PerfFlow( + type=perf_test, + generator=client_nic.netns, + generator_bind=client_bind, + generator_nic=client_nic, + receiver=server_nic.netns, + receiver_bind=server_bind, + receiver_nic=server_nic, + msg_size=msg_size, + duration=self.params.perf_duration, + parallel_streams=self.params.perf_parallel_streams, + cpupin=( + self.params.perf_tool_cpu + if "perf_tool_cpu" in self.params + else None + ), + ) diff --git a/lnst/Recipes/ENRT/MeasurementGenerators/__init__.py b/lnst/Recipes/ENRT/MeasurementGenerators/__init__.py new file mode 100644 index 0000000..e69de29

From: Ondrej Lichtner olichtne@redhat.com

The generation of combinations of perf measurements is being split off from the BaseEnrtRecipe class to make it easier to override in derived classes.

The functionality is now instead available in the Recipes.ENRT.MeasurementGenerators package.

We keep the evaluator registration in this class for now as we haven't found a better place for the evaluator properties yet. However we adjust it so that it doesn't access the "*_tool" parameters anymore and instead adds evaluators from "*_evaluators" properties based on the base types of measurements.

Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Recipes/ENRT/BaseEnrtRecipe.py | 165 ++++------------------------ 1 file changed, 23 insertions(+), 142 deletions(-)

diff --git a/lnst/Recipes/ENRT/BaseEnrtRecipe.py b/lnst/Recipes/ENRT/BaseEnrtRecipe.py index 180f922..0b881d9 100644 --- a/lnst/Recipes/ENRT/BaseEnrtRecipe.py +++ b/lnst/Recipes/ENRT/BaseEnrtRecipe.py @@ -18,13 +18,15 @@ DisableTurboboostMixin) from lnst.Recipes.ENRT.ConfigMixins.DisableIdleStatesMixin import ( DisableIdleStatesMixin) +from lnst.Recipes.ENRT.MeasurementGenerators.BaseMeasurementGenerator import ( + BaseMeasurementGenerator, +)

from lnst.RecipeCommon.Ping.Recipe import PingTestAndEvaluate, PingConf from lnst.RecipeCommon.Perf.Recipe import Recipe as PerfRecipe from lnst.RecipeCommon.Perf.Recipe import RecipeConf as PerfRecipeConf -from lnst.RecipeCommon.Perf.Measurements import Flow as PerfFlow -from lnst.RecipeCommon.Perf.Measurements import IperfFlowMeasurement -from lnst.RecipeCommon.Perf.Measurements import StatCPUMeasurement +from lnst.RecipeCommon.Perf.Measurements.BaseCPUMeasurement import BaseCPUMeasurement +from lnst.RecipeCommon.Perf.Measurements.BaseFlowMeasurement import BaseFlowMeasurement from lnst.RecipeCommon.Perf.Evaluators import NonzeroFlowEvaluator from lnst.RecipeCommon.Ping.Evaluators import RatePingEvaluator

@@ -40,7 +42,10 @@ class EnrtConfiguration(object): class BaseEnrtRecipe(CommonPerfTestTweakMixin, DisableTurboboostMixin, DisableIdleStatesMixin, BaseSubConfigMixin, - PingTestAndEvaluate, PerfRecipe): + BaseMeasurementGenerator, + PingTestAndEvaluate, + PerfRecipe, +): """Base Recipe class for the ENRT recipe package

This class defines the shared *test* method defining the common test @@ -114,43 +119,6 @@ class BaseEnrtRecipe(CommonPerfTestTweakMixin, test. :type ping_psize: :any:`IntParam` (default None)

- :param perf_tests: - Parameter used by the :any:`generate_flow_combinations` generator. - Tells the generator what types of network flow measurements to generate - perf test configurations for. - :type perf_tests: Tuple[str] (default ("tcp_stream", "udp_stream", - "sctp_stream")) - - :param perf_tool_cpu: - Parameter used by the :any:`generate_flow_combinations` generator. To - indicate that the flow measurement should be pinned to a specific CPU - core. - :type perf_tool_cpu: :any:`IntParam` (optional parameter) - - :param perf_duration: - Parameter used by the :any:`generate_perf_configurations` generator. To - specify the duration of the performance measurements, in seconds. - :type perf_duration: :any:`IntParam` (default 60) - - :param perf_iterations: - Parameter used by the :any:`generate_perf_configurations` generator. To - specify how many times should each performance measurement be repeated - to generate cumulative results which can be statistically analyzed. - :type perf_iterations: :any:`IntParam` (default 5) - - :param perf_parallel_streams: - Parameter used by the :any:`generate_flow_combinations` generator. To - specify how many parallel streams of the same network flow should be - measured at the same time. - :type perf_parallel_streams: :any:`IntParam` (default 1) - - :param perf_msg_sizes: - Parameter used by the :any:`generate_flow_combinations` generator. To - specify what different message sizes (in bytes) used generated for the - network flow should be tested - each message size resulting in a - separate performance measurement. - :type perf_msg_sizes: List[Int] (default [123]) - :param net_perf_tool: Parameter used by the :any:`generate_perf_configurations` generator to create a PerfRecipeConf object. @@ -179,17 +147,6 @@ class BaseEnrtRecipe(CommonPerfTestTweakMixin, ping_interval = FloatParam(default=0.2) ping_psize = IntParam(default=56)

- #common perf test params - perf_tests = Param(default=("tcp_stream", "udp_stream", "sctp_stream")) - perf_tool_cpu = IntParam(mandatory=False) - perf_duration = IntParam(default=60) - perf_iterations = IntParam(default=5) - perf_parallel_streams = IntParam(default=1) - perf_msg_sizes = ListParam(default=[123]) - - net_perf_tool = Param(default=IperfFlowMeasurement) - cpu_perf_tool = Param(default=StatCPUMeasurement) - def test(self): """Main test loop shared by all the Enrt recipes

@@ -441,103 +398,27 @@ def generate_perf_configurations(self, config): :return: list of Perf test configurations :rtype: List[:any:`PerfRecipeConf`] """ - for flows in self.generate_flow_combinations(config): - perf_recipe_conf=dict( - recipe_config=config, - flows=flows, - ) - - flows_measurement = self.params.net_perf_tool( - flows, - perf_recipe_conf - ) - - cpu_measurement_hosts = set() - for flow in flows: - cpu_measurement_hosts.add(flow.generator) - cpu_measurement_hosts.add(flow.receiver) - - cpu_measurement = self.params.cpu_perf_tool( - cpu_measurement_hosts, - perf_recipe_conf, - ) - + for measurements in self.generate_perf_measurements_combinations( + config + ): perf_conf = PerfRecipeConf( - measurements=[cpu_measurement, flows_measurement], + measurements=measurements, iterations=self.params.perf_iterations, )

- perf_conf.register_evaluators( - cpu_measurement, self.cpu_perf_evaluators - ) - perf_conf.register_evaluators( - flows_measurement, self.net_perf_evaluators - ) + self.register_perf_evaluators(perf_conf)

yield perf_conf

- def generate_flow_combinations(self, config): - """Base flow combination generator - - The generator loops over all endpoint pairs to test performance between - (generated by the :any:`generate_perf_endpoints` method) then over all - the selected :any:`ip_versions` and uses the first IP address fitting - these criteria. Then the generator loops over the selected performance - tests as selected via :any:`perf_tests`, then message sizes from - :any:`msg_sizes`. - - :return: list of Flow combinations to measure in parallel - :rtype: List[:any:`PerfFlow`] - """ - for client_nic, server_nic in self.generate_perf_endpoints(config): - for ipv in self.params.ip_versions: - ip_filter = {} - if ipv == "ipv4": - ip_filter.update(family = AF_INET) - elif ipv == "ipv6": - ip_filter.update(family = AF_INET6) - ip_filter.update(is_link_local = False) - - client_bind = client_nic.ips_filter(**ip_filter)[0] - server_bind = server_nic.ips_filter(**ip_filter)[0] - - for perf_test in self.params.perf_tests: - for size in self.params.perf_msg_sizes: - yield [self._create_perf_flow(perf_test, - client_nic, - client_bind, - server_nic, - server_bind, - size, - )] - - def _create_perf_flow(self, perf_test, client_nic, client_bind, server_nic, - server_bind, msg_size) -> PerfFlow: - """ - Wrapper to create a PerfFlow. Mixins that want to change this behavior (for example, to reverse the direction) - can override this method as an alternative to overriding :any:`generate_flow_combinations` - """ - cpupin = self.params.perf_tool_cpu if "perf_tool_cpu" in self.params else None - return PerfFlow(type=perf_test, - generator=client_nic.netns, generator_bind=client_bind, - generator_nic=client_nic, - receiver=server_nic.netns, receiver_bind=server_bind, - receiver_nic=server_nic, - msg_size=msg_size, - duration=self.params.perf_duration, - parallel_streams=self.params.perf_parallel_streams, - cpupin=cpupin, - ) - - def generate_perf_endpoints(self, config): - """Generator for perf endpoints - - To be overriden by a derived class. - - :return: list of device pairs - :rtype: List[Tuple[:any:`Device`, :any:`Device`]] - """ - return [] + def register_perf_evaluators(self, perf_conf): + for measurement in perf_conf.measurements: + if isinstance(measurement, BaseCPUMeasurement): + evaluators = self.cpu_perf_evaluators + elif isinstance(measurement, BaseFlowMeasurement): + evaluators = self.net_perf_evaluators + else: + evaluators = [] + perf_conf.register_evaluators(measurement, evaluators)

@property def cpu_perf_evaluators(self):

From: Ondrej Lichtner olichtne@redhat.com

This is a recipe grouping class for virtual recipes where we use the same list of mixin/composition classes to generate specific config combinations and measurements.

The main difference to the BaremetalEnrtRecipe is the different type of stat cpu measurement generation.

Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/Recipes/ENRT/VirtualEnrtRecipe.py | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lnst/Recipes/ENRT/VirtualEnrtRecipe.py

diff --git a/lnst/Recipes/ENRT/VirtualEnrtRecipe.py b/lnst/Recipes/ENRT/VirtualEnrtRecipe.py new file mode 100644 index 0000000..82d6e3a --- /dev/null +++ b/lnst/Recipes/ENRT/VirtualEnrtRecipe.py @@ -0,0 +1,28 @@ +from lnst.Recipes.ENRT.BaseEnrtRecipe import BaseEnrtRecipe +from lnst.Recipes.ENRT.PerfTestMixins import CommonPerfTestTweakMixin +from lnst.Recipes.ENRT.ConfigMixins.DisableTurboboostMixin import ( + DisableTurboboostMixin, +) +from lnst.Recipes.ENRT.ConfigMixins.DisableIdleStatesMixin import ( + DisableIdleStatesMixin, +) + +from lnst.Recipes.ENRT.MeasurementGenerators.IperfMeasurementGenerator import ( + IperfMeasurementGenerator, +) +from lnst.Recipes.ENRT.MeasurementGenerators.HypervisorsStatCPUMeasurementGenerator import ( + HypervisorsStatCPUMeasurementGenerator, +) + + +class VirtualEnrtRecipe( + CommonPerfTestTweakMixin, + DisableTurboboostMixin, + DisableIdleStatesMixin, + HypervisorsStatCPUMeasurementGenerator, + IperfMeasurementGenerator, + BaseEnrtRecipe, +): + @property + def hypervisor_hosts(self): + return set([self.matched.host1, self.matched.host2])

From: Ondrej Lichtner olichtne@redhat.com

Most of this is centered around adding a reference to the overall configuration of the Recipe to the Perf.* package code. I also started adding a lot of type hints to make it easier to understand which "conf" variable refers to which type of "*Conf" instance...

This is a preliminary version that will require at least one more round of refactoring to actually add all the skipped type hinting, ideally also splitting the commit into multiple parts.

Signed-off-by: Ondrej Lichtner olichtne@redhat.com --- lnst/RecipeCommon/BaseResultEvaluator.py | 9 ++- .../Evaluators/BaselineCPUAverageEvaluator.py | 39 ++++++++-- .../Perf/Evaluators/BaselineEvaluator.py | 77 +++++++++++++++---- .../BaselineFlowAverageEvaluator.py | 25 +++++- .../Perf/Evaluators/NonzeroFlowEvaluator.py | 18 +++-- lnst/RecipeCommon/Perf/Recipe.py | 64 ++++++++++----- lnst/Recipes/ENRT/BaseEnrtRecipe.py | 2 + 7 files changed, 185 insertions(+), 49 deletions(-)

diff --git a/lnst/RecipeCommon/BaseResultEvaluator.py b/lnst/RecipeCommon/BaseResultEvaluator.py index f8902b9..ef7a170 100644 --- a/lnst/RecipeCommon/BaseResultEvaluator.py +++ b/lnst/RecipeCommon/BaseResultEvaluator.py @@ -1,3 +1,10 @@ +from typing import Any + +from lnst.Controller.Recipe import BaseRecipe + + class BaseResultEvaluator(object): - def evaluate_results(self, recipe, results): + def evaluate_results( + self, recipe: BaseRecipe, recipe_conf: Any, results: Any + ): raise NotImplementedError() diff --git a/lnst/RecipeCommon/Perf/Evaluators/BaselineCPUAverageEvaluator.py b/lnst/RecipeCommon/Perf/Evaluators/BaselineCPUAverageEvaluator.py index 0125a31..03ac72b 100644 --- a/lnst/RecipeCommon/Perf/Evaluators/BaselineCPUAverageEvaluator.py +++ b/lnst/RecipeCommon/Perf/Evaluators/BaselineCPUAverageEvaluator.py @@ -1,18 +1,29 @@ from __future__ import division +from typing import List, Tuple

+ +from lnst.Controller.Recipe import BaseRecipe +from lnst.RecipeCommon.Perf.Recipe import RecipeConf as PerfRecipeConf +from lnst.RecipeCommon.Perf.Results import result_averages_difference +from lnst.RecipeCommon.Perf.Measurements.BaseMeasurement import ( + BaseMeasurementResults as PerfMeasurementResults, +) from lnst.RecipeCommon.Perf.Evaluators.BaselineEvaluator import ( BaselineEvaluator, )

-from lnst.RecipeCommon.Perf.Results import result_averages_difference -

class BaselineCPUAverageEvaluator(BaselineEvaluator): def __init__(self, pass_difference, evaluation_filter=None): self._pass_difference = pass_difference self._evaluation_filter = evaluation_filter

- def filter_results(self, recipe, results): + def filter_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[PerfMeasurementResults]: if self._evaluation_filter is None: return results

@@ -25,7 +36,12 @@ def filter_results(self, recipe, results): filtered.append(result) return filtered

- def group_results(self, recipe, results): + def group_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[List[PerfMeasurementResults]]: results_by_host = self._divide_results_by_host(results) for host_results in results_by_host.values(): yield host_results @@ -38,7 +54,12 @@ def _divide_results_by_host(self, results): results_by_host[result.host].append(result) return results_by_host

- def describe_group_results(self, recipe, results): + def describe_group_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[str]: return [ "CPU Baseline average evaluation for Host {hostid}:".format( hostid=results[0].host.hostid @@ -48,7 +69,13 @@ def describe_group_results(self, recipe, results): ), ]

- def compare_result_with_baseline(self, recipe, result, baseline): + def compare_result_with_baseline( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + result: PerfMeasurementResults, + baseline: PerfMeasurementResults, + ) -> Tuple[bool, List[str]]: comparison = True text = [] if baseline is None: diff --git a/lnst/RecipeCommon/Perf/Evaluators/BaselineEvaluator.py b/lnst/RecipeCommon/Perf/Evaluators/BaselineEvaluator.py index 20e3887..d2ff7f8 100644 --- a/lnst/RecipeCommon/Perf/Evaluators/BaselineEvaluator.py +++ b/lnst/RecipeCommon/Perf/Evaluators/BaselineEvaluator.py @@ -1,42 +1,89 @@ +from typing import List, Tuple +from lnst.Controller.Recipe import BaseRecipe from lnst.RecipeCommon.BaseResultEvaluator import BaseResultEvaluator +from lnst.RecipeCommon.Perf.Recipe import RecipeConf as PerfRecipeConf +from lnst.RecipeCommon.Perf.Measurements.BaseMeasurement import ( + BaseMeasurementResults as PerfMeasurementResults, +)

class BaselineEvaluator(BaseResultEvaluator): - def evaluate_results(self, recipe, results): - filtered_results = self.filter_results(recipe, results) + def evaluate_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ): + filtered_results = self.filter_results(recipe, recipe_conf, results)

- for group in self.group_results(recipe, filtered_results): - self.evaluate_group_results(recipe, group) + for group in self.group_results(recipe, recipe_conf, filtered_results): + self.evaluate_group_results(recipe, recipe_conf, group)

- def filter_results(self, recipe, results): + def filter_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[PerfMeasurementResults]: return results

- def group_results(self, recipe, results): + def group_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[List[PerfMeasurementResults]]: for result in results: yield [result]

- def evaluate_group_results(self, recipe, results): + def evaluate_group_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ): comparison_result = True - result_text = self.describe_group_results(recipe, results) + result_text = self.describe_group_results(recipe, recipe_conf, results)

- baselines = self.get_baselines(recipe, results) + baselines = self.get_baselines(recipe, recipe_conf, results) for result, baseline in zip(results, baselines): comparison, text = self.compare_result_with_baseline( - recipe, result, baseline + recipe, recipe_conf, result, baseline ) comparison_result = comparison_result and comparison result_text.extend(text)

recipe.add_result(comparison_result, "\n".join(result_text))

- def describe_group_results(self, recipe, results): + def describe_group_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[str]: return []

- def get_baselines(self, recipe, results): - return [self.get_baseline(recipe, result) for result in results] + def get_baselines( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[PerfMeasurementResults]: + return [self.get_baseline(recipe, recipe_conf, result) for result in results]

- def get_baseline(self, recipe, result): + def get_baseline( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + result: PerfMeasurementResults, + ) -> PerfMeasurementResults: return None

- def compare_result_with_baseline(self, recipe, result, baseline): + def compare_result_with_baseline( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + result: PerfMeasurementResults, + baseline: PerfMeasurementResults, + ) -> Tuple[bool, List[str]]: return False, ["Result to baseline comparison not implemented"] diff --git a/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py b/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py index f7a2fb1..05ee567 100644 --- a/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py +++ b/lnst/RecipeCommon/Perf/Evaluators/BaselineFlowAverageEvaluator.py @@ -1,11 +1,17 @@ from __future__ import division +from typing import List, Tuple

+from lnst.Controller.Recipe import BaseRecipe + +from lnst.RecipeCommon.Perf.Recipe import RecipeConf as PerfRecipeConf +from lnst.RecipeCommon.Perf.Results import result_averages_difference +from lnst.RecipeCommon.Perf.Measurements.BaseMeasurement import ( + BaseMeasurementResults as PerfMeasurementResults, +) from lnst.RecipeCommon.Perf.Evaluators.BaselineEvaluator import ( BaselineEvaluator, )

-from lnst.RecipeCommon.Perf.Results import result_averages_difference -

class BaselineFlowAverageEvaluator(BaselineEvaluator): def __init__(self, pass_difference, metrics_to_evaluate=None): @@ -21,7 +27,12 @@ def __init__(self, pass_difference, metrics_to_evaluate=None): "receiver_cpu_stats", ]

- def describe_group_results(self, recipe, results): + def describe_group_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ) -> List[str]: result = results[0] return [ "Baseline average evaluation of flow:", @@ -31,7 +42,13 @@ def describe_group_results(self, recipe, results): ), ]

- def compare_result_with_baseline(self, recipe, result, baseline): + def compare_result_with_baseline( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + result: PerfMeasurementResults, + baseline: PerfMeasurementResults, + ) -> Tuple[bool, List[str]]: comparison_result = True result_text = [] if baseline is None: diff --git a/lnst/RecipeCommon/Perf/Evaluators/NonzeroFlowEvaluator.py b/lnst/RecipeCommon/Perf/Evaluators/NonzeroFlowEvaluator.py index 0d0921c..8837807 100644 --- a/lnst/RecipeCommon/Perf/Evaluators/NonzeroFlowEvaluator.py +++ b/lnst/RecipeCommon/Perf/Evaluators/NonzeroFlowEvaluator.py @@ -1,13 +1,21 @@ -from lnst.RecipeCommon.BaseResultEvaluator import BaseResultEvaluator +from typing import List, Tuple + +from lnst.Controller.Recipe import BaseRecipe

-from lnst.RecipeCommon.Perf.Measurements.BaseFlowMeasurement import ( - FlowMeasurementResults, - AggregatedFlowMeasurementResults, +from lnst.RecipeCommon.Perf.Recipe import RecipeConf as PerfRecipeConf +from lnst.RecipeCommon.Perf.Measurements.BaseMeasurement import ( + BaseMeasurementResults as PerfMeasurementResults, ) +from lnst.RecipeCommon.BaseResultEvaluator import BaseResultEvaluator

class NonzeroFlowEvaluator(BaseResultEvaluator): - def evaluate_results(self, recipe, results): + def evaluate_results( + self, + recipe: BaseRecipe, + recipe_conf: PerfRecipeConf, + results: List[PerfMeasurementResults], + ): for flow_results in results: result = True result_text = [ diff --git a/lnst/RecipeCommon/Perf/Recipe.py b/lnst/RecipeCommon/Perf/Recipe.py index d1098fa..99aa786 100644 --- a/lnst/RecipeCommon/Perf/Recipe.py +++ b/lnst/RecipeCommon/Perf/Recipe.py @@ -1,21 +1,30 @@ import logging from collections import OrderedDict +from typing import Any, List

from lnst.Common.LnstError import LnstError from lnst.Controller.Recipe import BaseRecipe +from lnst.RecipeCommon.Perf.Measurements.BaseMeasurement import BaseMeasurement from lnst.RecipeCommon.Perf.Results import SequentialPerfResult from lnst.RecipeCommon.Perf.Results import ParallelPerfResult

from lnst.RecipeCommon.Perf.PerfTestMixins import ( - BasePerfTestTweakMixin, - BasePerfTestIterationTweakMixin, + BasePerfTestTweakMixin, + BasePerfTestIterationTweakMixin, )

+ class RecipeConf(object): - def __init__(self, measurements, iterations): + def __init__( + self, + measurements: List[BaseMeasurement], + iterations: int, + parent_recipe_config: Any = None, + ): self._measurements = measurements self._evaluators = dict() self._iterations = iterations + self._parent_recipe_config = parent_recipe_config

@property def measurements(self): @@ -27,7 +36,9 @@ def evaluators(self):

def register_evaluators(self, measurement, evaluators): if measurement not in self.measurements: - raise LnstError("Can't register evaluators for an unknown measurement") + raise LnstError( + "Can't register evaluators for an unknown measurement" + )

self._evaluators[measurement] = list(evaluators)

@@ -35,8 +46,13 @@ def register_evaluators(self, measurement, evaluators): def iterations(self): return self._iterations

+ @property + def parent_recipe_config(self): + return self._parent_recipe_config + + class RecipeResults(object): - def __init__(self, recipe_conf): + def __init__(self, recipe_conf: RecipeConf): self._recipe_conf = recipe_conf self._results = OrderedDict()

@@ -51,11 +67,15 @@ def results(self): def add_measurement_results(self, measurement, new_results): aggregated_results = self._results.get(measurement, None) aggregated_results = measurement.aggregate_results( - aggregated_results, new_results) + aggregated_results, new_results + ) self._results[measurement] = aggregated_results

-class Recipe(BasePerfTestTweakMixin, BasePerfTestIterationTweakMixin, BaseRecipe): - def perf_test(self, recipe_conf): + +class Recipe( + BasePerfTestTweakMixin, BasePerfTestIterationTweakMixin, BaseRecipe +): + def perf_test(self, recipe_conf: RecipeConf): results = RecipeResults(recipe_conf)

self.apply_perf_test_tweak(recipe_conf) @@ -69,7 +89,9 @@ def perf_test(self, recipe_conf):

return results

- def perf_test_iteration(self, recipe_conf, results): + def perf_test_iteration( + self, recipe_conf: RecipeConf, results: RecipeResults + ): self.apply_perf_test_iteration_tweak(recipe_conf) self.describe_perf_test_iteration_tweak(recipe_conf)

@@ -81,20 +103,23 @@ def perf_test_iteration(self, recipe_conf, results): for measurement in recipe_conf.measurements: measurement_results = measurement.collect_results() results.add_measurement_results( - measurement, measurement_results) + measurement, measurement_results + ) finally: self.remove_perf_test_iteration_tweak(recipe_conf)

- def describe_perf_test_iteration_tweak(self, perf_config): - description = self.generate_perf_test_iteration_tweak_description(perf_config) + def describe_perf_test_iteration_tweak(self, recipe_conf: RecipeConf): + description = self.generate_perf_test_iteration_tweak_description( + recipe_conf + ) self.add_result(True, "\n".join(description))

- def perf_report_and_evaluate(self, results): + def perf_report_and_evaluate(self, results: RecipeResults): self.perf_report(results)

self.perf_evaluate(results)

- def perf_report(self, recipe_results): + def perf_report(self, recipe_results: RecipeResults): if not recipe_results: self.add_result(False, "No results available to report.") return @@ -102,7 +127,7 @@ def perf_report(self, recipe_results): for measurement, results in list(recipe_results.results.items()): measurement.report_results(self, results)

- def perf_evaluate(self, recipe_results): + def perf_evaluate(self, recipe_results: RecipeResults): if not recipe_results: self.add_result(False, "No results available to evaluate.") return @@ -112,8 +137,11 @@ def perf_evaluate(self, recipe_results): for measurement, results in list(recipe_results.results.items()): evaluators = recipe_conf.evaluators.get(measurement, []) for evaluator in evaluators: - evaluator.evaluate_results(self, results) + evaluator.evaluate_results(self, recipe_conf, results)

if len(evaluators) == 0: - logging.debug("No evaluator registered for measurement {}" - .format(measurement)) + logging.debug( + "No evaluator registered for measurement {}".format( + measurement + ) + ) diff --git a/lnst/Recipes/ENRT/BaseEnrtRecipe.py b/lnst/Recipes/ENRT/BaseEnrtRecipe.py index 110ee88..9813ba6 100644 --- a/lnst/Recipes/ENRT/BaseEnrtRecipe.py +++ b/lnst/Recipes/ENRT/BaseEnrtRecipe.py @@ -1,4 +1,5 @@ import pprint +import copy from contextlib import contextmanager

from lnst.Common.LnstError import LnstError @@ -399,6 +400,7 @@ def generate_perf_configurations(self, config): perf_conf = PerfRecipeConf( measurements=measurements, iterations=self.params.perf_iterations, + parent_recipe_config=copy.deepcopy(config), )

self.register_perf_evaluators(perf_conf)