From 8993ddb4f38f2754ae3af1c61e69a2e747f32a67 Mon Sep 17 00:00:00 2001 From: Vratko Polak Date: Tue, 24 Oct 2023 15:44:53 +0200 Subject: [PATCH] feat(MLRsearch): use goal result as in draft05 No effect on NDRPDR results, just different result packaging between the MLRsearch library and the rest of CSIT. - PyPI metadata still to be updated in a separate Change. Change-Id: I547134da189d1d7761594e92f36cc7c1c232ee32 Signed-off-by: Vratko Polak --- resources/libraries/python/MLRsearch/__init__.py | 4 +- .../libraries/python/MLRsearch/goal_result.py | 72 ++++++++++++++++++++++ .../python/MLRsearch/multiple_loss_ratio_search.py | 8 +-- .../libraries/python/MLRsearch/target_stat.py | 1 + .../libraries/python/MLRsearch/trimmed_stat.py | 25 -------- resources/libraries/python/TrafficGenerator.py | 13 ++-- .../robot/performance/performance_display.robot | 4 +- .../robot/performance/performance_utils.robot | 2 +- 8 files changed, 89 insertions(+), 40 deletions(-) create mode 100644 resources/libraries/python/MLRsearch/goal_result.py diff --git a/resources/libraries/python/MLRsearch/__init__.py b/resources/libraries/python/MLRsearch/__init__.py index 349a9a96b3..09ce7e6719 100644 --- a/resources/libraries/python/MLRsearch/__init__.py +++ b/resources/libraries/python/MLRsearch/__init__.py @@ -23,8 +23,8 @@ __init__ file for Python package "MLRsearch". # Import user-facing (API) stuff, so users do not need to know submodules. from .config import Config -from .search_goal import SearchGoal +from .goal_result import GoalResult from .multiple_loss_ratio_search import MultipleLossRatioSearch from .pep3140 import Pep3140Dict +from .search_goal import SearchGoal from .trial_measurement import AbstractMeasurer, MeasurementResult -from .trimmed_stat import TrimmedStat diff --git a/resources/libraries/python/MLRsearch/goal_result.py b/resources/libraries/python/MLRsearch/goal_result.py new file mode 100644 index 0000000000..91dccec0bb --- /dev/null +++ b/resources/libraries/python/MLRsearch/goal_result.py @@ -0,0 +1,72 @@ +# Copyright (c) 2023 Cisco and/or its affiliates. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module defining GoalResult class.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +from .discrete_load import DiscreteLoad +from .relevant_bounds import RelevantBounds +from .trimmed_stat import TrimmedStat + + +@dataclass +class GoalResult: + """Composite to be mapped for each search goal at the end of the search. + + The values are stored as trimmed stats, + the conditional throughput is returned as a discrete loads. + Thus, users interested only in float values have to convert explicitly. + + Irregular goal results are supported as instances with a bound missing. + """ + + relevant_lower_bound: Optional[TrimmedStat] + """The relevant lower bound for the search goal.""" + relevant_upper_bound: Optional[TrimmedStat] + """The relevant lower upper for the search goal.""" + + @staticmethod + def from_bounds(bounds: RelevantBounds) -> GoalResult: + """Factory, so that the call site can be shorter. + + :param bounds: The relevant bounds as found in measurement database. + :type bounds: RelevantBounds + :returns: Newly created instance based on the bounds. + :rtype: GoalResult + """ + return GoalResult( + relevant_lower_bound=bounds.clo, + relevant_upper_bound=bounds.chi, + ) + + @property + def conditional_throughput(self) -> Optional[DiscreteLoad]: + """Compute conditional throughput from the relevant lower bound. + + If the relevant lower bound is missing, None is returned. + + The conditional throughput has the same semantics as load, + so if load is unidirectional and user wants bidirectional + throughput, the manager has to compensate. + + :return: Conditional throughput at the relevant lower bound. + :rtype: Optional[DiscreteLoad] + """ + if not (rlb := self.relevant_lower_bound): + return None + stat = next(iter(rlb.target_to_stat.values())) + return rlb * (1.0 - stat.pessimistic_loss_ratio) diff --git a/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py b/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py index 9f7be4fcd1..4d3ff7c4cb 100644 --- a/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py +++ b/resources/libraries/python/MLRsearch/multiple_loss_ratio_search.py @@ -25,6 +25,7 @@ from .dataclass import secondary_field from .discrete_load import DiscreteLoad from .discrete_result import DiscreteResult from .expander import GlobalWidth +from .goal_result import GoalResult from .limit_handler import LimitHandler from .load_rounding import LoadRounding from .measurement_database import MeasurementDatabase @@ -33,7 +34,6 @@ from .search_goal import SearchGoal from .selector import Selector from .target_scaling import TargetScaling from .trial_measurement import AbstractMeasurer -from .trimmed_stat import TrimmedStat @dataclass @@ -126,7 +126,7 @@ class MultipleLossRatioSearch: self, measurer: AbstractMeasurer, debug: Optional[Callable[[str], None]] = None, - ) -> Pep3140Dict[SearchGoal, Optional[TrimmedStat]]: + ) -> Pep3140Dict[SearchGoal, GoalResult]: """Perform initial trials, create state object, proceed with main loop. Stateful arguments (measurer and debug) are stored. @@ -139,7 +139,7 @@ class MultipleLossRatioSearch: :returns: Structure containing conditional throughputs and other stats, one for each search goal. If a value is None it means there is no lower bound (min load turned out to be an upper bound). - :rtype: Pep3140Dict[SearchGoal, Optional[TrimmedStat]] + :rtype: Pep3140Dict[SearchGoal, GoalResult] :raises RuntimeError: If total duration is larger than timeout, or if min load becomes an upper bound for a search goal that has fail fast true. @@ -168,7 +168,7 @@ class MultipleLossRatioSearch: for goal in self.config.goals: target = self.scaling.goal_to_final_target[goal] bounds = self.database.get_relevant_bounds(target=target) - ret_dict[goal] = bounds.clo + ret_dict[goal] = GoalResult.from_bounds(bounds=bounds) return ret_dict def measure(self, duration: float, load: DiscreteLoad) -> DiscreteResult: diff --git a/resources/libraries/python/MLRsearch/target_stat.py b/resources/libraries/python/MLRsearch/target_stat.py index e558139af7..18e1ff4161 100644 --- a/resources/libraries/python/MLRsearch/target_stat.py +++ b/resources/libraries/python/MLRsearch/target_stat.py @@ -117,6 +117,7 @@ class TargetStat: pessimistic = (effective_dursum - self.good_long) <= limit_dursum return optimistic, pessimistic + @property def pessimistic_loss_ratio(self) -> float: """Return the loss ratio for conditional throughput computation. diff --git a/resources/libraries/python/MLRsearch/trimmed_stat.py b/resources/libraries/python/MLRsearch/trimmed_stat.py index 088e8beaf8..74918d78b0 100644 --- a/resources/libraries/python/MLRsearch/trimmed_stat.py +++ b/resources/libraries/python/MLRsearch/trimmed_stat.py @@ -16,9 +16,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Optional -from .discrete_load import DiscreteLoad from .load_stats import LoadStats from .target_spec import TargetSpec @@ -52,26 +50,3 @@ class TrimmedStat(LoadStats): int_load=stats.int_load, target_to_stat={target: stats.target_to_stat[target]}, ) - - @property - def conditional_throughput(self) -> Optional[DiscreteLoad]: - """Compute conditional throughput from the load. - - The conditional throughput has the same semantics as load, - so if load is unicirectional and user wants bidirectional - throughput, the manager has to compensate. - - This only works correctly if the self load is a lower bound - for the self target, but this method does not check that. - Its should not matter, as MLRsearch controller only returns - the relevant lower bounds to the manager. - - :return: Conditional throughput assuming self is a relevant lower bound. - :rtype: Optional[DiscreteLoad] - :raises RuntimeError: If target is unclear or load is spurious. - """ - target = list(self.target_to_stat.keys())[0] - stat = self.target_to_stat[target] - loss_ratio = stat.pessimistic_loss_ratio() - ret = self * (1.0 - loss_ratio) - return ret diff --git a/resources/libraries/python/TrafficGenerator.py b/resources/libraries/python/TrafficGenerator.py index 9faa3c49f6..f28c77856e 100644 --- a/resources/libraries/python/TrafficGenerator.py +++ b/resources/libraries/python/TrafficGenerator.py @@ -24,8 +24,8 @@ from robot.libraries.BuiltIn import BuiltIn from .Constants import Constants from .DropRateSearch import DropRateSearch from .MLRsearch import ( - AbstractMeasurer, Config, MeasurementResult, - MultipleLossRatioSearch, SearchGoal, TrimmedStat, + AbstractMeasurer, Config, GoalResult, MeasurementResult, + MultipleLossRatioSearch, SearchGoal, ) from .PLRsearch.PLRsearch import PLRsearch from .OptionString import OptionString @@ -1448,7 +1448,7 @@ class OptimizedSearch: ramp_up_rate: float = 0.0, ramp_up_duration: float = 0.0, state_timeout: float = 240.0, - ) -> List[TrimmedStat]: + ) -> List[GoalResult]: """Setup initialized TG, perform optimized search, return intervals. If transaction_scale is nonzero, all init and non-init trial durations @@ -1514,9 +1514,10 @@ class OptimizedSearch: :type ramp_up_rate: float :type ramp_up_duration: float :type state_timeout: float - :returns: Structure containing narrowed down NDR and PDR intervals - and their measurements. - :rtype: List[TrimmedStat] + :returns: Goal result (based on unidirectional tps) for each goal. + The result contains both the offered load for stat trial, + and the conditional throughput for display. + :rtype: List[GoalResult] :raises RuntimeError: If search duration exceeds search_duration_max or if min load becomes an upper bound for any search goal. """ diff --git a/resources/libraries/robot/performance/performance_display.robot b/resources/libraries/robot/performance/performance_display.robot index 791a45db62..6ee493a17e 100644 --- a/resources/libraries/robot/performance/performance_display.robot +++ b/resources/libraries/robot/performance/performance_display.robot @@ -92,8 +92,8 @@ | | | | [Arguments] | ${result} | | -| | ${ndr} = | Convert To Number | ${result[0]} -| | ${pdr} = | Convert To Number | ${result[1]} +| | ${ndr} = | Convert To Number | ${result[0].relevant_lower_bound} +| | ${pdr} = | Convert To Number | ${result[1].relevant_lower_bound} | | Display single bound | NDR | ${result[0].conditional_throughput} | | Display single bound | PDR | ${result[1].conditional_throughput} | | Return From Keyword | ${ndr} | ${pdr} diff --git a/resources/libraries/robot/performance/performance_utils.robot b/resources/libraries/robot/performance/performance_utils.robot index fd03583ac9..4211486a19 100644 --- a/resources/libraries/robot/performance/performance_utils.robot +++ b/resources/libraries/robot/performance/performance_utils.robot @@ -278,7 +278,7 @@ | | ... | use_latency=${use_latency} | | ... | ramp_up_duration=${ramp_up_duration} | | ... | ramp_up_rate=${ramp_up_rate} -| | ${ret} = | Convert To Number | ${result[0]} +| | ${ret} = | Convert To Number | ${result[0].relevant_lower_bound} | | Return From Keyword | ${ret} | Measure and show latency at specified rate -- 2.16.6