Repository navigation
Add support to track the duration of fuzz task steps #5555
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,7 @@ | |
| import bisect | ||
| import collections | ||
| import contextlib | ||
| import datetime | ||
| import functools | ||
| import itertools | ||
| import re | ||
|
|
@@ -424,6 +425,12 @@ def monitoring_v3_time_series(self, time_series, labels, start_time, end_time, | |
|
|
||
| return time_series | ||
|
|
||
| def has_field(self, field): | ||
| for f in self.field_spec: | ||
| if f.value_type == field.value_type and f.name == field.name: | ||
| return True | ||
| return False | ||
|
|
||
|
|
||
| class _CounterMetric(Metric): | ||
| """Counter metric.""" | ||
|
|
@@ -658,6 +665,74 @@ def fail(self): | |
| self.failed = True | ||
|
|
||
|
|
||
| class TimeStatTrackerFailureException(Exception): | ||
| pass | ||
|
|
||
|
|
||
| class TimeStatTracker: | ||
|
ViniciustCosta marked this conversation as resolved.
Resolved
|
||
| """Reusable context manager to keep track of time stats. | ||
|
|
||
| Time measurements are emitted to the supplied Cloud Monitoring metric and | ||
| are saved as datetime.timedelta in the |self.stats| dictionary as well. | ||
| """ | ||
|
|
||
| def __init__(self, metric: Metric | None = None, common_labels=None): | ||
| """Initializes the tracker. | ||
|
|
||
| Args: | ||
| metric: The Cloud Monitoring metric to submit measurements to, if not | ||
| None. The metric must contain the fields "step" and "success", which | ||
| are automatically filled by the tracker upon emission. | ||
| common_labels: The dictionary of labels to be submitted to the metric. | ||
| The "step" and "success" labels are ignored if present. | ||
| """ | ||
| self.stats: dict[str, datetime.timedelta] = {} | ||
| self.metric = metric | ||
| self.common_labels = dict(common_labels) if common_labels else {} | ||
| if metric: | ||
|
ViniciustCosta marked this conversation as resolved.
Resolved
|
||
| if not metric.has_field(StringField('step')): | ||
| logs.error('Missing "step" string field in the metric.') | ||
| self.metric = None | ||
| if not metric.has_field(BooleanField('success')): | ||
| logs.error('Missing "success" boolean field in the metric.') | ||
| self.metric = None | ||
| if 'step' in self.common_labels or 'success' in self.common_labels: | ||
| logs.warning('Labels "step" or "success" are in common_labels ' | ||
| 'and will be overwritten.') | ||
|
|
||
| @contextlib.contextmanager | ||
| def track(self, name: str): | ||
| """Tracks the stat named |name|. | ||
|
|
||
| Any raised exception is treated as a failure and re-raised, except for | ||
| TimeStatTrackerFailureException, which marks the step as failed without | ||
| propagating. | ||
| """ | ||
| start = time.time() | ||
| success = True | ||
| try: | ||
| yield | ||
| except TimeStatTrackerFailureException: | ||
| success = False | ||
| except: | ||
| success = False | ||
| raise | ||
| finally: | ||
|
Collaborator
There was a problem hiding this comment. Are there cases when the same step runs more than once (loops, retries)? Right now it warns and overwrites. Should durations be summed instead?
Contributor
Author
There was a problem hiding this comment. Yes, there are. In this case I opted to make a tracker for each loop (as in PR #5569). Maybe summing it up is a simpler solution. As a matter of fact, I opted for the first solution (make a tracker for each loop) in order to get the stats for TestcaseRun, but later on (PR #5571) I had to do the same for blackbox fuzzing, so now I don't think it's the best solution anymore.
Contributor
Author
There was a problem hiding this comment. Done, now the time measurements are accumulated.
Collaborator
There was a problem hiding this comment. Hmmm with this accumulation, stats holds the total per session, but the metric still gets one sample per call in Cloud Monitorign. For steps inside loops, BQ (session total) and CM (per-iteration distribution) will mean different things. We could send the total once per step at the end. In any case, please document it in the metric description.
Contributor
Author
There was a problem hiding this comment. Ah, yes, that's intended, I forgot to add a comment explaining this. Sending the total once the steps end is a little bit difficult to do because of the following reasons: Also, by sending the measurements in each step you gain granularity, so I don't see any problem in doing this.
Contributor
Author
There was a problem hiding this comment. I updated the metric description to include this scenario.
Collaborator
There was a problem hiding this comment. Makes sense to me! As we discussed offline, we might need to revisit that since some steps can loop over more than 100 iterations.
|
||
| duration_secs = time.time() - start | ||
| duration = datetime.timedelta(seconds=duration_secs) | ||
| if name in self.stats: | ||
| self.stats[name] += duration | ||
| else: | ||
| self.stats[name] = duration | ||
| if self.metric: | ||
| self.metric.insert( | ||
| duration_secs, | ||
| labels={ | ||
| **self.common_labels, 'step': name, | ||
| 'success': success | ||
| }) | ||
|
|
||
|
|
||
| # Global state. | ||
| _metrics_store = _MetricsStore() | ||
| _monitoring_v3_client = None | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.