From 2e851c8ab6e759a75d04d20d763e01e5eb06692f Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Tue, 1 Nov 2016 17:02:16 +0100 Subject: [PATCH] fix moving average calculation These are downstream patches from: https://github.com/python-progress/python-progress/pull/1 Per recent discussions, this seems to be fixed for latest verigak/progress upstream too, OTOH upstream changed API probably -- so once we are there, we could revert those: https://github.com/verigak/progress/issues/34 Resolves: rhbz#1299634 --- ...-dequeue-to-give-us-better-statistic.patch | 73 ++++++++++++++++ 0002-make-the-progress-bar-more-stable.patch | 87 +++++++++++++++++++ python-progress.spec | 10 ++- 3 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch create mode 100644 0002-make-the-progress-bar-more-stable.patch diff --git a/0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch b/0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch new file mode 100644 index 0000000..5ad8b39 --- /dev/null +++ b/0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch @@ -0,0 +1,73 @@ +From 1911decab746ad8046d00768f6125c7a98ed4fd3 Mon Sep 17 00:00:00 2001 +From: Pavel Raiskup +Date: Mon, 18 Jan 2016 18:32:20 +0100 +Subject: [PATCH 1/2] (possibly) enlarge dequeue() to give us better statistics + +It happened on Fedora 23 x86_64 that if the next() was called with +constant window of n=8196 units (in Bytes, default with +MultipartEncoderMonitor) -- some cache/buffer on the road caused +that the sum of deltas started to be highly inaccurate (bar shown +like 40MB instead of 1MB _most_ of the time). This could be +solved by higher sma_window; but we would have to rather large +numbers there and that would result in high complexity. + +So store into deque() also "actual" time() and count all units in +window in separate variable (self._in_window) to avoid sum() +complexity issues. That way we may count: + + (actual_time - last_item_time) / items_in_window + +This would allow us to raise the value of sma_window. + +This resolves verigak/progress/pull/24. +--- + progress/__init__.py | 19 ++++++++++++------- + 1 file changed, 12 insertions(+), 7 deletions(-) + +diff --git a/progress/__init__.py b/progress/__init__.py +index a860e2c..3130fe1 100644 +--- a/progress/__init__.py ++++ b/progress/__init__.py +@@ -32,7 +32,8 @@ class Infinite(object): + self.index = 0 + self.start_ts = time() + self._ts = self.start_ts +- self._dt = deque(maxlen=self.sma_window) ++ self._dt = deque() ++ self._in_window = 0 + for key, val in kwargs.items(): + setattr(self, key, val) + +@@ -43,7 +44,9 @@ class Infinite(object): + + @property + def avg(self): +- return sum(self._dt) / len(self._dt) if self._dt else 0 ++ if not self._in_window: ++ return 0 ++ return (self._ts - self._dt[0]['t']) / self._in_window + + @property + def elapsed(self): +@@ -63,11 +66,13 @@ class Infinite(object): + pass + + def next(self, n=1): +- if n > 0: +- now = time() +- dt = (now - self._ts) / n +- self._dt.append(dt) +- self._ts = now ++ self._ts = time() ++ self._dt.append({'t': self._ts, 'n': n}) ++ self._in_window = self._in_window + n ++ ++ if len(self._dt) > self.sma_window: ++ item = self._dt.popleft() ++ self._in_window = self._in_window - item['n'] + + self.index = self.index + n + self.update() +-- +2.7.4 + diff --git a/0002-make-the-progress-bar-more-stable.patch b/0002-make-the-progress-bar-more-stable.patch new file mode 100644 index 0000000..ba0fedc --- /dev/null +++ b/0002-make-the-progress-bar-more-stable.patch @@ -0,0 +1,87 @@ +From 4691a1efdcd8cbe4bc66d586307daf6505f21db4 Mon Sep 17 00:00:00 2001 +From: Pavel Raiskup +Date: Mon, 18 Jan 2016 19:05:25 +0100 +Subject: [PATCH 2/2] make the progress bar more stable + +Add new sma_delta parameter. That parameter represents time delta +(in seconds) for which _one item_ in deque() is valid until we +create new one. Practically, one item represents number of items +processed during 'sma_delta' time (by default 0.3s). + +Taking into account sma_delta and sma_window values, progress bar +now returns statistics for the last (sma_delta*sma_window) +seconds, which is by default 3s. + +Document a bit. + +This resolves verigak/progress/pull/24. +--- + progress/__init__.py | 43 +++++++++++++++++++++++++++++++++++++++++-- + 1 file changed, 41 insertions(+), 2 deletions(-) + +diff --git a/progress/__init__.py b/progress/__init__.py +index 3130fe1..37b4547 100644 +--- a/progress/__init__.py ++++ b/progress/__init__.py +@@ -24,9 +24,35 @@ from time import time + __version__ = '1.2' + + ++""" ++The moving average is calculated by this formula: ++ ++ _in_window / (time() - oldest_timestamp_in_window) ++ ++Because the frequency of calling next() callback might be rather very high, ++doing the naive next()/avg() implementation leads to issues (PR 23): ++ ++ def next(N): ++ t = time() ++ delta = N / (t - last_timestamp) # very small number ++ _dt.append(delta) ++ _avg = sum(_dt)/len(_dt) ++ last_timestamp = t ++ ++.. Even if 'len(_dt)' was 1000, the time frame can be very small (e.g. file ++download for next(N=8k in bytes) on 1Gbit network. With the mentioned formula ++the window size is limited by *time*, not by the next() callback frequency. ++ ++The moving average is calculated -at most- for the last 3 seconds by default: ++ ++ sma_window x sma_delta = 10 x 0.3 = 3s ++ ++Users can change this default window size, when needed. ++""" + class Infinite(object): + file = stderr +- sma_window = 10 ++ sma_window = 10 # Size of the window -- (max) number of sma_delta items. ++ sma_delta = 0.3 # Time-length of one item in window, in seconds. + + def __init__(self, *args, **kwargs): + self.index = 0 +@@ -67,7 +93,20 @@ class Infinite(object): + + def next(self, n=1): + self._ts = time() +- self._dt.append({'t': self._ts, 'n': n}) ++ ++ item = {'t': self._ts, 'n': 0} ++ if len(self._dt): ++ old_item = self._dt.pop() ++ if self._ts > old_item['t'] + self.sma_delta: ++ # Already reached timeout, we are not going to ++ # touch this item. Return it back. ++ self._dt.append(old_item) ++ else: ++ item = old_item ++ ++ item['n'] = item['n'] + n ++ ++ self._dt.append(item) + self._in_window = self._in_window + n + + if len(self._dt) > self.sma_window: +-- +2.7.4 + diff --git a/python-progress.spec b/python-progress.spec index 25a1f5d..314ffb8 100644 --- a/python-progress.spec +++ b/python-progress.spec @@ -4,7 +4,7 @@ Name: python-%{pypi_name} Version: 1.2 -Release: 8%{?dist} +Release: 9%{?dist} Summary: Easy to use progress bars License: ISC @@ -20,6 +20,9 @@ BuildRequires: python3-devel BuildRequires: python3-setuptools %endif # if with_python3 +Patch1: 0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch +Patch2: 0002-make-the-progress-bar-more-stable.patch + %description Collection of easy to use progress bars and spinners. @@ -34,7 +37,7 @@ Collection of easy to use progress bars and spinners. %prep -%setup -q -n %{pypi_name}-%{version} +%autosetup -p1 -n %{pypi_name}-%{version} # Remove bundled egg-info rm -rf %{pypi_name}.egg-info @@ -78,6 +81,9 @@ popd %endif # with_python3 %changelog +* Tue Nov 01 2016 Pavel Raiskup - 1.2-9 +- fix avg counter for copr-cli (rhbz#1299634) + * Tue Jul 19 2016 Fedora Release Engineering - 1.2-8 - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages