fix moving average calculation
These are downstream patches from: https://github.com/python-progress/python-progress/pull/1 Per recent discussions, this seems to be fixed for latest verigak/progress upstream too, OTOH upstream changed API probably -- so once we are there, we could revert those: https://github.com/verigak/progress/issues/34 Resolves: rhbz#1299634
This commit is contained in:
parent
7a12183260
commit
2e851c8ab6
3 changed files with 168 additions and 2 deletions
|
|
@ -0,0 +1,73 @@
|
|||
From 1911decab746ad8046d00768f6125c7a98ed4fd3 Mon Sep 17 00:00:00 2001
|
||||
From: Pavel Raiskup <praiskup@redhat.com>
|
||||
Date: Mon, 18 Jan 2016 18:32:20 +0100
|
||||
Subject: [PATCH 1/2] (possibly) enlarge dequeue() to give us better statistics
|
||||
|
||||
It happened on Fedora 23 x86_64 that if the next() was called with
|
||||
constant window of n=8196 units (in Bytes, default with
|
||||
MultipartEncoderMonitor) -- some cache/buffer on the road caused
|
||||
that the sum of deltas started to be highly inaccurate (bar shown
|
||||
like 40MB instead of 1MB _most_ of the time). This could be
|
||||
solved by higher sma_window; but we would have to rather large
|
||||
numbers there and that would result in high complexity.
|
||||
|
||||
So store into deque() also "actual" time() and count all units in
|
||||
window in separate variable (self._in_window) to avoid sum()
|
||||
complexity issues. That way we may count:
|
||||
|
||||
(actual_time - last_item_time) / items_in_window
|
||||
|
||||
This would allow us to raise the value of sma_window.
|
||||
|
||||
This resolves verigak/progress/pull/24.
|
||||
---
|
||||
progress/__init__.py | 19 ++++++++++++-------
|
||||
1 file changed, 12 insertions(+), 7 deletions(-)
|
||||
|
||||
diff --git a/progress/__init__.py b/progress/__init__.py
|
||||
index a860e2c..3130fe1 100644
|
||||
--- a/progress/__init__.py
|
||||
+++ b/progress/__init__.py
|
||||
@@ -32,7 +32,8 @@ class Infinite(object):
|
||||
self.index = 0
|
||||
self.start_ts = time()
|
||||
self._ts = self.start_ts
|
||||
- self._dt = deque(maxlen=self.sma_window)
|
||||
+ self._dt = deque()
|
||||
+ self._in_window = 0
|
||||
for key, val in kwargs.items():
|
||||
setattr(self, key, val)
|
||||
|
||||
@@ -43,7 +44,9 @@ class Infinite(object):
|
||||
|
||||
@property
|
||||
def avg(self):
|
||||
- return sum(self._dt) / len(self._dt) if self._dt else 0
|
||||
+ if not self._in_window:
|
||||
+ return 0
|
||||
+ return (self._ts - self._dt[0]['t']) / self._in_window
|
||||
|
||||
@property
|
||||
def elapsed(self):
|
||||
@@ -63,11 +66,13 @@ class Infinite(object):
|
||||
pass
|
||||
|
||||
def next(self, n=1):
|
||||
- if n > 0:
|
||||
- now = time()
|
||||
- dt = (now - self._ts) / n
|
||||
- self._dt.append(dt)
|
||||
- self._ts = now
|
||||
+ self._ts = time()
|
||||
+ self._dt.append({'t': self._ts, 'n': n})
|
||||
+ self._in_window = self._in_window + n
|
||||
+
|
||||
+ if len(self._dt) > self.sma_window:
|
||||
+ item = self._dt.popleft()
|
||||
+ self._in_window = self._in_window - item['n']
|
||||
|
||||
self.index = self.index + n
|
||||
self.update()
|
||||
--
|
||||
2.7.4
|
||||
|
||||
87
0002-make-the-progress-bar-more-stable.patch
Normal file
87
0002-make-the-progress-bar-more-stable.patch
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
From 4691a1efdcd8cbe4bc66d586307daf6505f21db4 Mon Sep 17 00:00:00 2001
|
||||
From: Pavel Raiskup <praiskup@redhat.com>
|
||||
Date: Mon, 18 Jan 2016 19:05:25 +0100
|
||||
Subject: [PATCH 2/2] make the progress bar more stable
|
||||
|
||||
Add new sma_delta parameter. That parameter represents time delta
|
||||
(in seconds) for which _one item_ in deque() is valid until we
|
||||
create new one. Practically, one item represents number of items
|
||||
processed during 'sma_delta' time (by default 0.3s).
|
||||
|
||||
Taking into account sma_delta and sma_window values, progress bar
|
||||
now returns statistics for the last (sma_delta*sma_window)
|
||||
seconds, which is by default 3s.
|
||||
|
||||
Document a bit.
|
||||
|
||||
This resolves verigak/progress/pull/24.
|
||||
---
|
||||
progress/__init__.py | 43 +++++++++++++++++++++++++++++++++++++++++--
|
||||
1 file changed, 41 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/progress/__init__.py b/progress/__init__.py
|
||||
index 3130fe1..37b4547 100644
|
||||
--- a/progress/__init__.py
|
||||
+++ b/progress/__init__.py
|
||||
@@ -24,9 +24,35 @@ from time import time
|
||||
__version__ = '1.2'
|
||||
|
||||
|
||||
+"""
|
||||
+The moving average is calculated by this formula:
|
||||
+
|
||||
+ _in_window / (time() - oldest_timestamp_in_window)
|
||||
+
|
||||
+Because the frequency of calling next() callback might be rather very high,
|
||||
+doing the naive next()/avg() implementation leads to issues (PR 23):
|
||||
+
|
||||
+ def next(N):
|
||||
+ t = time()
|
||||
+ delta = N / (t - last_timestamp) # very small number
|
||||
+ _dt.append(delta)
|
||||
+ _avg = sum(_dt)/len(_dt)
|
||||
+ last_timestamp = t
|
||||
+
|
||||
+.. Even if 'len(_dt)' was 1000, the time frame can be very small (e.g. file
|
||||
+download for next(N=8k in bytes) on 1Gbit network. With the mentioned formula
|
||||
+the window size is limited by *time*, not by the next() callback frequency.
|
||||
+
|
||||
+The moving average is calculated -at most- for the last 3 seconds by default:
|
||||
+
|
||||
+ sma_window x sma_delta = 10 x 0.3 = 3s
|
||||
+
|
||||
+Users can change this default window size, when needed.
|
||||
+"""
|
||||
class Infinite(object):
|
||||
file = stderr
|
||||
- sma_window = 10
|
||||
+ sma_window = 10 # Size of the window -- (max) number of sma_delta items.
|
||||
+ sma_delta = 0.3 # Time-length of one item in window, in seconds.
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.index = 0
|
||||
@@ -67,7 +93,20 @@ class Infinite(object):
|
||||
|
||||
def next(self, n=1):
|
||||
self._ts = time()
|
||||
- self._dt.append({'t': self._ts, 'n': n})
|
||||
+
|
||||
+ item = {'t': self._ts, 'n': 0}
|
||||
+ if len(self._dt):
|
||||
+ old_item = self._dt.pop()
|
||||
+ if self._ts > old_item['t'] + self.sma_delta:
|
||||
+ # Already reached timeout, we are not going to
|
||||
+ # touch this item. Return it back.
|
||||
+ self._dt.append(old_item)
|
||||
+ else:
|
||||
+ item = old_item
|
||||
+
|
||||
+ item['n'] = item['n'] + n
|
||||
+
|
||||
+ self._dt.append(item)
|
||||
self._in_window = self._in_window + n
|
||||
|
||||
if len(self._dt) > self.sma_window:
|
||||
--
|
||||
2.7.4
|
||||
|
||||
|
|
@ -4,7 +4,7 @@
|
|||
|
||||
Name: python-%{pypi_name}
|
||||
Version: 1.2
|
||||
Release: 8%{?dist}
|
||||
Release: 9%{?dist}
|
||||
Summary: Easy to use progress bars
|
||||
|
||||
License: ISC
|
||||
|
|
@ -20,6 +20,9 @@ BuildRequires: python3-devel
|
|||
BuildRequires: python3-setuptools
|
||||
%endif # if with_python3
|
||||
|
||||
Patch1: 0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch
|
||||
Patch2: 0002-make-the-progress-bar-more-stable.patch
|
||||
|
||||
%description
|
||||
Collection of easy to use progress bars and spinners.
|
||||
|
||||
|
|
@ -34,7 +37,7 @@ Collection of easy to use progress bars and spinners.
|
|||
|
||||
|
||||
%prep
|
||||
%setup -q -n %{pypi_name}-%{version}
|
||||
%autosetup -p1 -n %{pypi_name}-%{version}
|
||||
# Remove bundled egg-info
|
||||
rm -rf %{pypi_name}.egg-info
|
||||
|
||||
|
|
@ -78,6 +81,9 @@ popd
|
|||
%endif # with_python3
|
||||
|
||||
%changelog
|
||||
* Tue Nov 01 2016 Pavel Raiskup <praiskup@redhat.com> - 1.2-9
|
||||
- fix avg counter for copr-cli (rhbz#1299634)
|
||||
|
||||
* Tue Jul 19 2016 Fedora Release Engineering <rel-eng@lists.fedoraproject.org> - 1.2-8
|
||||
- https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue