rpms/python-progress
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

New upstream release 1.4

Browse source 
Version: 1.4-1
This commit is contained in:
Pavel Raiskup 2018-10-04 10:10:08 +02:00
commit 70075234a2
6 changed files with 133 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1 +1 @@
/progress-1.2.tar.gz
/progress-*.tar.gz

124
0001-fixup-moving-average-window.patch Normal file
View file

@ -0,0 +1,124 @@
Fix fast upload avg calculation
Upstream verigak/progress is broken for very fast uploads; e.g.
with 'copr build' command, the next() call is called so often that
the avg() calculation probably suffers from some small
floating-point numbers problems:
With 15MB/s => next() called for each 8096B => ~2000 calls/s
Since the upstream default window size is only of size 10 items
(by default), it calculates the average speed only for the last
~0.005s. We could enlarge the size of window (sma_window param),
but the algorithm is so naive that it would decrease the
performance.
This has been discussed very extensively with upstream (PR 24 and
friends) but I neither was not able to explain the problem, nor I
was able to convince upstream to accept my patches.
This downstream patch - while it keeps the backward API
compatibility - changes the algorithm so the average speed is
calculation is fast enough, and much more stable (by default it
calculates speed for window of 2 seconds).
Fork with this patch backported is maintained in
https://github.com/python-progress/python-progress
diff --git a/progress/__init__.py b/progress/__init__.py
index a41f65d..1147462 100644
--- a/progress/__init__.py
+++ b/progress/__init__.py
@@ -24,16 +24,53 @@ from time import time
__version__ = '1.4'
+class _Window(object):
+ max_seconds = 2
+ max_items = None
+
+ def __init__(self, max_seconds=2, max_items=None):
+ self.max_seconds = max_seconds
+ self.max_items = max_items
+
+ stamp = time()
+ self.last = stamp - 0.001
+ self.counter = 0
+ self.deque = deque()
+ self.next(0, stamp)
+
+ def pop(self):
+ item = self.deque.popleft()
+ self.counter -= item[1]
+
+ def clean(self):
+ if self.max_items:
+ while len(self.deque) > self.max_items:
+ self.pop()
+ while len(self.deque) > 2 and self.last - self.deque[0][0] > float(self.max_seconds):
+ self.pop()
+
+ def next(self, n, t):
+ self.clean()
+ self.deque.append((self.last, n))
+ self.last = t
+ self.counter += n
+
+ @property
+ def avg(self):
+ return self.counter / (self.last - self.deque[0][0])
+
+
class Infinite(object):
file = stderr
- sma_window = 10 # Simple Moving Average window
+ # Maximum number of next() calls to be held in Simple Moving Average
+ # window structure (in memory), default is unlimited.
+ sma_window_seconds = 2
+ sma_window = None
def __init__(self, *args, **kwargs):
self.index = 0
self.start_ts = time()
- self.avg = 0
- self._ts = self.start_ts
- self._xput = deque(maxlen=self.sma_window)
+ self.window = _Window(self.sma_window_seconds, self.sma_window)
for key, val in kwargs.items():
setattr(self, key, val)
@@ -46,15 +83,17 @@ class Infinite(object):
def elapsed(self):
return int(time() - self.start_ts)
+ @property
+ def avg(self):
+ speed = self.window.avg
+ if speed:
+ return 1/speed
+ return 3600 # better constant?
+
@property
def elapsed_td(self):
return timedelta(seconds=self.elapsed)
- def update_avg(self, n, dt):
- if n > 0:
- self._xput.append(dt / n)
- self.avg = sum(self._xput) / len(self._xput)
-
def update(self):
pass
@@ -65,10 +104,7 @@ class Infinite(object):
pass
def next(self, n=1):
- now = time()
- dt = now - self._ts
- self.update_avg(n, dt)
- self._ts = now
+ self.window.next(n, time())
self.index = self.index + n
self.update()

73
0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch
View file

@ -1,73 +0,0 @@
From 1911decab746ad8046d00768f6125c7a98ed4fd3 Mon Sep 17 00:00:00 2001
From: Pavel Raiskup <praiskup@redhat.com>
Date: Mon, 18 Jan 2016 18:32:20 +0100
Subject: [PATCH 1/2] (possibly) enlarge dequeue() to give us better statistics
It happened on Fedora 23 x86_64 that if the next() was called with
constant window of n=8196 units (in Bytes, default with
MultipartEncoderMonitor) -- some cache/buffer on the road caused
that the sum of deltas started to be highly inaccurate (bar shown
like 40MB instead of 1MB _most_ of the time). This could be
solved by higher sma_window; but we would have to rather large
numbers there and that would result in high complexity.
So store into deque() also "actual" time() and count all units in
window in separate variable (self._in_window) to avoid sum()
complexity issues. That way we may count:
(actual_time - last_item_time) / items_in_window
This would allow us to raise the value of sma_window.
This resolves verigak/progress/pull/24.
---
progress/__init__.py | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/progress/__init__.py b/progress/__init__.py
index a860e2c..3130fe1 100644
--- a/progress/__init__.py
+++ b/progress/__init__.py
@@ -32,7 +32,8 @@ class Infinite(object):
self.index = 0
self.start_ts = time()
self._ts = self.start_ts
- self._dt = deque(maxlen=self.sma_window)
+ self._dt = deque()
+ self._in_window = 0
for key, val in kwargs.items():
setattr(self, key, val)
@@ -43,7 +44,9 @@ class Infinite(object):
@property
def avg(self):
- return sum(self._dt) / len(self._dt) if self._dt else 0
+ if not self._in_window:
+ return 0
+ return (self._ts - self._dt[0]['t']) / self._in_window
@property
def elapsed(self):
@@ -63,11 +66,13 @@ class Infinite(object):
pass
def next(self, n=1):
- if n > 0:
- now = time()
- dt = (now - self._ts) / n
- self._dt.append(dt)
- self._ts = now
+ self._ts = time()
+ self._dt.append({'t': self._ts, 'n': n})
+ self._in_window = self._in_window + n
+
+ if len(self._dt) > self.sma_window:
+ item = self._dt.popleft()
+ self._in_window = self._in_window - item['n']
self.index = self.index + n
self.update()
--
2.7.4

87
0002-make-the-progress-bar-more-stable.patch
View file

@ -1,87 +0,0 @@
From 4691a1efdcd8cbe4bc66d586307daf6505f21db4 Mon Sep 17 00:00:00 2001
From: Pavel Raiskup <praiskup@redhat.com>
Date: Mon, 18 Jan 2016 19:05:25 +0100
Subject: [PATCH 2/2] make the progress bar more stable
Add new sma_delta parameter. That parameter represents time delta
(in seconds) for which _one item_ in deque() is valid until we
create new one. Practically, one item represents number of items
processed during 'sma_delta' time (by default 0.3s).
Taking into account sma_delta and sma_window values, progress bar
now returns statistics for the last (sma_delta*sma_window)
seconds, which is by default 3s.
Document a bit.
This resolves verigak/progress/pull/24.
---
progress/__init__.py | 43 +++++++++++++++++++++++++++++++++++++++++--
1 file changed, 41 insertions(+), 2 deletions(-)
diff --git a/progress/__init__.py b/progress/__init__.py
index 3130fe1..37b4547 100644
--- a/progress/__init__.py
+++ b/progress/__init__.py
@@ -24,9 +24,35 @@ from time import time
__version__ = '1.2'
+"""
+The moving average is calculated by this formula:
+
+ _in_window / (time() - oldest_timestamp_in_window)
+
+Because the frequency of calling next() callback might be rather very high,
+doing the naive next()/avg() implementation leads to issues (PR 23):
+
+ def next(N):
+ t = time()
+ delta = N / (t - last_timestamp) # very small number
+ _dt.append(delta)
+ _avg = sum(_dt)/len(_dt)
+ last_timestamp = t
+
+.. Even if 'len(_dt)' was 1000, the time frame can be very small (e.g. file
+download for next(N=8k in bytes) on 1Gbit network. With the mentioned formula
+the window size is limited by *time*, not by the next() callback frequency.
+
+The moving average is calculated -at most- for the last 3 seconds by default:
+
+ sma_window x sma_delta = 10 x 0.3 = 3s
+
+Users can change this default window size, when needed.
+"""
class Infinite(object):
file = stderr
- sma_window = 10
+ sma_window = 10 # Size of the window -- (max) number of sma_delta items.
+ sma_delta = 0.3 # Time-length of one item in window, in seconds.
def __init__(self, *args, **kwargs):
self.index = 0
@@ -67,7 +93,20 @@ class Infinite(object):
def next(self, n=1):
self._ts = time()
- self._dt.append({'t': self._ts, 'n': n})
+
+ item = {'t': self._ts, 'n': 0}
+ if len(self._dt):
+ old_item = self._dt.pop()
+ if self._ts > old_item['t'] + self.sma_delta:
+ # Already reached timeout, we are not going to
+ # touch this item. Return it back.
+ self._dt.append(old_item)
+ else:
+ item = old_item
+
+ item['n'] = item['n'] + n
+
+ self._dt.append(item)
self._in_window = self._in_window + n
if len(self._dt) > self.sma_window:
--
2.7.4

12
python-progress.spec
View file

@ -19,13 +19,13 @@
%global pypi_name progress
Name: python-%{pypi_name}
Version: 1.2
Release: 19%{?dist}
Version: 1.4
Release: 1%{?dist}
Summary: Easy to use progress bars
License: ISC
URL: http://github.com/verigak/progress/
Source0: https://pypi.python.org/packages/source/p/%{pypi_name}/%{pypi_name}-%{version}.tar.gz
Source0: %pypi_source
BuildArch: noarch
%if %{with python2}
@ -38,8 +38,7 @@ BuildRequires: python3-devel
BuildRequires: python3-setuptools
%endif
Patch1: 0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch
Patch2: 0002-make-the-progress-bar-more-stable.patch
Patch1: 0001-fixup-moving-average-window.patch
%global _description\
Collection of easy to use progress bars and spinners.\
@ -99,6 +98,9 @@ rm -rf %{pypi_name}.egg-info
%changelog
* Thu Oct 04 2018 Pavel Raiskup <praiskup@redhat.com> - 1.4-1
- latest upstream release
* Wed Oct 03 2018 Pavel Raiskup <praiskup@redhat.com> - 1.2-19
- no python2 in f30+ (rhbz#1634951)
- fix el6 build (rhbz#1310704)

2
sources
View file

@ -1 +1 @@
c1dbf49a41e80408d3874d976ba894cc progress-1.2.tar.gz
SHA512 (progress-1.4.tar.gz) = f9973ac4f670632fa4ab27232f53c0c74992329a298f95ea6a39ba7a4cbdccfacd44a49c367bf45d739d4fb652e13c271792eff509c9277e2250c2cdb3e82c48