diff --git a/.gitignore b/.gitignore index 877e889..bf39ebb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -/progress-1.2.tar.gz +/progress-*.tar.gz diff --git a/0001-fixup-moving-average-window.patch b/0001-fixup-moving-average-window.patch new file mode 100644 index 0000000..0f3c8bc --- /dev/null +++ b/0001-fixup-moving-average-window.patch @@ -0,0 +1,124 @@ +Fix fast upload avg calculation + +Upstream verigak/progress is broken for very fast uploads; e.g. +with 'copr build' command, the next() call is called so often that +the avg() calculation probably suffers from some small +floating-point numbers problems: + + With 15MB/s => next() called for each 8096B => ~2000 calls/s + +Since the upstream default window size is only of size 10 items +(by default), it calculates the average speed only for the last +~0.005s. We could enlarge the size of window (sma_window param), +but the algorithm is so naive that it would decrease the +performance. + +This has been discussed very extensively with upstream (PR 24 and +friends) but I neither was not able to explain the problem, nor I +was able to convince upstream to accept my patches. + +This downstream patch - while it keeps the backward API +compatibility - changes the algorithm so the average speed is +calculation is fast enough, and much more stable (by default it +calculates speed for window of 2 seconds). + +Fork with this patch backported is maintained in +https://github.com/python-progress/python-progress + +diff --git a/progress/__init__.py b/progress/__init__.py +index a41f65d..1147462 100644 +--- a/progress/__init__.py ++++ b/progress/__init__.py +@@ -24,16 +24,53 @@ from time import time + __version__ = '1.4' + + ++class _Window(object): ++ max_seconds = 2 ++ max_items = None ++ ++ def __init__(self, max_seconds=2, max_items=None): ++ self.max_seconds = max_seconds ++ self.max_items = max_items ++ ++ stamp = time() ++ self.last = stamp - 0.001 ++ self.counter = 0 ++ self.deque = deque() ++ self.next(0, stamp) ++ ++ def pop(self): ++ item = self.deque.popleft() ++ self.counter -= item[1] ++ ++ def clean(self): ++ if self.max_items: ++ while len(self.deque) > self.max_items: ++ self.pop() ++ while len(self.deque) > 2 and self.last - self.deque[0][0] > float(self.max_seconds): ++ self.pop() ++ ++ def next(self, n, t): ++ self.clean() ++ self.deque.append((self.last, n)) ++ self.last = t ++ self.counter += n ++ ++ @property ++ def avg(self): ++ return self.counter / (self.last - self.deque[0][0]) ++ ++ + class Infinite(object): + file = stderr +- sma_window = 10 # Simple Moving Average window ++ # Maximum number of next() calls to be held in Simple Moving Average ++ # window structure (in memory), default is unlimited. ++ sma_window_seconds = 2 ++ sma_window = None + + def __init__(self, *args, **kwargs): + self.index = 0 + self.start_ts = time() +- self.avg = 0 +- self._ts = self.start_ts +- self._xput = deque(maxlen=self.sma_window) ++ self.window = _Window(self.sma_window_seconds, self.sma_window) + for key, val in kwargs.items(): + setattr(self, key, val) + +@@ -46,15 +83,17 @@ class Infinite(object): + def elapsed(self): + return int(time() - self.start_ts) + ++ @property ++ def avg(self): ++ speed = self.window.avg ++ if speed: ++ return 1/speed ++ return 3600 # better constant? ++ + @property + def elapsed_td(self): + return timedelta(seconds=self.elapsed) + +- def update_avg(self, n, dt): +- if n > 0: +- self._xput.append(dt / n) +- self.avg = sum(self._xput) / len(self._xput) +- + def update(self): + pass + +@@ -65,10 +104,7 @@ class Infinite(object): + pass + + def next(self, n=1): +- now = time() +- dt = now - self._ts +- self.update_avg(n, dt) +- self._ts = now ++ self.window.next(n, time()) + self.index = self.index + n + self.update() + diff --git a/0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch b/0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch deleted file mode 100644 index 5ad8b39..0000000 --- a/0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch +++ /dev/null @@ -1,73 +0,0 @@ -From 1911decab746ad8046d00768f6125c7a98ed4fd3 Mon Sep 17 00:00:00 2001 -From: Pavel Raiskup -Date: Mon, 18 Jan 2016 18:32:20 +0100 -Subject: [PATCH 1/2] (possibly) enlarge dequeue() to give us better statistics - -It happened on Fedora 23 x86_64 that if the next() was called with -constant window of n=8196 units (in Bytes, default with -MultipartEncoderMonitor) -- some cache/buffer on the road caused -that the sum of deltas started to be highly inaccurate (bar shown -like 40MB instead of 1MB _most_ of the time). This could be -solved by higher sma_window; but we would have to rather large -numbers there and that would result in high complexity. - -So store into deque() also "actual" time() and count all units in -window in separate variable (self._in_window) to avoid sum() -complexity issues. That way we may count: - - (actual_time - last_item_time) / items_in_window - -This would allow us to raise the value of sma_window. - -This resolves verigak/progress/pull/24. ---- - progress/__init__.py | 19 ++++++++++++------- - 1 file changed, 12 insertions(+), 7 deletions(-) - -diff --git a/progress/__init__.py b/progress/__init__.py -index a860e2c..3130fe1 100644 ---- a/progress/__init__.py -+++ b/progress/__init__.py -@@ -32,7 +32,8 @@ class Infinite(object): - self.index = 0 - self.start_ts = time() - self._ts = self.start_ts -- self._dt = deque(maxlen=self.sma_window) -+ self._dt = deque() -+ self._in_window = 0 - for key, val in kwargs.items(): - setattr(self, key, val) - -@@ -43,7 +44,9 @@ class Infinite(object): - - @property - def avg(self): -- return sum(self._dt) / len(self._dt) if self._dt else 0 -+ if not self._in_window: -+ return 0 -+ return (self._ts - self._dt[0]['t']) / self._in_window - - @property - def elapsed(self): -@@ -63,11 +66,13 @@ class Infinite(object): - pass - - def next(self, n=1): -- if n > 0: -- now = time() -- dt = (now - self._ts) / n -- self._dt.append(dt) -- self._ts = now -+ self._ts = time() -+ self._dt.append({'t': self._ts, 'n': n}) -+ self._in_window = self._in_window + n -+ -+ if len(self._dt) > self.sma_window: -+ item = self._dt.popleft() -+ self._in_window = self._in_window - item['n'] - - self.index = self.index + n - self.update() --- -2.7.4 - diff --git a/0002-make-the-progress-bar-more-stable.patch b/0002-make-the-progress-bar-more-stable.patch deleted file mode 100644 index ba0fedc..0000000 --- a/0002-make-the-progress-bar-more-stable.patch +++ /dev/null @@ -1,87 +0,0 @@ -From 4691a1efdcd8cbe4bc66d586307daf6505f21db4 Mon Sep 17 00:00:00 2001 -From: Pavel Raiskup -Date: Mon, 18 Jan 2016 19:05:25 +0100 -Subject: [PATCH 2/2] make the progress bar more stable - -Add new sma_delta parameter. That parameter represents time delta -(in seconds) for which _one item_ in deque() is valid until we -create new one. Practically, one item represents number of items -processed during 'sma_delta' time (by default 0.3s). - -Taking into account sma_delta and sma_window values, progress bar -now returns statistics for the last (sma_delta*sma_window) -seconds, which is by default 3s. - -Document a bit. - -This resolves verigak/progress/pull/24. ---- - progress/__init__.py | 43 +++++++++++++++++++++++++++++++++++++++++-- - 1 file changed, 41 insertions(+), 2 deletions(-) - -diff --git a/progress/__init__.py b/progress/__init__.py -index 3130fe1..37b4547 100644 ---- a/progress/__init__.py -+++ b/progress/__init__.py -@@ -24,9 +24,35 @@ from time import time - __version__ = '1.2' - - -+""" -+The moving average is calculated by this formula: -+ -+ _in_window / (time() - oldest_timestamp_in_window) -+ -+Because the frequency of calling next() callback might be rather very high, -+doing the naive next()/avg() implementation leads to issues (PR 23): -+ -+ def next(N): -+ t = time() -+ delta = N / (t - last_timestamp) # very small number -+ _dt.append(delta) -+ _avg = sum(_dt)/len(_dt) -+ last_timestamp = t -+ -+.. Even if 'len(_dt)' was 1000, the time frame can be very small (e.g. file -+download for next(N=8k in bytes) on 1Gbit network. With the mentioned formula -+the window size is limited by *time*, not by the next() callback frequency. -+ -+The moving average is calculated -at most- for the last 3 seconds by default: -+ -+ sma_window x sma_delta = 10 x 0.3 = 3s -+ -+Users can change this default window size, when needed. -+""" - class Infinite(object): - file = stderr -- sma_window = 10 -+ sma_window = 10 # Size of the window -- (max) number of sma_delta items. -+ sma_delta = 0.3 # Time-length of one item in window, in seconds. - - def __init__(self, *args, **kwargs): - self.index = 0 -@@ -67,7 +93,20 @@ class Infinite(object): - - def next(self, n=1): - self._ts = time() -- self._dt.append({'t': self._ts, 'n': n}) -+ -+ item = {'t': self._ts, 'n': 0} -+ if len(self._dt): -+ old_item = self._dt.pop() -+ if self._ts > old_item['t'] + self.sma_delta: -+ # Already reached timeout, we are not going to -+ # touch this item. Return it back. -+ self._dt.append(old_item) -+ else: -+ item = old_item -+ -+ item['n'] = item['n'] + n -+ -+ self._dt.append(item) - self._in_window = self._in_window + n - - if len(self._dt) > self.sma_window: --- -2.7.4 - diff --git a/python-progress.spec b/python-progress.spec index ca384d3..c92b545 100644 --- a/python-progress.spec +++ b/python-progress.spec @@ -1,39 +1,68 @@ +%if 0%{?fedora} + %bcond_without python3 + %if 0%{?fedora} > 29 + %bcond_with python2 + %else + %bcond_without python2 + %endif +%else + %if 0%{?rhel} > 7 + %bcond_with python2 + %bcond_without python3 + %else + %bcond_without python2 + %bcond_with python3 + %endif +%endif + # Created by pyp2rpm-0.5.2 %global pypi_name progress -%global with_python3 0%{?fedora} Name: python-%{pypi_name} -Version: 1.2 -Release: 10%{?dist} +Version: 1.4 +Release: 2%{?dist} Summary: Easy to use progress bars License: ISC URL: http://github.com/verigak/progress/ -Source0: https://pypi.python.org/packages/source/p/%{pypi_name}/%{pypi_name}-%{version}.tar.gz +Source0: %pypi_source BuildArch: noarch - + +%if %{with python2} BuildRequires: python2-devel -BuildRequires: python-setuptools - -%if %{?with_python3} +BuildRequires: python2-setuptools +%endif + +%if %{with python3} BuildRequires: python3-devel BuildRequires: python3-setuptools -%endif # if with_python3 +%endif -Patch1: 0001-possibly-enlarge-dequeue-to-give-us-better-statistic.patch -Patch2: 0002-make-the-progress-bar-more-stable.patch +Patch1: 0001-fixup-moving-average-window.patch -%description -Collection of easy to use progress bars and spinners. +%global _description\ +Collection of easy to use progress bars and spinners.\ -%if 0%{?with_python3} +%description %_description + + +%if 0%{with python2} +%package -n python2-%{pypi_name} +Summary: %summary +%{?python_provide:%python_provide python2-%{pypi_name}} + +%description -n python2-%{pypi_name} %_description +%endif # python2 + + +%if 0%{with python3} %package -n python3-%{pypi_name} Summary: Easy to use progress bars %description -n python3-%{pypi_name} Collection of easy to use progress bars and spinners. -%endif # with_python3 +%endif # python3 %prep @@ -42,48 +71,68 @@ Collection of easy to use progress bars and spinners. # Remove bundled egg-info rm -rf %{pypi_name}.egg-info -%if 0%{?with_python3} -rm -rf %{py3dir} -cp -a . %{py3dir} -find %{py3dir} -name '*.py' | xargs sed -i '1s|^#!python|#!%{__python3}|' -%endif # with_python3 %build -%{__python2} setup.py build +%{?with_python2: %py2_build} +%{?with_python3: %py3_build} -%if 0%{?with_python3} -pushd %{py3dir} -%{__python3} setup.py build -popd -%endif # with_python3 %install -# Must do the subpackages' install first because the scripts in /usr/bin are -# overwritten with every setup.py install (and we want the python2 version -# to be the default for now). -%if 0%{?with_python3} -pushd %{py3dir} -%{__python3} setup.py install --skip-build --root %{buildroot} -popd -%endif # with_python3 +%{?with_python2: %py2_install} +%{?with_python3: %py3_install} -%{__python2} setup.py install --skip-build --root %{buildroot} -%files +%if 0%{with python2} +%files -n python2-%{pypi_name} %doc README.rst LICENSE %{python2_sitelib}/%{pypi_name} %{python2_sitelib}/%{pypi_name}-%{version}-py?.?.egg-info +%endif -%if 0%{?with_python3} +%if 0%{with python3} %files -n python3-%{pypi_name} %doc README.rst LICENSE %{python3_sitelib}/%{pypi_name} %{python3_sitelib}/%{pypi_name}-%{version}-py?.?.egg-info -%endif # with_python3 +%endif + %changelog -* Sat Sep 01 2018 Pavel Raiskup - 1.2-10 -- el6 build requires newline after %%autosetup (rhbz#1310704) +* Tue Dec 11 2018 Pavel Raiskup - 1.4-2 +- followup for previous commit, PR#3 + +* Wed Dec 05 2018 Gwyn Ciesla - 1.4-1 +- 1.4 + +* Wed Oct 03 2018 Pavel Raiskup - 1.2-19 +- no python2 in f30+ (rhbz#1634951) +- fix el6 build (rhbz#1310704) + +* Sat Jul 14 2018 Fedora Release Engineering - 1.2-18 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild + +* Tue Jun 19 2018 Miro Hrončok - 1.2-17 +- Rebuilt for Python 3.7 + +* Fri Feb 09 2018 Fedora Release Engineering - 1.2-16 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild + +* Sat Jan 27 2018 Iryna Shcherbina - 1.2-15 +- Update Python 2 dependency declarations to new packaging standards + (See https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3) + +* Sat Aug 19 2017 Zbigniew Jędrzejewski-Szmek - 1.2-14 +- Python 2 binary package renamed to python2-progress + See https://fedoraproject.org/wiki/FinalizingFedoraSwitchtoPython3 + +* Thu Jul 27 2017 Fedora Release Engineering - 1.2-13 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild + +* Sat Feb 11 2017 Fedora Release Engineering - 1.2-12 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild + +* Mon Dec 19 2016 Miro Hrončok - 1.2-11 +- Rebuild for Python 3.6 * Thu Dec 08 2016 Pavel Raiskup - 1.2-10 - keep enabled python3 subpackage only in fedora, and merge into epel7 diff --git a/sources b/sources index 569ca80..487c028 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -c1dbf49a41e80408d3874d976ba894cc progress-1.2.tar.gz +SHA512 (progress-1.4.tar.gz) = f9973ac4f670632fa4ab27232f53c0c74992329a298f95ea6a39ba7a4cbdccfacd44a49c367bf45d739d4fb652e13c271792eff509c9277e2250c2cdb3e82c48