Fix fast upload avg calculation Upstream verigak/progress is broken for very fast uploads; e.g. with 'copr build' command, the next() call is called so often that the avg() calculation probably suffers from some small floating-point numbers problems: With 15MB/s => next() called for each 8096B => ~2000 calls/s Since the upstream default window size is only of size 10 items (by default), it calculates the average speed only for the last ~0.005s. We could enlarge the size of window (sma_window param), but the algorithm is so naive that it would decrease the performance (O(N): sum(queue)/len(queue)). This has been discussed very extensively with upstream (PR 24 and friends) but I neither was not able to explain the problem, nor I was able to convince upstream to accept my patches. This downstream patch - while it keeps the backward API compatibility - changes the algorithm so the average speed is calculation is fast enough, and much more stable (by default it calculates speed for window of 2 seconds). Fork with this patch backported is maintained in https://github.com/python-progress/python-progress diff --git a/progress/__init__.py b/progress/__init__.py index b434b30..f26056d 100644 --- a/progress/__init__.py +++ b/progress/__init__.py @@ -30,19 +30,55 @@ HIDE_CURSOR = '\x1b[?25l' SHOW_CURSOR = '\x1b[?25h' +class _Window(object): + max_seconds = 2 + max_items = None + + def __init__(self, max_seconds=2, max_items=None): + self.max_seconds = max_seconds + self.max_items = max_items + + stamp = monotonic() + self.last = stamp - 0.001 + self.counter = 0 + self.deque = deque() + self.next(0, stamp) + + def pop(self): + item = self.deque.popleft() + self.counter -= item[1] + + def clean(self): + if self.max_items: + while len(self.deque) > self.max_items: + self.pop() + while len(self.deque) > 2 and self.last - self.deque[1][0] > float(self.max_seconds): + self.pop() + + def next(self, n, t): + self.clean() + self.deque.append((self.last, n)) + self.last = t + self.counter += n + + @property + def avg(self): + return self.counter / (self.last - self.deque[0][0]) + + class Infinite(object): file = stderr - sma_window = 10 # Simple Moving Average window + # Maximum number of next() calls to be held in Simple Moving Average + # window structure (in memory), default is unlimited. + sma_window_seconds = 2 + sma_window = None check_tty = True hide_cursor = True def __init__(self, message='', **kwargs): self.index = 0 self.start_ts = monotonic() - self.avg = 0 - self._avg_update_ts = self.start_ts - self._ts = self.start_ts - self._xput = deque(maxlen=self.sma_window) + self.window = _Window(self.sma_window_seconds, self.sma_window) for key, val in kwargs.items(): setattr(self, key, val) @@ -69,21 +105,17 @@ class Infinite(object): def elapsed(self): return int(monotonic() - self.start_ts) + @property + def avg(self): + speed = self.window.avg + if speed: + return 1/speed + return float("inf") + @property def elapsed_td(self): return timedelta(seconds=self.elapsed) - def update_avg(self, n, dt): - if n > 0: - xput_len = len(self._xput) - self._xput.append(dt / n) - now = monotonic() - # update when we're still filling _xput, then after every second - if (xput_len < self.sma_window or - now - self._avg_update_ts > 1): - self.avg = sum(self._xput) / len(self._xput) - self._avg_update_ts = now - def update(self): pass @@ -112,14 +144,10 @@ class Infinite(object): try: return self.file.isatty() if self.check_tty else True except AttributeError: - msg = "%s has no attribute 'isatty'. Try setting check_tty=False." % self - raise AttributeError(msg) + raise AttributeError('\'{}\' object has no attribute \'isatty\'. Try setting parameter check_tty=False.'.format(self)) def next(self, n=1): - now = monotonic() - dt = now - self._ts - self.update_avg(n, dt) - self._ts = now + self.window.next(n, monotonic()) self.index = self.index + n self.update()