Fix non-removal of corrupt metadata files. BZ 908870.

- No async downloading when --cacheonly.  BZ 830523.
- compare mtime without sub second precision. BZ 831918
- show_lock_owner: report errors if we fail.  BZ 745281
- quote uids to keep cachedir ascii-clean.  BZ 832195
- A solution to the obsoletes but don't provide problem. BZ 834530
- completion-helper: use the system cachedir

BZ-908870-MD-files-bad.patch

@@ -0,0 +1,241 @@
+commit 83fcbe745c3ee8f5f0fa29626a86c824db059b22
+Author: James Antill <james@and.org>
+Date:   Thu Feb 7 12:58:07 2013 -0500
+
+    Fix problems with mirrors like wtfnix.com, delete bad MD files.
+
+diff --git a/yum/yumRepo.py b/yum/yumRepo.py
+index dfcf8f9..efbc42a 100644
+--- a/yum/yumRepo.py
++++ b/yum/yumRepo.py
+@@ -940,6 +941,7 @@ Insufficient space in download directory %s
+                                     range=(start, end),
+                                     )
+             except URLGrabError, e:
++                self._del_dl_file(local, size)
+                 errstr = "failed to retrieve %s from %s\nerror was %s" % (relative, self, e)
+                 if self.mirrorurls:
+                     errstr +="\n  You could try running: yum clean expire-cache"
+@@ -961,6 +963,7 @@ Insufficient space in download directory %s
+                                            **kwargs
+                                            )
+             except URLGrabError, e:
++                self._del_dl_file(local, size)
+                 errstr = "failure: %s from %s: %s" % (relative, self, e)
+                 errors = getattr(e, 'errors', None)
+                 raise Errors.NoMoreMirrorsRepoError(errstr, errors)
+@@ -1652,6 +1655,18 @@ Insufficient space in download directory %s
+             raise URLGrabError(-1, 'repomd.xml does not match metalink for %s' %
+                                self)
+ 
++    def _del_dl_file(self, local, size):
++        """ Delete a downloaded file if it's the correct size. """
++
++        sd = misc.stat_f(local)
++        if not sd: # File doesn't exist...
++            return
++
++        if size and sd.st_size < size:
++            return # Still more to get...
++
++        # Is the correct size, or too big ... delete it so we'll try again.
++        misc.unlink_f(local)
+ 
+     def checkMD(self, fn, mdtype, openchecksum=False):
+         """check the metadata type against its checksum"""
+@@ -1681,7 +1696,7 @@ Insufficient space in download directory %s
+         if size is not None:
+             size = int(size)
+ 
+-        if fast:
++        if fast and skip_old_DBMD_check:
+             fsize = misc.stat_f(file)
+             if fsize is None: # File doesn't exist...
+                 return None
+@@ -1756,16 +1771,21 @@ Insufficient space in download directory %s
+ 
+         try:
+             def checkfunc(obj):
+-                self.checkMD(obj, mdtype)
++                try:
++                    self.checkMD(obj, mdtype)
++                except URLGrabError:
++                    #  Don't share MD among mirrors, in theory we could use:
++                    #     self._del_dl_file(local, int(thisdata.size))
++                    # ...but this is safer.
++                    misc.unlink_f(obj.filename)
++                    raise
+                 self.retrieved[mdtype] = 1
+             text = "%s/%s" % (self, mdtype)
+             if thisdata.size is None:
+                 reget = None
+             else:
+                 reget = 'simple'
+-                if os.path.exists(local):
+-                    if os.stat(local).st_size >= int(thisdata.size):
+-                        misc.unlink_f(local)
++                self._del_dl_file(local, int(thisdata.size))
+             local = self._getFile(relative=remote,
+                                   local=local, 
+                                   copy_local=1,
+commit c148eb10b798270b3d15087433c8efb2a79a69d0
+Author: James Antill <james@and.org>
+Date:   Mon Feb 18 16:17:06 2013 -0500
+
+    Use xattr data as well as file size for "fast checksumming".
+
+diff --git a/yum/yumRepo.py b/yum/yumRepo.py
+index efbc42a..8c38093 100644
+--- a/yum/yumRepo.py
++++ b/yum/yumRepo.py
+@@ -52,15 +52,54 @@ import stat
+ import errno
+ import tempfile
+ 
+-#  If you want yum to _always_ check the MD .sqlite files then set this to
+-# False (this doesn't affect .xml files or .sqilte files derived from them).
+-# With this as True yum will only check when a new repomd.xml or
+-# new MD is downloaded.
+-#  Note that with atomic MD, we can't have old MD lying around anymore so
+-# the only way we need this check is if someone does something like:
+-#   cp primary.sqlite /var/cache/yum/blah
+-# ...at which point you lose.
+-skip_old_DBMD_check = True
++# This is unused now, probably nothing uses it but it was global/public.
++skip_old_DBMD_check = False
++
++try:
++    import xattr
++    if not hasattr(xattr, 'get') or not hasattr(xattr, 'set'):
++        xattr = None # This is a "newer" API.
++except ImportError:
++    xattr = None
++
++#  The problem we are trying to solve here is that:
++#
++# 1. We rarely want to be downloading MD/pkgs/etc.
++# 2. We want to check those files are valid (match checksums) when we do
++#    download them.
++# 3. We _really_ don't want to checksum all the files everytime we
++#    run (100s of MBs).
++# 4. We can continue to download files from bad mirrors, or retry files due to
++#    C-c etc.
++#
++# ...we used to solve this by just checking the file size, and assuming the
++# files had been downloaded and checksumed as correct if that matched. But that
++# was error prone on bad mirrors, so now we store the checksum in an
++# xattr ... this does mean that if you can't store xattrs (Eg. NFS) you will
++# rechecksum everything constantly.
++
++def _xattr_get_chksum(filename, chktype):
++    if not xattr:
++        return None
++
++    try:
++        ret = xattr.get(filename, 'user.yum.checksum.' + chktype)
++    except: # Documented to be "EnvironmentError", but make sure
++        return None
++
++    return ret
++
++def _xattr_set_chksum(filename, chktype, chksum):
++    if not xattr:
++        return None
++
++    try:
++        xattr.set(filename, 'user.yum.checksum.' + chktype, chksum)
++    except:
++        return False # Data too long. = IOError ... ignore everything.
++
++    return True
++
+ 
+ warnings.simplefilter("ignore", Errors.YumFutureDeprecationWarning)
+ 
+@@ -228,7 +267,7 @@ class YumPackageSack(packageSack.PackageSack):
+         # get rid of all this stuff we don't need now
+         del repo.cacheHandler
+ 
+-    def _check_uncompressed_db_gen(self, repo, mdtype, fast=True):
++    def _check_uncompressed_db_gen(self, repo, mdtype):
+         """return file name of db in gen/ dir if good, None if not"""
+ 
+         mydbdata         = repo.repoXML.getData(mdtype)
+@@ -238,7 +277,7 @@ class YumPackageSack(packageSack.PackageSack):
+         db_un_fn         = mdtype + '.sqlite'
+ 
+         if not repo._checkMD(compressed_fn, mdtype, data=mydbdata,
+-                             check_can_fail=fast, fast=fast):
++                             check_can_fail=True):
+             return None
+ 
+         ret = misc.repo_gen_decompress(compressed_fn, db_un_fn,
+@@ -261,8 +300,7 @@ class YumPackageSack(packageSack.PackageSack):
+         result = None
+ 
+         if os.path.exists(db_un_fn):
+-            if skip_old_DBMD_check and repo._using_old_MD:
+-                return db_un_fn
++
+ 
+             try:
+                 repo.checkMD(db_un_fn, mdtype, openchecksum=True)
+@@ -296,7 +334,6 @@ class YumRepository(Repository, config.RepoConf):
+                                               # eventually want
+         self.repoMDFile = 'repodata/repomd.xml'
+         self._repoXML = None
+-        self._using_old_MD = None
+         self._oldRepoMDData = {}
+         self.cache = 0
+         self.mirrorlistparsed = 0
+@@ -1407,7 +1444,6 @@ Insufficient space in download directory %s
+             self._revertOldRepoXML()
+             return False
+ 
+-        self._using_old_MD = caching
+         if caching:
+             return False # Skip any work.
+ 
+@@ -1673,7 +1709,7 @@ Insufficient space in download directory %s
+         return self._checkMD(fn, mdtype, openchecksum)
+ 
+     def _checkMD(self, fn, mdtype, openchecksum=False,
+-                 data=None, check_can_fail=False, fast=False):
++                 data=None, check_can_fail=False):
+         """ Internal function, use .checkMD() from outside yum. """
+ 
+         thisdata = data # So the argument name is nicer
+@@ -1696,17 +1732,15 @@ Insufficient space in download directory %s
+         if size is not None:
+             size = int(size)
+ 
+-        if fast and skip_old_DBMD_check:
++        l_csum = _xattr_get_chksum(file, r_ctype)
++        if l_csum:
+             fsize = misc.stat_f(file)
+-            if fsize is None: # File doesn't exist...
+-                return None
+-            if size is None:
+-                return 1
+-            if size == fsize.st_size:
+-                return 1
+-            if check_can_fail:
+-                return None
+-            raise URLGrabError(-1, 'Metadata file does not match size')
++            if fsize is not None: # We just got an xattr, so it should be there
++                if size is None and l_csum == r_csum:
++                    return 1
++                if size == fsize.st_size and l_csum == r_csum:
++                    return 1
++            # Anything goes wrong, run the checksums as normal...
+ 
+         try: # get the local checksum
+             l_csum = self._checksum(r_ctype, file, datasize=size)
+@@ -1716,6 +1750,7 @@ Insufficient space in download directory %s
+             raise URLGrabError(-3, 'Error performing checksum')
+ 
+         if l_csum == r_csum:
++            _xattr_set_chksum(file, r_ctype, l_csum)
+             return 1
+         else:
+             if check_can_fail:

arm-arch-detection.patch

@@ -1,104 +0,0 @@
-commit 24dde23643f98ed355fcf50fb09807e2b08ea620
-Author: Phil Knirsch <pknirsch@redhat.com>
-Date:   Fri Apr 27 09:09:15 2012 -0400
-
-     Do arm arch detection in the same way rpm does it. Once we have the rpm API
-    in place we can switch over to that easily.
-    
-     Minor cleanups by James Antill.
-
-diff --git a/rpmUtils/arch.py b/rpmUtils/arch.py
-index 7d67907..2226dc5 100644
---- a/rpmUtils/arch.py
-+++ b/rpmUtils/arch.py
-@@ -3,6 +3,8 @@
- 
- import os
- import rpm
-+import ctypes
-+import struct
- 
- _ppc64_native_is_best = True
- 
-@@ -31,6 +33,7 @@ arches = {
-     "ia32e": "x86_64",
-     
-     # ppc
-+    "ppc64p7": "ppc64",
-     "ppc64pseries": "ppc64",
-     "ppc64iseries": "ppc64",    
-     "ppc64": "ppc",
-@@ -82,6 +85,13 @@ arches = {
-     "ia64": "noarch",
-     }
- 
-+#  Will contain information parsed from /proc/self/auxv via _parse_auxv().
-+# Should move into rpm really.
-+_aux_vector = {
-+    "platform": "",
-+    "hwcap": 0,
-+    }
-+
- def legitMultiArchesInSameLib(arch=None):
-     # this is completely crackrock - if anyone has a better way I
-     # am all ears
-@@ -222,6 +232,32 @@ def _try_read_cpuinfo():
-     except:
-         return []
- 
-+def _parse_auxv():
-+    """ Read /proc/self/auxv and parse it into global dict for easier access
-+        later on, very similar to what rpm does. """
-+    # In case we can't open and read /proc/self/auxv, just return
-+    try:
-+        data = open("/proc/self/auxv", "rb").read()
-+    except:
-+        return
-+
-+    # Define values from /usr/include/elf.h
-+    AT_PLATFORM = 15
-+    AT_HWCAP = 16
-+    fmtlen = struct.calcsize("LL")
-+    offset = 0
-+    platform = ctypes.c_char_p()
-+
-+    # Parse the data and fill in _aux_vector dict
-+    while offset <= len(data) - fmtlen:
-+        at_type, at_val = struct.unpack_from("LL", data, offset)
-+        if at_type == AT_PLATFORM:
-+            platform.value = at_val
-+            _aux_vector["platform"] = platform.value
-+        if at_type == AT_HWCAP:
-+            _aux_vector["hwcap"] = at_val
-+        offset = offset + fmtlen
-+
- def getCanonX86Arch(arch):
-     # 
-     if arch == "i586":
-@@ -260,6 +296,17 @@ def getCanonPPCArch(arch):
-         if line.find("machine") != -1:
-             machine = line.split(':')[1]
-             break
-+
-+    platform = _aux_vector["platform"]
-+    if machine is None and not platform:
-+        return arch
-+
-+    try:
-+        if platform.startswith("power") and int(platform[5:]) >= 7:
-+            return "ppc64p7"
-+    except:
-+        pass
-+
-     if machine is None:
-         return arch
- 
-@@ -324,6 +371,8 @@ def getCanonArch(skipRpmPlatform = 0):
-         
-     arch = os.uname()[4]
- 
-+    _parse_auxv()
-+
-     if (len(arch) == 4 and arch[0] == "i" and arch[2:4] == "86"):
-         return getCanonX86Arch(arch)
- 

ui_id-prevent-TB-on-invalid-repos-with-no-URLs.patch

@@ -0,0 +1,28 @@
+From 40d9d5f4c3a00b2efa6ee0b64fe4d1ec52d46a69 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Zden=C4=9Bk=20Pavlas?= <zpavlas@redhat.com>
+Date: Mon, 29 Oct 2012 09:22:02 +0100
+Subject: [PATCH] ui_id: prevent TB on invalid repos with no URLs.  BZ 870691.
+
+---
+ yum/yumRepo.py |    4 +++-
+ 1 files changed, 3 insertions(+), 1 deletions(-)
+
+diff --git a/yum/yumRepo.py b/yum/yumRepo.py
+index e362c43..414f1d9 100644
+--- a/yum/yumRepo.py
++++ b/yum/yumRepo.py
+@@ -381,8 +381,10 @@ class YumRepository(Repository, config.RepoConf):
+                 val = ini['metalink']
+             elif 'mirrorlist' in ini:
+                 val = ini['mirrorlist']
+-            else:
++            elif 'baseurl' in ini:
+                 val = ini['baseurl']
++            else:
++                val = ''
+         ret = self.id
+         if '$releasever' in val:
+             ret += '/'
+-- 
+1.7.4.4
+

yum-completion-helper.patch

@@ -0,0 +1,11 @@
+diff -up yum-3.4.3/completion-helper.py.old yum-3.4.3/completion-helper.py
+--- yum-3.4.3/completion-helper.py.old	2012-08-22 17:00:47.444104234 +0200
++++ yum-3.4.3/completion-helper.py	2012-08-22 17:00:57.954647525 +0200
+@@ -70,6 +70,7 @@ def get_pattern(extcmds):
+ 
+ def main(args):
+     base = cli.YumBaseCli()
++    base.setCacheDir = lambda *x: True # use the system cachedir
+     base.yum_cli_commands.clear()
+     base.registerCommand(GroupsCompletionCommand())
+     base.registerCommand(ListCompletionCommand())

yum-ppc64-preferred.patch

@@ -2,8 +2,8 @@ diff -ru yum-3.4.3-orig/rpmUtils/arch.py yum-3.4.3/rpmUtils/arch.py
 --- yum-3.4.3-orig/rpmUtils/arch.py	2011-06-28 17:01:10.009680846 -0400
 +++ yum-3.4.3/rpmUtils/arch.py	2011-06-28 17:01:31.849916539 -0400
 @@ -3,7 +3,7 @@
- import os
- import rpm
+ import ctypes
+ import struct
  
 -_ppc64_native_is_best = False
 +_ppc64_native_is_best = True

yum.spec

@@ -18,7 +18,7 @@
 Summary: RPM package installer/updater/manager
 Name: yum
 Version: 3.4.3
-Release: 24%{?dist}
+Release: 31%{?dist}
 License: GPLv2+
 Group: System Environment/Base
 Source0: http://yum.baseurl.org/download/3.4/%{name}-%{version}.tar.gz
@@ -30,8 +30,10 @@ Patch5: geode-arch.patch
 Patch6: yum-HEAD.patch
 Patch7: yum-ppc64-preferred.patch
 Patch8: BZ-803346-no-only-update.patch
-Patch9: arm-arch-detection.patch
+Patch9: BZ-908870-MD-files-bad.patch
 Patch20: yum-manpage-files.patch
+Patch21: yum-completion-helper.patch
+Patch22: ui_id-prevent-TB-on-invalid-repos-with-no-URLs.patch
 
 URL: http://yum.baseurl.org/
 BuildArchitectures: noarch
@@ -56,6 +58,7 @@ Requires: python-sqlite
 Requires: python-urlgrabber >= 3.9.0-8
 Requires: yum-metadata-parser >= 1.1.0
 Requires: pygpgme
+Requires: pyxattr
 
 Conflicts: rpm >= 5-0
 # Zif is a re-implementation of yum in C, however:
@@ -139,9 +142,11 @@ Install this package if you want auto yum updates nightly via cron.
 %patch6 -p1
 %patch7 -p1
 %patch8 -p1
-%patch9 -p1
 %patch20 -p1
+%patch21 -p1
+%patch22 -p1
 %patch1 -p1
+%patch9 -p1
 
 %build
 make
@@ -294,7 +299,7 @@ exit 0
 %files cron
 %defattr(-,root,root)
 %doc COPYING
-%config(noreplace) %{_sysconfdir}/cron.daily/yum-update.cron
+%config(noreplace) %{_sysconfdir}/cron.daily/0yum-update.cron
 %config(noreplace) %{_sysconfdir}/cron.daily/yum-cleanup.cron
 %{_sysconfdir}/rc.d/init.d/yum-cron
 %{_sbindir}/yum-cron
@@ -315,6 +320,45 @@ exit 0
 %endif
 
 %changelog
+* Tue Feb 19 2013 James Antill <james at fedoraproject.org> - 3.4.3-32
+- Fix non-removal of corrupt metadata files. BZ 908870.
+
+* Mon Jan 7 2013 Zdenek Pavlas <zpavlas at redhat.com> - 3.4.3-31
+- ui_id: prevent TB on invalid repos with no URLs.  BZ 870691.
+
+* Mon Nov 12 2012 Zdenek Pavlas <zpavlas at redhat.com> - 3.4.3-30
+- update to HEAD as of Oct 16 2012 (just before --downloadonly
+  and yum-cron changes)
+- Fixes BZ 832166, 826419, 854974, 858632, 856969, 861264, 864018, 864294,
+  864643, 864717, 864717, 859202.
+
+* Tue Aug 28 2012 Zdenek Pavlas <zpavlas at redhat.com> - 3.4.3-29
+- update to latest HEAD.
+- new groups code
+- lots of bugfixes
+
+* Mon Jun 25 2012 Zdenek Pavlas <zpavlas at redhat.com> - 3.4.3-28
+- update to latest HEAD.
+- No async downloading when --cacheonly.  BZ 830523.
+- compare mtime without sub second precision. BZ 831918
+- show_lock_owner: report errors if we fail.  BZ 745281
+- quote uids to keep cachedir ascii-clean.  BZ 832195
+- A solution to the obsoletes but don't provide problem. BZ 834530
+- completion-helper: use the system cachedir
+
+* Fri Jun  8 2012 James Antill <james at fedoraproject.org> - 3.4.3-27
+- update to latest HEAD.
+- Fix for ppc64p7 detection.
+
+* Thu Jun  7 2012 Zdenek Pavlas <zpavlas at redhat.com> - 3.4.3-26
+- update to latest HEAD
+- more completion helper patches
+- parallel downloading of packages and metadata
+- revert a hack that probably caused BZ 829505
+
+* Thu May 31 2012 Zdenek Pavlas <zpavlas at redhat.com> - 3.4.3-25
+- backported completion-helper.py patches from HEAD, BZ 809469.
+
 * Fri Apr 27 2012 James Antill <james at fedoraproject.org> - 3.4.3-24
 - Add code for arm detection.