Migrate STI to tmt

Resolves: FC-1842

.fmf/version

@@ -0,0 +1 @@
+1

.gitignore

@@ -3,3 +3,7 @@ noarch
 clog
 /.*build.log
 /ca-certificates
+certdata.txt.orig
+codesign-release.txt
+microsoft_sign_obj_ca.pem
+

README.edk2

@@ -0,0 +1,13 @@
+This directory /etc/pki/ca-trust/extracted/edk2/ contains a
+CA certificate bundle file which is automatically created
+based on the information found in the
+/usr/share/pki/ca-trust-source/ and /etc/pki/ca-trust/source/
+directories.
+
+The file is in the EDK2 (EFI Development Kit II) file format.
+
+Please never manually edit the files stored in this directory,
+because your changes will be lost and the files automatically overwritten,
+each time the update-ca-trust command gets executed.
+
+Please refer to the update-ca-trust(8) manual page for additional information.

README.etcssl

@@ -0,0 +1,20 @@
+This directory (/etc/ssl) is provided as a courtesy attempt to provide
+compatibility with software which assumes its existence. It is not a
+supported or canonical location. Software which assumes and relies on
+the existence and layout of this directory is making a wrong assumption
+(this directory is not any kind of 'standard', it is a configuration
+detail of Debian and its derivatives) and should be improved. No
+software packaged in this distribution should use this directory.
+
+An attempt is made to make the layout of /etc/ssl/certs match that
+provided by Debian: it is an OpenSSL 'CApath'-style hashed directory
+of individual certificate files, and also contains a certificate bundle
+file named ca-certificates.crt, as Debian does. It also contains a
+bundle named ca-bundle.crt, as this distribution has long provided
+such a file, and it is possible some software has come to expect its
+existence.
+
+/etc/ssl/certs itself and the bundle files are in fact symlinks to
+some of the output of the 'update-ca-trust' script which forms a part
+of a system of consolidated CA certificates. Please refer to the
+update-ca-trust(8) manual page for additional information.

README.openssl

@@ -1,18 +0,0 @@
-This directory /etc/pki/ca-trust/extracted/openssl/ contains 
-CA certificate bundle files which are automatically created
-based on the information found in the
-/usr/share/pki/ca-trust-source/ and /etc/pki/ca-trust/source/
-directories.
-
-All files are in the BEGIN/END TRUSTED CERTIFICATE file format, 
-as described in the x509(1) manual page.
-
-If your application isn't able to load the PKCS#11 module p11-kit-trust.so,
-then you can use these files in your application to load a list of global
-root CA certificates.
-
-Please never manually edit the files stored in this directory,
-because your changes will be lost and the files automatically overwritten,
-each time the update-ca-trust command gets executed.
-
-Please refer to the update-ca-trust(8) manual page for additional information.

certdata2pem.py

@@ -26,17 +26,17 @@ import os.path
 import re
 import sys
 import textwrap
-import urllib
+import urllib.request, urllib.parse, urllib.error
 import subprocess
 
 objects = []
 
 def printable_serial(obj):
-  return ".".join(map(lambda x:str(ord(x)), obj['CKA_SERIAL_NUMBER']))
+  return ".".join([str(x) for x in obj['CKA_SERIAL_NUMBER']])
 
 # Dirty file parser.
 in_data, in_multiline, in_obj = False, False, False
-field, type, value, obj = None, None, None, dict()
+field, ftype, value, binval, obj = None, None, None, bytearray(), dict()
 for line in open('certdata.txt', 'r'):
     # Ignore the file header.
     if not in_data:
@@ -56,33 +56,36 @@ for line in open('certdata.txt', 'r'):
         continue
     if in_multiline:
         if not line.startswith('END'):
-            if type == 'MULTILINE_OCTAL':
+            if ftype == 'MULTILINE_OCTAL':
                 line = line.strip()
                 for i in re.finditer(r'\\([0-3][0-7][0-7])', line):
-                    value += chr(int(i.group(1), 8))
+                    integ = int(i.group(1), 8)
+                    binval.extend((integ).to_bytes(1, sys.byteorder))
+                obj[field] = binval
             else:
                 value += line
+                obj[field] = value
             continue
-        obj[field] = value
         in_multiline = False
         continue
     if line.startswith('CKA_CLASS'):
         in_obj = True
     line_parts = line.strip().split(' ', 2)
     if len(line_parts) > 2:
-        field, type = line_parts[0:2]
+        field, ftype = line_parts[0:2]
         value = ' '.join(line_parts[2:])
     elif len(line_parts) == 2:
-        field, type = line_parts
+        field, ftype = line_parts
         value = None
     else:
-        raise NotImplementedError, 'line_parts < 2 not supported.\n' + line
-    if type == 'MULTILINE_OCTAL':
+        raise NotImplementedError('line_parts < 2 not supported.\n' + line)
+    if ftype == 'MULTILINE_OCTAL':
         in_multiline = True
         value = ""
+        binval = bytearray()
         continue
     obj[field] = value
-if len(obj.items()) > 0:
+if len(list(obj.items())) > 0:
     objects.append(obj)
 
 # Build up trust database.
@@ -92,7 +95,7 @@ for obj in objects:
         continue
     key = obj['CKA_LABEL'] + printable_serial(obj)
     trustmap[key] = obj
-    print " added trust", key
+    print(" added trust", key)
 
 # Build up cert database.
 certmap = dict()
@@ -101,7 +104,7 @@ for obj in objects:
         continue
     key = obj['CKA_LABEL'] + printable_serial(obj)
     certmap[key] = obj
-    print " added cert", key
+    print(" added cert", key)
 
 def obj_to_filename(obj):
     label = obj['CKA_LABEL'][1:-1]
@@ -110,7 +113,18 @@ def obj_to_filename(obj):
         .replace('(', '=')\
         .replace(')', '=')\
         .replace(',', '_')
-    label = re.sub(r'\\x[0-9a-fA-F]{2}', lambda m:chr(int(m.group(0)[2:], 16)), label)
+    labelbytes = bytearray()
+    i = 0
+    imax = len(label)
+    while i < imax:
+        if i < imax-3 and label[i] == '\\' and label[i+1] == 'x':
+            labelbytes.extend(bytes.fromhex(label[i+2:i+4]))
+            i += 4
+            continue
+        labelbytes.extend(str.encode(label[i]))
+        i = i+1
+        continue
+    label = labelbytes.decode('utf-8')
     serial = printable_serial(obj)
     return label + ":" + serial
 
@@ -163,34 +177,39 @@ openssl_trust = {
   "CKA_TRUST_EMAIL_PROTECTION": "emailProtection",
 }
 
+cert_distrust_types = {
+  "CKA_NSS_SERVER_DISTRUST_AFTER": "nss-server-distrust-after",
+  "CKA_NSS_EMAIL_DISTRUST_AFTER": "nss-email-distrust-after",
+}
+
 for tobj in objects:
     if tobj['CKA_CLASS'] == 'CKO_NSS_TRUST':
         key = tobj['CKA_LABEL'] + printable_serial(tobj)
-        print "producing trust for " + key
+        print("producing trust for " + key)
         trustbits = []
         distrustbits = []
         openssl_trustflags = []
         openssl_distrustflags = []
         legacy_trustbits = []
         legacy_openssl_trustflags = []
-        for t in trust_types.keys():
-            if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
+        for t in list(trust_types.keys()):
+            if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
                 trustbits.append(t)
                 if t in openssl_trust:
                     openssl_trustflags.append(openssl_trust[t])
-            if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
+            if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
                 distrustbits.append(t)
                 if t in openssl_trust:
                     openssl_distrustflags.append(openssl_trust[t])
 
-        for t in legacy_trust_types.keys():
-            if tobj.has_key(t) and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
+        for t in list(legacy_trust_types.keys()):
+            if t in tobj and tobj[t] == 'CKT_NSS_TRUSTED_DELEGATOR':
                 real_t = legacy_to_real_trust_types[t]
                 legacy_trustbits.append(real_t)
                 if real_t in openssl_trust:
                     legacy_openssl_trustflags.append(openssl_trust[real_t])
-            if tobj.has_key(t) and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
-                raise NotImplementedError, 'legacy distrust not supported.\n' + line
+            if t in tobj and tobj[t] == 'CKT_NSS_NOT_TRUSTED':
+                raise NotImplementedError('legacy distrust not supported.\n' + line)
 
         fname = obj_to_filename(tobj)
         try:
@@ -206,10 +225,10 @@ for tobj in objects:
         #dumpf.close();
 
         is_legacy = 0
-        if tobj.has_key('LEGACY_CKA_TRUST_SERVER_AUTH') or tobj.has_key('LEGACY_CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('LEGACY_CKA_TRUST_CODE_SIGNING'):
+        if 'LEGACY_CKA_TRUST_SERVER_AUTH' in tobj or 'LEGACY_CKA_TRUST_EMAIL_PROTECTION' in tobj or 'LEGACY_CKA_TRUST_CODE_SIGNING' in tobj:
             is_legacy = 1
             if obj == None:
-                raise NotImplementedError, 'found legacy trust without certificate.\n' + line
+                raise NotImplementedError('found legacy trust without certificate.\n' + line)
 
             legacy_fname = "legacy-default/" + fname + ".crt"
             f = open(legacy_fname, 'w')
@@ -218,11 +237,13 @@ for tobj in objects:
             if legacy_openssl_trustflags:
                 f.write("# openssl-trust=" + " ".join(legacy_openssl_trustflags) + "\n")
             f.write("-----BEGIN CERTIFICATE-----\n")
-            f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
+            temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
+            temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
+            f.write("\n".join(temp_wrapped))
             f.write("\n-----END CERTIFICATE-----\n")
             f.close()
 
-            if tobj.has_key('CKA_TRUST_SERVER_AUTH') or tobj.has_key('CKA_TRUST_EMAIL_PROTECTION') or tobj.has_key('CKA_TRUST_CODE_SIGNING'):
+            if 'CKA_TRUST_SERVER_AUTH' in tobj or 'CKA_TRUST_EMAIL_PROTECTION' in tobj or 'CKA_TRUST_CODE_SIGNING' in tobj:
                 legacy_fname = "legacy-disable/" + fname + ".crt"
                 f = open(legacy_fname, 'w')
                 f.write("# alias=%s\n"%tobj['CKA_LABEL'])
@@ -244,7 +265,9 @@ for tobj in objects:
             cert_fname = "cert-" + fname
             fc = open(cert_fname, 'w')
             fc.write("-----BEGIN CERTIFICATE-----\n")
-            fc.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
+            temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
+            temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
+            fc.write("\n".join(temp_wrapped))
             fc.write("\n-----END CERTIFICATE-----\n")
             fc.close();
             pk_fname = "pubkey-" + fname
@@ -262,7 +285,7 @@ for tobj in objects:
             fcout.close()
             sed_command = ["sed", "--in-place", "s/^/#/", comment_fname]
             subprocess.call(sed_command)
-            with open (comment_fname, "r") as myfile:
+            with open (comment_fname, "r", errors = 'replace') as myfile:
                 cert_comment=myfile.read()
 
         fname += ".tmp-p11-kit"
@@ -274,19 +297,19 @@ for tobj in objects:
             has_email_trust = False
             has_code_trust = False
 
-            if tobj.has_key('CKA_TRUST_SERVER_AUTH'):
+            if 'CKA_TRUST_SERVER_AUTH' in tobj:
                 if tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED':
                     is_distrusted = True
                 elif tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_TRUSTED_DELEGATOR':
                     has_server_trust = True
 
-            if tobj.has_key('CKA_TRUST_EMAIL_PROTECTION'):
+            if 'CKA_TRUST_EMAIL_PROTECTION' in tobj:
                 if tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED':
                     is_distrusted = True
                 elif tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_TRUSTED_DELEGATOR':
                     has_email_trust = True
 
-            if tobj.has_key('CKA_TRUST_CODE_SIGNING'):
+            if 'CKA_TRUST_CODE_SIGNING' in tobj:
                 if tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED':
                     is_distrusted = True
                 elif tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_TRUSTED_DELEGATOR':
@@ -351,8 +374,20 @@ for tobj in objects:
             f.write("nss-mozilla-ca-policy: true\n")
             f.write("modifiable: false\n");
 
+            # requires p11-kit >= 0.23.19
+            for t in list(cert_distrust_types.keys()):
+                if t in obj:
+                    value = obj[t]
+                    if value == 'CK_FALSE':
+                        value = bytearray(1)
+                    f.write(cert_distrust_types[t] + ": \"")
+                    f.write(urllib.parse.quote(value));
+                    f.write("\"\n")
+
             f.write("-----BEGIN CERTIFICATE-----\n")
-            f.write("\n".join(textwrap.wrap(base64.b64encode(obj['CKA_VALUE']), 64)))
+            temp_encoded_b64 = base64.b64encode(obj['CKA_VALUE'])
+            temp_wrapped = textwrap.wrap(temp_encoded_b64.decode(), 64)
+            f.write("\n".join(temp_wrapped))
             f.write("\n-----END CERTIFICATE-----\n")
             f.write(cert_comment)
             f.write("\n")
@@ -366,13 +401,13 @@ for tobj in objects:
             f.write("certificate-type: x-509\n")
             f.write("modifiable: false\n");
             f.write("issuer: \"");
-            f.write(urllib.quote(tobj['CKA_ISSUER']));
+            f.write(urllib.parse.quote(tobj['CKA_ISSUER']));
             f.write("\"\n")
             f.write("serial-number: \"");
-            f.write(urllib.quote(tobj['CKA_SERIAL_NUMBER']));
+            f.write(urllib.parse.quote(tobj['CKA_SERIAL_NUMBER']));
             f.write("\"\n")
             if (tobj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED') or (tobj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED'):
               f.write("x-distrusted: true\n")
             f.write("\n\n")
         f.close()
-        print " -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags)
+        print(" -> written as '%s', trust = %s, openssl-trust = %s, distrust = %s, openssl-distrust = %s" % (fname, trustbits, openssl_trustflags, distrustbits, openssl_distrustflags))

check_certs.sh

@@ -0,0 +1,118 @@
+#!/bin/perl
+
+sub adjust {
+   my $newLine = $_[0];
+   my @neg = @{$_[1]};
+   my @pos = @{$_[2]};
+   my $found = 0;
+   my @newneg = ();
+
+   foreach my $cline (@neg) {
+	if ($cline eq $newLine) {
+	   $found = 1;
+	} else {
+            push(@newneg ,$cline );
+	}
+   }
+   if (! $found ) {
+ 	push(@pos, $newLine);
+   }
+   @neg=@newneg;
+}
+
+sub removeLine {
+   my $newLine = $_[0];
+   my @neg = @{$_[1]};
+   my $found = 0;
+   my @newneg = ();
+
+   foreach my $cline (@neg) {
+        if ($found) {
+            push(@newneg ,$cline );
+	} elsif ($cline eq $newLine) {
+	   $found = 1;
+	} else {
+            push(@newneg ,$cline );
+	}
+   }
+   return @newneg;
+}
+
+sub filter {
+   my @list = @{$_[0]};
+   my $string = $_[1];
+   my @filteredList = ();
+   foreach my $cline (@list) {
+	if ($cline =~ m/$string/) {
+            push(@filteredList ,$cline );
+	}
+   }
+   return @filteredList;
+}
+
+sub lineExists {
+   my $newLine = $_[0];
+   my @neg = @{$_[1]};
+
+   foreach my $cline (@neg) {
+	if ($cline eq $newLine) {
+           return 1;
+	}
+   }
+   return 0;
+}
+
+sub lineExists {
+   my $newLine = $_[0];
+   my @neg = @{$_[1]};
+
+   foreach my $cline (@neg) {
+	if ($cline eq $newLine) {
+           return 1;
+	}
+   }
+   return 0;
+}
+
+sub printeach {
+   my @args = @{$_[0]};
+   foreach my $arg (@args) {
+	chomp $arg;
+	print "    $arg\n";
+    }
+}
+
+open my $handle, "git diff certdata.txt|";
+my @diff_lines = <$handle>;
+close $handle;
+my @adds = ();
+my @subs = ();
+foreach my $line (@diff_lines) {
+    $type = substr $line,0,1;
+    $lline = substr $line,1;
+    if ($type eq "+") {
+          if (lineExists($lline, \@subs)) {
+               @subs = removeLine($lline,\@subs);
+          } else {
+               push(@adds, $lline);
+          }
+    };
+    if ($type eq "-") {
+          if (lineExists($lline, \@adds)) {
+                @adds = removeLine($lline,\@adds);
+          } else {
+                push(@subs, $lline);
+          }
+    };
+}
+
+my @tmp = filter(\@subs, "# Certificate");
+if (@tmp) {
+    print "   Removing: \n";
+    printeach(\@tmp);
+}
+my @tmp = filter(\@adds, "# Certificate");
+if (@tmp) {
+    print "   Adding: \n";
+    printeach(\@tmp);
+}

fetch.sh

@@ -1,4 +1,185 @@
 #!/bin/sh
-exec cvs -d :pserver:anonymous@cvs-mirror.mozilla.org:/cvsroot   \
-     co -p mozilla/security/nss/lib/ckfw/builtins/certdata.txt   \
-     > certdata.txt
+#
+# This script fetches the latest released certdata.txt and updates the 
+# ca-certificates.spec file
+#
+baseurl="https://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib"
+force=0
+skip_signed_obj=0
+release_type="RTM"
+release="3_65"
+while [ -n "$1" ]; do
+   case $1 in
+   "-d")
+	baseurl="https://hg.mozilla.org/projects/nss/raw-file/default/lib"
+	;;
+   -t*)
+	release_type=`echo $1 | sed -e 's;-t;;'`
+	if [ "${release_type}" = "" ]; then
+	   shift
+	   release_type=$1
+	fi
+	baseurl="https://hg.mozilla.org/projects/nss/raw-file/NSS_${release}_${release_type}/lib"
+	;;
+   -n*)
+	release=`echo $1 | sed -e 's;-n;;'`
+	if [ "${release}" = "" ]; then
+	   shift
+	   release=$1
+	fi
+	release=`echo ${release} | sed -e 's;\\.;_;g'`
+	baseurl="https://hg.mozilla.org/projects/nss/raw-file/NSS_${release}_${release_type}/lib"
+	;;
+   "-f")
+	force=1
+	;;
+   "-s")
+        skip_signed_obj=1
+        ;;
+    *)
+	echo "usage: $0 [-r] [-n release] [-f]"
+	echo "-d           use the development tip rather than the latest release"
+	echo "-n release   fetch a specific nss release"
+	echo "-f           skip the verify check"
+	echo "-s           skip fetching signed objects"
+	exit 1
+	;;
+    esac
+    shift
+done
+
+# get the current certdata version number
+# nss version number
+# user making the change
+# email of user
+# 
+# versions from the latest nss code in mozilla
+echo "Getting CKBI version number"
+ckbi_version=`wget ${baseurl}/ckfw/builtins/nssckbi.h -O - | grep "NSS_BUILTINS_LIBRARY_VERSION " | awk '{print $NF}' | sed -e "s;\";;g" `
+if [ "${ckbi_version}" = "" ]; then
+    echo "Didn't find ckbi version from ${baseurl}"
+    exit 1;
+fi
+echo "Getting NSS version number"
+nss_version=`wget ${baseurl}/nss/nss.h -O - | grep "NSS_VERSION" | awk '{print $3}' | sed -e "s;\";;g" `
+if [ "${nss_version}" = "" ]; then
+    echo "Didn't find nss version from ${baseurl}"
+    exit 1;
+fi
+# date from the current system date on this machine
+echo "Creating change log"
+export LANG=C
+year=`date +%Y`
+log_date=`date +"%a %b %d %Y"`
+# user name from the environment, fallback to git, fallback to the current user
+username=`whoami`
+name=${NAME}
+if [ "${name}" = "" ]; then
+   name=`git config user.name`
+fi
+if [ "${name}" = "" ]; then
+   name=`getent passwd $username`
+fi
+email=${EMAIL}
+if [ "${email}" = "" ]; then
+   email=`git config user.email`
+fi
+if [ "${email}" = "" ]; then
+   email=$username@`hostname`
+fi
+# rawhide >=2, branches 1.x
+cwd=$(pwd)
+if [ `basename ${cwd}` = rawhide ]; then
+    release="2"
+else
+    release="1.0"
+fi
+
+
+# fetch the codesigning certs now so we can get
+# the code signing version number
+if [ ${skip_signed_obj} -eq 0 ]; then
+   ./fetch_objsign.sh
+   if [ -f codesign-release.txt ]; then
+      mcs_version=$(cat codesign-release.txt)
+      if [[ $ms_version != "unknown" ]]; then
+          ckbi_version="${ckbi_version}_${mcs_version}"
+      fi
+      signobjects="and Microsoft Signed Objects version $ms_version"
+   fi
+fi
+
+version=${year}.${ckbi_version}
+
+#make sure the the current version is newer than what is already there
+current_version=`grep ^Version: ca-certificates.spec | awk '{ print $NF }'`
+if [ ${current_version} \> ${version} -o ${current_version} = ${version} ]; then
+   echo "Can't downgrade current version: ${current_version} new version: ${version}"
+   exit 1;
+fi
+
+# now get our new certdata.txt
+echo "Fetching new certdata.txt"
+wget ${baseurl}/ckfw/builtins/certdata.txt -O certdata.txt
+if [ $? -ne 0 ]; then
+   echo fetching certdata.text from ${baseurl} failed!
+   echo " To restore the old certdata.txt use:"
+   echo "    git checkout -- certdata.txt"
+   exit 1;
+fi
+
+# merge the signing certs into the normal certdata.txt file.
+if [ ${skip_signed_obj} -eq 0 ]; then
+   cp certdata.txt certdata.txt.orig
+   python3 ./mergepem2certdata.py -c "certdata.txt.orig" -p "microsoft_sign_obj_ca.pem" -o "certdata.txt" -t "CKA_TRUST_CODE_SIGNING" -l "Microsoft Code Signing Only Certificate" -x "NEVER"
+fi
+
+# Verify everything is good with the user
+echo -e "Upgrading ${current_version} -> ${version}:"
+echo -e "*${log_date} ${name} <$email> ${version}-${release}\n - Update to CKBI ${ckbi_version} from NSS ${nss_version}${sign_objects}"
+./check_certs.sh
+echo ""
+
+yn=""
+if [ ! ${force} ]; then
+	echo -n "Do you want to continue (Y/N default Y)? "
+	read yn
+	echo ""
+fi
+if [ "${yn}" != "" -a "${yn}" != "y" -a "${yn}" != "Y" -a "${yn}" != "yes" -a "${yn}" != "YES" ]; then
+    echo "Skipping ca-certificate.spec upgrade."
+    echo " NOTE: certdata.txt has been upgraded."
+    echo " To restore the old certdata.txt use:"
+    echo "    git checkout -- certdata.txt"
+    exit 1;
+fi
+
+echo "Updating .spec file"
+cat ca-certificates.spec | while IFS= read -r line
+do
+    echo $line | grep "^Version: " 1>&2
+    if [ $? -eq 0 ]; then
+	echo "Version: ${version}"
+	echo "New Version: ${version}" 1>&2
+	continue
+    fi
+    echo $line | grep "^Release: " 1>&2
+    if [ $?  -eq 0 ]; then
+	echo "Release: ${release}%{?dist}"
+	echo "New Release: ${release}%{?dist}" 1>&2
+	continue
+    fi
+    echo $line | grep "^%changelog" 1>&2
+    if [ $?  -eq 0 ]; then
+	echo "$line"
+	echo -e "*${log_date} ${name} <$email> ${version}-${release}\n - Update to CKBI ${ckbi_version} from NSS ${nss_version}"
+	echo -e "*${log_date} ${name} <$email> ${version}-${release}\n - Update to CKBI ${ckbi_version} from NSS ${nss_version}"  1>&2
+	./check_certs.sh
+       echo ""
+	continue
+    fi
+    echo "$line"
+done > /tmp/ca-certificates.spec.$$
+mv /tmp/ca-certificates.spec.$$ ca-certificates.spec
+git status
+exit 0

fetch_objsign.sh

@@ -0,0 +1,123 @@
+#!/bin/sh
+#
+# This script fetches the object signing list from the Microsoft list. It then
+# mergest that list into the fetched certdata.txt.
+#
+giturl="https://github.com/dotnet/sdk"
+gitrawurl="https://raw.githubusercontent.com/dotnet/sdk"
+release="latest"
+treedir="src/Layout/redist/trustedroots/codesignctl.pem"
+target="microsoft_sign_obj_ca.pem"
+certdata="./certdata.txt"
+baseurl=""
+merge=1
+diff=0
+
+function getlatest
+{
+    local url=$1
+    local latest="0"
+    local tags=($(git ls-remote --tags ${url}))
+    for tag in "${tags[@]}"
+    do
+        if [[ ! ${tag} =~ refs/.* ]];  then
+            continue # skip hashes
+        fi
+        if [[ ${tag} =~ .*preview.* ]];  then
+            continue # skip preview tags, we only want release tags
+        fi
+        if [[ ${tag} =~ .*rc.* ]];  then
+            continue # skip release candidate tags, we only want release tags
+        fi
+        if [[ ${latest} < ${tag} ]]; then
+            latest=$tag
+        fi
+    done
+    latest=${latest##refs/tags/}
+    echo $latest
+}
+
+while [ -n "$1" ]; do
+   case $1 in
+   "-g")
+        shift
+	giturl=$1
+	;;
+   "-r")
+        shift
+	gitrawurl=$1
+	;;
+   "-t")
+        shift
+	treedir=$1
+	;;
+   "-r")
+        shift
+	release=$1
+	;;
+   "-u")
+        shift
+	baseurl=$1
+        release="unknown"
+	;;
+   "-o")
+        shift
+	target=$1
+	;;
+   "-c")
+        shift
+	certdata=$1
+	;;
+   "-n")
+        merge=0
+        ;;
+   "-d")
+        shift
+        diff=1
+        difffile=$1
+        ;;
+    *)
+	echo "usage: $0 [-u URL] [-o target] [-c certdata] [-n]"
+	echo "-g URL      git URL to fetch code signing list"
+	echo "-r URL      raw git URL to fetch code signing list"
+	echo "-t URL      git tree directory to fetch code signing list"
+	echo "-r release  code signing list release version"
+	echo "-u URL      base URL to fetch code signing list"
+	echo "-o target   name of the codesigning target"
+	echo "-c certdata patch to certdata.txt to merge with"
+	echo "-d diff     optional diff file"
+        echo "-n          don't merge"
+	exit 1
+	;;
+    esac
+    shift
+done
+
+if [ "${release}" = "latest" ]; then
+     release=$(getlatest ${giturl} )
+fi
+
+if [ "${baseurl}" = "" ]; then
+     baseurl="${gitrawurl}/${release}/${treedir}"
+fi
+
+echo $release > "./codesign-release.txt"
+
+echo "Fetching release=${release}, ${target} from ${baseurl}"
+
+wget ${baseurl} -O ${target}
+
+if [ ${merge} -eq 0 ]; then
+    exit 0;
+fi
+
+out=${certdata}
+if [ ${diff} -eq 1 ]; then
+   out=${certdata}.out
+fi
+python3 ./mergepem2certdata.py -c "${certdata}" -p "${target}" -o "${out}" -t "CKA_TRUST_CODE_SIGNING" -l "Microsoft Code Signing Only Certificate"
+
+if [ ${diff} -eq 1 ]; then
+    diff -u ${certdata} ${out} > ${difffile}
+    mv ${out} ${certdata}
+fi

mergepem2certdata.py

@@ -0,0 +1,442 @@
+#!/usr/bin/python
+# vim:set et sw=4:
+#
+# certdata2pem.py - splits certdata.txt into multiple files
+#
+# Copyright (C) 2009 Philipp Kern <pkern@debian.org>
+# Copyright (C) 2013 Kai Engert <kaie@redhat.com>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+import base64
+import os.path
+import re
+import sys
+import textwrap
+import subprocess
+import getopt
+import asn1
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from datetime import datetime
+from dateutil.parser import parse
+
+objects = []
+
+pemcerts = []
+
+certdata='./certdata.txt'
+pem='./cert.pem'
+output='./certdata_out.txt'
+trust='CKA_TRUST_CODE_SIGNING'
+merge_label="Non-Mozilla Object Signing Only Certificate"
+dateString='thisyear'
+
+trust_types = {
+  "CKA_TRUST_SERVER_AUTH",
+  "CKA_TRUST_EMAIL_PROTECTION",
+  "CKA_TRUST_CODE_SIGNING"
+}
+
+attribute_types = {
+    "CKA_CLASS" : "CK_OBJECT_CLASS",
+    "CKA_TOKEN" : "CK_BBOOL",
+    "CKA_PRIVATE" : "CK_BBOOL",
+    "CKA_MODIFIABLE" : "CK_BBOOL",
+    "CKA_LABEL" : "UTF8",
+    "CKA_CERTIFICATE_TYPE" : "CK_CERTIFICATE_TYPE",
+    "CKA_SUBJECT" : "MULTILINE_OCTAL",
+    "CKA_ID" : "UTF8",
+    "CKA_CERT_SHA1_HASH" : "MULTILINE_OCTAL",
+    "CKA_CERT_MD5_HASH" : "MULTILINE_OCTAL",
+    "CKA_ISSUER" : "MULTILINE_OCTAL",
+    "CKA_SERIAL_NUMBER" : "MULTILINE_OCTAL",
+    "CKA_VALUE" : "MULTILINE_OCTAL",
+    "CKA_NSS_MOZILLA_CA_POLICY" : "CK_BBOOL",
+    "CKA_NSS_SERVER_DISTRUST_AFTER" : "Distrust",
+    "CKA_NSS_EMAIL_DISTRUST_AFTER" : "Distrust",
+    "CKA_TRUST_SERVER_AUTH" : "CK_TRUST",
+    "CKA_TRUST_EMAIL_PROTECTION" : "CK_TRUST",
+    "CKA_TRUST_CODE_SIGNING" : "CK_TRUST",
+    "CKA_TRUST_STEP_UP_APPROVED" : "CK_BBOOL"
+}
+
+def printable_serial(obj):
+  return ".".join([str(x) for x in obj['CKA_SERIAL_NUMBER']])
+
+def getSerial(cert):
+    encoder = asn1.Encoder()
+    encoder.start()
+    encoder.write(cert.serial_number)
+    return encoder.output()
+
+def dumpOctal(f,value):
+    for i in range(len(value)) :
+        if  i % 16 == 0 :
+            f.write("\n")
+        f.write("\\%03o"%int.from_bytes(value[i:i+1],sys.byteorder))
+    f.write("\nEND\n")
+
+# in python 3.8 this can be replaced with return byteval.hex(':',1)
+def formatHex(byteval) :
+    string=byteval.hex()
+    string_out=""
+    for i in range(0,len(string)-2,2) :
+         string_out += string[i:i+2] + ':'
+    string_out += string[-2:]
+    return string_out
+
+def getdate(dateString):
+    print("dateString= %s"%dateString)
+    if dateString.upper() == "THISYEAR":
+        return datetime(datetime.today().year,12,31,11,59,59,9999)
+    if dateString.upper() == "TODAY":
+        return datetime.today()
+    return parse(dateString, fuzzy=True);
+
+def getTrust(objlist, serial, issuer) :
+    for obj in objlist:
+        if obj['CKA_CLASS'] == 'CKO_NSS_TRUST' and obj['CKA_SERIAL_NUMBER'] == serial and obj['CKA_ISSUER'] == issuer:
+            return obj
+    return None
+
+def isDistrusted(obj) :
+    if (obj == None):
+        return False
+    return obj['CKA_TRUST_SERVER_AUTH'] == 'CKT_NSS_NOT_TRUSTED' and obj['CKA_TRUST_EMAIL_PROTECTION'] == 'CKT_NSS_NOT_TRUSTED' and obj['CKA_TRUST_CODE_SIGNING'] == 'CKT_NSS_NOT_TRUSTED'
+
+
+def stripQuotes(label) :
+    if label[:1] == "\"" :
+        label=label[1:]
+    if label[-1] == "\"" :
+        label = label[:-1]
+    return label
+
+# another object of the same class has the same label
+def labelExists(objlist, obj) :
+    for iobj in objlist:
+        if obj['CKA_CLASS'] == iobj['CKA_CLASS'] and obj['CKA_LABEL'] == iobj['CKA_LABEL']:
+            return True
+    return False
+
+# add an object, make sure that label is unique
+def addObj(objlist, newObj, specialLabel, drop) :
+    label = stripQuotes(newObj['CKA_LABEL'])
+    count=1
+    if specialLabel != None :
+        count=0
+        label=label+' '+specialLabel
+    # make sure the label is unique
+    while labelExists(objlist, newObj) :
+        if drop :
+            return 'DROPPED'
+        if count != 0 :
+            newObj['CKA_LABEL'] = "\"%s %d\""%(label,count)
+        else :
+            newObj['CKA_LABEL'] = "\"%s\""%label
+        count=count+1
+    objlist.append(obj)
+    return stripQuotes(newObj['CKA_LABEL'])
+
+try:
+    opts, args = getopt.getopt(sys.argv[1:],"c:o:p:t:l:x:",)
+except getopt.GetoptError as err:
+    print(err)
+    print(sys.argv[0] + ' [-c certdata] [-p pem] [-o certdata_target] [-t trustvalue] [-l merge_label]')
+    print('-c certdata         certdata file to merge to (default="'+certdata+'")');
+    print('-p pem              pem file with CAs to merge from (default="'+pem+'")');
+    print('-o certdata_target  resulting output file (default="'+output+'")');
+    print('-t trustvalue       what these CAs are trusted for (default="'+trust+'")');
+    print('-l merge_label      what label CAs that aren\'t in certdata (default="'+merge_label+'")');
+    print('-x date             remove all certs that expire before data (default='+dateString+')');
+    sys.exit(2)
+
+for opt, arg in opts:
+    if opt == '-c' :
+        certdata = arg
+    elif opt == '-p' :
+        pem = arg
+    elif opt == '-o' :
+        output = arg
+    elif opt == '-t' :
+        trust = arg
+    elif opt == '-l' :
+        merge_label = arg
+    elif opt == '-x' :
+        dateString = arg
+
+# parse dateString
+print ("datastring=",dateString)
+verifyDate = True
+if dateString.upper() == "NEVER":
+   verifyDate = False
+else:
+   date = getdate(dateString)
+print ("verifyDate=",verifyDate)
+
+
+# read the pem file
+in_cert, certvalue = False, ""
+for line in open(pem, 'r'):
+    if not in_cert:
+       if line.find("BEGIN CERTIFICATE") != -1:
+            in_cert = True;
+       continue
+    # Ignore comment lines and blank lines.
+    if line.startswith('#'):
+        continue
+    if len(line.strip()) == 0:
+        continue
+    if line.find("END CERTIFICATE") != -1 :
+       pemcerts.append(certvalue);
+       certvalue = "";
+       in_cert = False;
+       continue
+    certvalue += line;
+
+# read the certdata.txt file
+in_data, in_multiline, in_obj = False, False, False
+field, ftype, value, binval, obj = None, None, None, bytearray(), dict()
+header, comment = "", ""
+for line in open(certdata, 'r'):
+    # Ignore the file header.
+    if not in_data:
+        header += line
+        if line.startswith('BEGINDATA'):
+            in_data = True
+        continue
+    # Ignore comment lines. 
+    if line.startswith('#'):
+        comment += line
+        continue
+
+    # Empty lines are significant if we are inside an object.
+    if in_obj and len(line.strip()) == 0:
+        # collect all the inline comments in this object
+        obj['Comment'] += comment
+        comment = ""
+        addObj(objects, obj, None, False)
+        obj = dict()
+        in_obj = False
+        continue
+    if len(line.strip()) == 0:
+        continue
+    if in_multiline:
+        if not line.startswith('END'):
+            if ftype == 'MULTILINE_OCTAL':
+                line = line.strip()
+                for i in re.finditer(r'\\([0-3][0-7][0-7])', line):
+                    integ = int(i.group(1), 8)
+                    binval.extend((integ).to_bytes(1, sys.byteorder))
+                obj[field] = binval
+            else:
+                value += line
+                obj[field] = value
+            continue
+        in_multiline = False
+        continue
+    if line.startswith('CKA_CLASS'):
+        in_obj = True
+        obj['Comment'] = comment
+        comment = ""
+    line_parts = line.strip().split(' ', 2)
+    if len(line_parts) > 2:
+        field, ftype = line_parts[0:2]
+        value = ' '.join(line_parts[2:])
+    elif len(line_parts) == 2:
+        field, ftype = line_parts
+        value = None
+    else:
+        raise NotImplementedError('line_parts < 2 not supported.\n' + line)
+    if ftype == 'MULTILINE_OCTAL':
+        in_multiline = True
+        value = ""
+        binval = bytearray()
+        continue
+    obj[field] = value
+
+if len(list(obj.items())) > 0:
+    addObj(objects, obj, None, False)
+
+# strip out expired certificates from certdata.txt
+if verifyDate :
+    for obj in objects:
+        if obj['CKA_CLASS'] == 'CKO_CERTIFICATE' :
+            cert = x509.load_der_x509_certificate(bytes(obj['CKA_VALUE']))
+            if (cert.not_valid_after <= date) :
+                trust_obj = getTrust(objects,obj['CKA_SERIAL_NUMBER'],obj['CKA_ISSUER'])
+                # we don't remove distrusted expired certificates
+                if  not isDistrusted(trust_obj) :
+                    print("  Remove cert %s"%obj['CKA_LABEL'])
+                    print("     Expires: %s"%cert.not_valid_after.strftime("%m/%d/%Y"))
+                    print("     Prune time %s: "%date.strftime("%m/%d/%Y"))
+                    obj['Comment'] = None;
+                    if (trust_obj != None):
+                        trust_obj['Comment'] = None;
+
+# now merge the results
+for certval in pemcerts:
+    certder = base64.b64decode(certval)
+    cert = x509.load_der_x509_certificate(certder)
+    try:
+        label=cert.subject.get_attributes_for_oid(x509.oid.NameOID.COMMON_NAME)[0].value
+    except:
+        try:
+            label=cert.subject.get_attributes_for_oid(x509.oid.NameOID.ORGANIZATION_UNIT_NAME)[0].value
+        except:
+            try:
+                label=cert.subject.get_attributes_for_oid(x509.oid.NameOID.ORGANIZATION_NAME)[0].value
+            except:
+                label="Unknown Certificate"
+    if verifyDate :
+        if cert.not_valid_after <= date:
+            print("  Skipping code signing cert %s"%label)
+            print("     Expires: %s"%cert.not_valid_after.strftime("%m/%d/%Y"))
+            print("     Prune time %s: "%date.strftime("%m/%d/%Y"))
+            continue
+    certhashsha1 = cert.fingerprint(hashes.SHA1())
+    certhashmd5 =  cert.fingerprint(hashes.MD5())
+    
+    
+    found = False
+    # see if it exists in certdata.txt
+    for obj in objects:
+        # we only need to check the trust objects, because
+        # that is the object we would modify if it exists
+        if obj['CKA_CLASS'] != 'CKO_NSS_TRUST':
+            continue
+        # explicitly distrusted certs don't have a hash value
+        if not 'CKA_CERT_SHA1_HASH' in obj:
+            continue
+        if obj['CKA_CERT_SHA1_HASH'] != certhashsha1:
+            continue
+        obj[trust] = 'CKT_NSS_TRUSTED_DELEGATOR'
+        found = True
+        print('Updating "'+label+'" with code signing');
+        break
+    if  found :
+        continue
+
+    # check for almost duplicates, certs with the same subject and key, but
+    # different values. If they exist, treat them as the same certificate
+    for obj in objects:
+        if obj['CKA_CLASS'] != 'CKO_CERTIFICATE':
+            continue
+        # do they have the same subject?
+        if obj['CKA_SUBJECT'] != cert.subject.public_bytes():
+            continue
+        # do they have the same public key?
+        cert2 = x509.load_der_x509_certificate(bytes(obj['CKA_VALUE']))
+        if cert2.public_key().public_bytes(serialization.Encoding.DER,serialization.PublicFormat.SubjectPublicKeyInfo) != cert.public_key().public_bytes(serialization.Encoding.DER,serialization.PublicFormat.SubjectPublicKeyInfo) :
+            continue
+        #found now update trust record
+        trust_obj = getTrust(objects,obj['CKA_SERIAL_NUMBER'],obj['CKA_ISSUER'])
+        if trust_obj is None :
+            print('Couldn\'t find trust object for "'+obj['CKA_LABEL']);
+            exit
+        trust_obj[trust] = 'CKT_NSS_TRUSTED_DELEGATOR'
+        found = True
+        print('Updating sister certificate "'+obj['CKA_LABEL']+'" with code signing based on Microsoft "'+label+'"');
+        break
+        if found :
+            break
+    if  found :
+        continue
+    # append this certificate
+    obj=dict()
+    time='%a %b %d %H:%M:%S %Y'
+    comment  = '# ' + merge_label + '\n# %s "'+label+'"\n'
+    comment +=  '# Issuer: ' + cert.issuer.rfc4514_string() + '\n'
+    comment +=  '# Serial Number:'
+    sn=cert.serial_number
+    if sn < 0x100000:
+        comment +=  ' %d (0x%x)\n'%(sn,sn)
+    else:
+        comment +=  formatHex(sn.to_bytes((sn.bit_length()+7)//8,"big")) + '\n'
+    comment +=  '# Subject: ' + cert.subject.rfc4514_string() + '\n'
+    comment +=  '# Not Valid Before: ' + cert.not_valid_before.strftime(time) + '\n'
+    comment +=  '# Not Valid After: ' + cert.not_valid_after.strftime(time) + '\n'
+    comment +=  '# Fingerprint (MD5): ' + formatHex(certhashmd5) + '\n'
+    comment +=  '# Fingerprint (SHA1): ' + formatHex(certhashsha1) + '\n'
+    obj['Comment']= comment%"Certificate"
+    obj['CKA_CLASS'] = 'CKO_CERTIFICATE'
+    obj['CKA_TOKEN'] = 'CK_TRUE'
+    obj['CKA_PRIVATE'] = 'CK_FALSE'
+    obj['CKA_MODIFIABLE'] = 'CK_FALSE'
+    obj['CKA_LABEL'] = '"' + label + '"'
+    obj['CKA_CERTIFICATE_TYPE'] = 'CKC_X_509'
+    obj['CKA_SUBJECT'] = cert.subject.public_bytes()
+    obj['CKA_ID'] = '"0"'
+    obj['CKA_ISSUER'] = cert.issuer.public_bytes()
+    obj['CKA_SERIAL_NUMBER'] = getSerial(cert)
+    obj['CKA_VALUE'] = certder
+    obj['CKA_NSS_MOZILLA_CA_POLICY'] = 'CK_FALSE'
+    obj['CKA_NSS_SERVER_DISTRUST_AFTER'] = 'CK_FALSE'
+    obj['CKA_NSS_EMAIL_DISTRUST_AFTER'] = 'CK_FALSE'
+    label = addObj(objects, obj, 'CodeSigning', True)
+    if label == 'DROPPED' :
+        continue
+
+    # append the trust values
+    obj=dict()
+    obj['Comment']= comment%"Trust for"
+    obj['CKA_CLASS'] = 'CKO_NSS_TRUST'
+    obj['CKA_TOKEN'] = 'CK_TRUE'
+    obj['CKA_PRIVATE'] = 'CK_FALSE'
+    obj['CKA_MODIFIABLE'] = 'CK_FALSE'
+    obj['CKA_LABEL'] = '"' + label + '"'
+    obj['CKA_CERT_SHA1_HASH'] = certhashsha1
+    obj['CKA_CERT_MD5_HASH'] = certhashmd5
+    obj['CKA_ISSUER'] = cert.issuer.public_bytes()
+    obj['CKA_SERIAL_NUMBER'] = getSerial(cert)
+    for t in list(trust_types):
+       if t == trust:
+          obj[t] = 'CKT_NSS_TRUSTED_DELEGATOR'
+       else:
+          obj[t] = 'CKT_NSS_MUST_VERIFY_TRUST'
+    obj['CKA_TRUST_STEP_UP_APPROVED'] = 'CK_FALSE'
+    label = addObj(objects, obj, 'CodeSigning', True)
+    print('Adding code signing cert "'+label+'"');
+
+# now dump the results
+f = open(output, 'w')
+f.write(header)
+for obj in objects:
+    if 'Comment' in obj:
+        # if comment is None, we've deleted the entry above
+        if obj['Comment'] == None:
+            continue
+        f.write(obj['Comment'])
+    else:
+        print("Object with no comment!!")
+        print(obj)
+    for field in list(attribute_types.keys()):
+        if not field in obj:
+            continue
+        ftype = attribute_types[field];
+        if ftype == 'Distrust':
+            if obj[field] == 'CK_FALSE':
+                ftype = 'CK_BBOOL'
+            else:
+                ftype = 'MULTILINE_OCTAL'
+        f.write("%s %s"%(field,ftype));
+        if ftype == 'MULTILINE_OCTAL':
+           dumpOctal(f,obj[field])
+        else:
+           f.write(" %s\n"%obj[field])
+    f.write("\n")
+f.close

nssckbi.h

@@ -32,7 +32,7 @@
  * - whenever possible, if older branches require a modification to the
  *   list, these changes should be made on the main line of development (trunk),
  *   and the older branches should update to the most recent list.
- * 
+ *
  * - ODD minor version numbers are reserved to indicate a snapshot that has
  *   deviated from the main line of development, e.g. if it was necessary
  *   to modify the list on a stable branch.
@@ -46,8 +46,8 @@
  * It's recommend to switch back to 0 after having reached version 98/99.
  */
 #define NSS_BUILTINS_LIBRARY_VERSION_MAJOR 2
-#define NSS_BUILTINS_LIBRARY_VERSION_MINOR 22
-#define NSS_BUILTINS_LIBRARY_VERSION "2.22"
+#define NSS_BUILTINS_LIBRARY_VERSION_MINOR 80
+#define NSS_BUILTINS_LIBRARY_VERSION "2.80"
 
 /* These version numbers detail the semantic changes to the ckfw engine. */
 #define NSS_BUILTINS_HARDWARE_VERSION_MAJOR 1

plans/smoke.fmf

@@ -0,0 +1,4 @@
+ discover:
+     how: fmf
+ execute:
+     how: tmt

sort-blocks.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 # Expected input is a file, where blocks of lines are separated by newline.
 # Blocks will be sorted.
@@ -9,7 +9,7 @@ import sys
 import string
 
 if (len(sys.argv) != 2):
-    print "syntax: " + sys.argv[0] + " input-filename"
+    print("syntax: " + sys.argv[0] + " input-filename")
     sys.exit(1)
 
 filename = sys.argv[1]
@@ -31,4 +31,4 @@ with open(filename, 'r') as f:
 block_list.sort()
 
 for block in block_list:
-    print block
+    print(block)

tests/smoke-test/Makefile

@@ -1,64 +0,0 @@
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-#
-#   Makefile of /CoreOS/ca-certificates/Sanity/smoke-test
-#   Description: Check presence of Verisign root.
-#   Author: Ondrej Moris <omoris@redhat.com>
-#
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-#
-#   Copyright (c) 2010 Red Hat, Inc. All rights reserved.
-#
-#   This copyrighted material is made available to anyone wishing
-#   to use, modify, copy, or redistribute it subject to the terms
-#   and conditions of the GNU General Public License version 2.
-#
-#   This program is distributed in the hope that it will be
-#   useful, but WITHOUT ANY WARRANTY; without even the implied
-#   warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-#   PURPOSE. See the GNU General Public License for more details.
-#
-#   You should have received a copy of the GNU General Public
-#   License along with this program; if not, write to the Free
-#   Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
-#   Boston, MA 02110-1301, USA.
-#
-# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-export TEST=/CoreOS/ca-certificates/Sanity/smoke-test
-export TESTVERSION=1.0
-
-BUILT_FILES=
-
-FILES=$(METADATA) runtest.sh Makefile PURPOSE
-
-.PHONY: all install download clean
-
-run: $(FILES) build
-	./runtest.sh
-
-build: $(BUILT_FILES)
-	chmod a+x runtest.sh
-
-clean:
-	rm -f *~ $(BUILT_FILES)
-
-
-include /usr/share/rhts/lib/rhts-make.include
-
-$(METADATA): Makefile
-	@echo "Owner:           Ondrej Moris <omoris@redhat.com>" > $(METADATA)
-	@echo "Name:            $(TEST)" >> $(METADATA)
-	@echo "TestVersion:     $(TESTVERSION)" >> $(METADATA)
-	@echo "Path:            $(TEST_DIR)" >> $(METADATA)
-	@echo "Description:     Check presence of Verisign root." >> $(METADATA)
-	@echo "Type:            Sanity" >> $(METADATA)
-	@echo "TestTime:        5m" >> $(METADATA)
-	@echo "RunFor:          ca-certificates" >> $(METADATA)
-	@echo "Requires:        ca-certificates" >> $(METADATA)
-	@echo "Requires:        wget" >> $(METADATA)
-	@echo "Priority:        Normal" >> $(METADATA)
-	@echo "License:         GPLv2" >> $(METADATA)
-	@echo "Confidential:    no" >> $(METADATA)
-	@echo "Destructive:     no" >> $(METADATA)
-
-	rhts-lint $(METADATA)

tests/smoke-test/PURPOSE

@@ -1,3 +0,0 @@
-PURPOSE of /CoreOS/ca-certificates/Sanity/smoke-test
-Description: Check presence of Verisign root.
-Author: Ondrej Moris <omoris@redhat.com>

tests/smoke-test/main.fmf

@@ -0,0 +1,5 @@
+summary: Check presence of Verisign root.
+test: bash ./runtest.sh
+framework: beakerlib
+recommend:
+  - beakerlib

tests/smoke-test/runtest.sh

@@ -27,7 +27,7 @@
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 # Include rhts environment
-. /usr/lib/beakerlib/beakerlib.sh
+. /usr/share/beakerlib/beakerlib.sh || exit 1
 
 PACKAGE="ca-certificates"
 

tests/tests.yml

@@ -1,13 +0,0 @@
----
-# This first play always runs on the local staging system
-- hosts: localhost
-  roles:
-  - role: standard-test-beakerlib
-    tags:
-    - atomic
-    - classic
-    - container
-    tests:
-    - smoke-test
-    required_packages:
-    - findutils         # beakerlib needs find command

update-ca-trust

@@ -1,21 +1,183 @@
 #!/bin/sh
 
 #set -vx
+set -eu
 
-# At this time, while this script is trivial, we ignore any parameters given.
-# However, for backwards compatibility reasons, future versions of this script must 
-# support the syntax "update-ca-trust extract" trigger the generation of output 
+# For backwards compatibility reasons, future versions of this script must
+# support the syntax "update-ca-trust extract" trigger the generation of output
 # files in $DEST.
 
 DEST=/etc/pki/ca-trust/extracted
+DEST_CERTS=/etc/pki/tls/certs
 
 # Prevent p11-kit from reading user configuration files.
 export P11_KIT_NO_USER_CONFIG=1
 
-# OpenSSL PEM bundle that includes trust flags
-# (BEGIN TRUSTED CERTIFICATE)
-/usr/bin/p11-kit extract --format=openssl-bundle --filter=certificates --overwrite --comment $DEST/openssl/ca-bundle.trust.crt
-/usr/bin/p11-kit extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose server-auth $DEST/pem/tls-ca-bundle.pem
-/usr/bin/p11-kit extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose email $DEST/pem/email-ca-bundle.pem
-/usr/bin/p11-kit extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose code-signing $DEST/pem/objsign-ca-bundle.pem
-/usr/bin/p11-kit extract --format=java-cacerts --filter=ca-anchors --overwrite --purpose server-auth $DEST/java/cacerts
+usage() {
+	fold -s -w 76 >&2 <<-EOF
+		Usage: $0 [extract] [-o DIR|--output DIR]
+
+		Update the system trust store in $DEST.
+
+		COMMANDS
+		(absent/empty command): Same as the extract command without arguments.
+
+		extract: Instruct update-ca-trust to scan the source configuration in
+		/usr/share/pki/ca-trust-source and /etc/pki/ca-trust/source and produce
+		updated versions of the consolidated configuration files stored below
+		the $DEST directory hierarchy.
+
+		EXTRACT OPTIONS
+		-o DIR, --output DIR: Write the extracted trust store into the given
+		directory instead of updating $DEST. (Note: This option will not
+		populate the ../pki/tls/certs with the directory-hash symbolic links.)
+
+		--rhbz2387674: A temporary compatibility option that restores several
+		legacy certificate-bundle symlinks (e.g., /etc/ssl/cert.pem) to
+		address issues with older software.
+		These symlinks will be removed on ca-certificate updates or reinstalls,
+		so you'll have to re-run this command after ca-certificates updates if
+		the issue is still not fixed.
+		WARNING: Do not use in automation or build scripts. This flag
+		is going to be removed in a future release, and any scripts relying on
+		it will inevitably break!
+	EOF
+}
+
+rhbz2387674_msg() {
+	fold -s -w 76 >&2 <<-EOF
+	----------------------------------------------------------------------------
+	** DEPRECATION WARNING **
+	----------------------------------------------------------------------------
+	The option --rhbz2387674 is a temporary workaround and will be removed in a
+	future release. Please do not use it in build scripts or automation.
+
+	----------------------------------------------------------------------------
+	** ACTION REQUIRED **
+	----------------------------------------------------------------------------
+	To ensure the affected package works correctly in the future, a bug report must
+	be filed.
+
+	1.  Check if a bug already exists for the affected package: https://bugzilla.redhat.com/buglist.cgi?component=ca-certificates&product=Fedora&short_desc=droppingOfCertPemFile%20package%3A&short_desc_type=allwordssubstr
+
+	2.  If no bug exists, please file a new one using this template: https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora&component=ca-certificates&version=rawhide&short_desc=droppingOfCertPemFile+package:+<<package_name>>+is+affected
+
+	Thank you for helping improve Fedora.
+	EOF
+}
+
+extract() {
+	USER_DEST=
+	compat=
+
+        # can't use getopt here. ca-certificates can't depend on a lot
+        # of other libraries since openssl depends on ca-certificates
+        # just fail when we hand parse
+
+        while [ $# -ne 0 ]; do
+	    case "$1" in
+		"-o"|"--output")
+		  if [ $# -lt 2 ]; then
+			  echo >&2 "Error: missing argument for '$1' option. See 'update-ca-trust --help' for usage."
+			  echo >&2
+			  exit 1
+		  fi
+	          USER_DEST=$2
+		  shift 2
+		  continue
+		  ;;
+		"--rhbz2387674")
+		  compat="true"
+		  shift
+		  continue
+		  ;;
+		"--")
+		  shift
+		  break
+		  ;;
+		*)
+		  echo >&2 "Error: unknown extract argument '$1'. See 'update-ca-trust --help' for usage."
+		  exit 1
+		  ;;
+	    esac
+	done
+
+	if [[ "$compat" = "true" && -n "$USER_DEST" ]]; then
+		echo "Error: arguments '-o DIR|--output DIR' and '--rhbz2387674' can't be used together"
+		exit 1
+	
+	fi
+
+	if [ -n "$USER_DEST" ]; then
+		DEST=$USER_DEST
+	        # Attempt to create the directories if they do not exist
+                # yet (rhbz#2241240)
+	        /usr/bin/mkdir -p \
+		    "$DEST"/openssl \
+		    "$DEST"/pem \
+		    "$DEST"/java \
+		    "$DEST"/edk2
+	fi
+
+
+	# Delete all directory hash symlinks from the cert directory
+	if [ -z "$USER_DEST" ]; then
+		find "$DEST_CERTS" -type l -regextype posix-extended \
+		     -regex '.*/[0-9a-f]{8}\.[0-9]+' -exec rm -f {} \;
+	fi
+
+	# OpenSSL PEM bundle that includes trust flags
+	# (BEGIN TRUSTED CERTIFICATE)
+	/usr/bin/trust extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose server-auth "$DEST/pem/tls-ca-bundle.pem"
+	/usr/bin/trust extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose email "$DEST/pem/email-ca-bundle.pem"
+	/usr/bin/trust extract --format=pem-bundle --filter=ca-anchors --overwrite --comment --purpose code-signing "$DEST/pem/objsign-ca-bundle.pem"
+	/usr/bin/trust extract --format=java-cacerts --filter=ca-anchors --overwrite --purpose server-auth "$DEST/java/cacerts"
+	/usr/bin/trust extract --format=edk2-cacerts --filter=ca-anchors --overwrite --purpose=server-auth "$DEST/edk2/cacerts.bin"
+	# Hashed directory of BEGIN TRUSTED-style certs (usable as OpenSSL CApath and
+	# by GnuTLS)
+	/usr/bin/trust extract --format=pem-directory-hash --filter=ca-anchors --overwrite --purpose server-auth "$DEST/pem/directory-hash"
+
+
+	if [ -n "$compat" ]; then
+		# print warning message
+		rhbz2387674_msg
+
+		# bring back bundle in openssl trust format
+		/usr/bin/trust extract --format=openssl-bundle --filter=certificates --overwrite --comment "$DEST_CERTS/ca-bundle.trust.crt"
+
+		# create symlinks to /etc/pki/tls/..
+		ln -sf "$DEST/pem/tls-ca-bundle.pem" "$DEST_CERTS/../cert.pem"
+		ln -sf "$DEST/pem/tls-ca-bundle.pem" "$DEST_CERTS/ca-certificates.crt"
+		ln -sf "$DEST/pem/tls-ca-bundle.pem" "$DEST_CERTS/ca-bundle.crt"
+
+		# create symlinks to /etc/ssl/ the certs folder is already sym-linked
+		ln -sf "$DEST/pem/tls-ca-bundle.pem" "/etc/ssl/cert.pem"
+	fi
+
+	if [ -z "$USER_DEST" ]; then
+		find "$DEST/pem/directory-hash" -type l -regextype posix-extended \
+		     -regex '.*/[0-9a-f]{8}\.[0-9]+' | while read link; do
+			target=$(readlink -f "$link")
+			new_link="$DEST_CERTS/$(basename "$link")"
+			ln -s "$target" "$new_link"
+		done
+	fi
+}
+if [ $# -lt 1 ]; then
+    set -- extract
+fi
+
+case "$1" in
+	"extract")
+		shift
+		extract "$@"
+	;;
+	"--help")
+		usage
+		exit 0
+	;;
+	*)
+		echo >&2 "Error: unknown command: '$1', see 'update-ca-trust --help' for usage."
+		exit 1
+	;;
+esac

update-ca-trust.8.txt

@@ -27,7 +27,7 @@ certificates and associated trust
 
 SYNOPSIS
 --------
-*update-ca-trust* ['COMMAND']
+*update-ca-trust* [extract] [-o 'DIR'|--output='DIR']
 
 
 DESCRIPTION
@@ -98,13 +98,13 @@ subdirectory in the /etc hierarchy.
 * add it as a new file to directory /etc/pki/ca-trust/source/anchors/
 * run 'update-ca-trust extract'
 
-.*QUICK HELP 2*: If your certificate is in the extended BEGIN TRUSTED file format (which may contain distrust/blacklist trust flags, or trust flags for usages other than TLS) then:
+.*QUICK HELP 2*: If your certificate is in the extended BEGIN TRUSTED file format (which may contain distrust/blocklist trust flags, or trust flags for usages other than TLS) then:
 * add it as a new file to directory /etc/pki/ca-trust/source/
 * run 'update-ca-trust extract'
 
 .In order to offer simplicity and flexibility, the way certificate files are treated depends on the subdirectory they are installed to.
 * simple trust anchors subdirectory: /usr/share/pki/ca-trust-source/anchors/ or /etc/pki/ca-trust/source/anchors/
-* simple blacklist (distrust) subdirectory: /usr/share/pki/ca-trust-source/blacklist/ or /etc/pki/ca-trust/source/blacklist/
+* simple blocklist (distrust) subdirectory: /usr/share/pki/ca-trust-source/blocklist/ or /etc/pki/ca-trust/source/blocklist/
 * extended format directory: /usr/share/pki/ca-trust-source/ or /etc/pki/ca-trust/source/
 
 .In the main directories /usr/share/pki/ca-trust-source/ or /etc/pki/ca-trust/source/ you may install one or multiple files in the following file formats:
@@ -134,7 +134,7 @@ you may install one or multiple certificates in either the DER file
 format or in the PEM (BEGIN/END CERTIFICATE) file format.
 Each certificate will be treated as *trusted* for all purposes.
 
-In the blacklist subdirectories /usr/share/pki/ca-trust-source/blacklist/ or /etc/pki/ca-trust/source/blacklist/
+In the blocklist subdirectories /usr/share/pki/ca-trust-source/blocklist/ or /etc/pki/ca-trust/source/blocklist/
 you may install one or multiple certificates in either the DER file
 format or in the PEM (BEGIN/END CERTIFICATE) file format.
 Each certificate will be treated as *distrusted* for all purposes.
@@ -202,29 +202,39 @@ trusted for E-Mail protection.
 File objsign-ca-bundle.pem contains CA certificates 
 trusted for code signing.
 
+The directory /etc/pki/ca-trust/extracted/edk2/ contains a CA
+certificate bundle ("cacerts.bin") in the "sequence of
+EFI_SIGNATURE_LISTs" format, defined in the UEFI-2.7 specification,
+sections "31.4.1 Signature Database" and
+"EFI_CERT_X509_GUID". Distrust information cannot be represented in
+this file format, and distrusted certificates are missing from these
+files. File "cacerts.bin" contains CA certificates trusted for TLS
+server authentication.
+
 
 COMMANDS
 --------
-(absent/empty command)::
-    Same as the *extract* command described below. (However, the command may
-    print fewer warnings, as this command is being run during rpm package 
-    installation, where non-fatal status output is undesired.)
+(absent/empty command)
+~~~~~~~~~~~~~~~~~~~~~~
+Same as the *extract* command described below. (However, the command may print
+fewer warnings, as this command is being run during rpm package installation,
+where non-fatal status output is undesired.)
 
-*extract*::
-    Instruct update-ca-trust to scan the <<sourceconf,SOURCE CONFIGURATION>> and produce 
-    updated versions of the consolidated configuration files stored below
-    the /etc/pki/ca-trust/extracted directory hierarchy.
+extract
+~~~~~~~
+Instruct update-ca-trust to scan the <<sourceconf,SOURCE CONFIGURATION>> and
+produce updated versions of the consolidated configuration files stored below
+the /etc/pki/ca-trust/extracted directory hierarchy.
+
+EXTRACT OPTIONS
+^^^^^^^^^^^^^^^
+*-o DIR*, *--output=DIR*::
+	Write the extracted trust store into the given directory instead of
+	updating /etc/pki/ca-trust/extracted. (Note: This option will not
+	populate the ../pki/tls/certs with the directory-hash symbolic links.)
 
 FILES
 -----
-/etc/pki/tls/certs/ca-bundle.crt::
-	Classic filename, file contains a list of CA certificates trusted for TLS server authentication usage, in the simple BEGIN/END CERTIFICATE file format, without distrust information.
-	This file is a symbolic link that refers to the consolidated output created by the update-ca-trust command.
-
-/etc/pki/tls/certs/ca-bundle.trust.crt::
-	Classic filename, file contains a list of CA certificates in the extended BEGIN/END TRUSTED CERTIFICATE file format, which includes trust (and/or distrust) flags specific to certificate usage.
-	This file is a symbolic link that refers to the consolidated output created by the update-ca-trust command.
-
 /etc/pki/java/cacerts::
 	Classic filename, file contains a list of CA certificates trusted for TLS server authentication usage, in the Java keystore file format, without distrust information.
 	This file is a symbolic link that refers to the consolidated output created by the update-ca-trust command.
@@ -240,6 +250,27 @@ FILES
 	which are created using the 'update-ca-trust extract' command. Don't edit files in this directory, because they will be overwritten.
 	See section <<extractconf,EXTRACTED CONFIGURATION>> for additional details.
 
+/etc/pki/tls/certs::
+	Contains symbolic links to the directory-hash format certificates generated by update-ca-trust command, they are inteded as a internal format for OpenSSL and not to be used directly by the other crypto libraries or applications.
+
+LEGACY FILES
+------------
+The following file paths were used in legacy versions of the utility
+and have since been replaced. Scripts and configurations referencing
+these old paths should be updated.
+
+/etc/pki/cert.pem::
+    This file has been replaced by /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem.
+
+/etc/pki/tls/certs/ca-certificates.crt::
+    This file has been replaced by /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem.
+
+/etc/pki/tls/certs/ca-bundle.crt::
+    This file has been replaced by /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem.
+
+/etc/pki/tls/certs/ca-bundle.trust.crt::
+    This has been replaced by the directory-hash format certificates stored in /etc/pki/ca-trust/extracted/pem/directory-hash/ directory.
+
 AUTHOR
 ------
 Written by Kai Engert and Stef Walter.