124 lines
3.8 KiB
Bash
Executable file
124 lines
3.8 KiB
Bash
Executable file
#!/bin/sh
|
||
set -o nounset
|
||
set -o errexit
|
||
|
||
FORGEURL='https://github.com/zip-rs/zip2'
|
||
|
||
# These are test data files that have been audited for obvious issues. What
|
||
# kind of issues might there be?
|
||
#
|
||
# - Acceptable license, but no license text
|
||
# - No known license / presumed or explicitly proprietary
|
||
# - Mysterious precompiled binaries
|
||
#
|
||
# Files we have determined we should *not* include:
|
||
#
|
||
# - chinese.zip:
|
||
# The single text file it contains, 七个房间.txt, contains a significant
|
||
# amount of narrative text in GB2312 encoding: some kind of story, or an
|
||
# excerpt from one, perhaps a work of fiction. It’s not clear what the origin
|
||
# is or what license, if any, might apply.
|
||
# - lin-ub_iwd-v11.zip:
|
||
# LICENSE.lin-ub_iwd-v11.zip.txt clearly shows that this is at *best* not
|
||
# under a license acceptable in Fedora.
|
||
# - pandoc_soft_links.zip:
|
||
# Contains a copy of pandoc 3.2 compiled for aarch64: pandoc itself is
|
||
# open-source, but who can say what was linked into it? The complete list of
|
||
# applicable licenses is not knowable.
|
||
OK_FILES="$(grep -vE '^(#|$)' <<EOF
|
||
# Contents are trivial: symlinks, zero-bytes files, sample text files
|
||
# containing only a few words, etc.
|
||
data_descriptor.zip
|
||
files_and_dirs.zip
|
||
linux-7z.zip
|
||
mimetype.zip
|
||
non_utf8.zip
|
||
symlink.zip
|
||
windows-7zip.zip
|
||
zip64_demo.zip
|
||
zip64_magic_in_filename_1.zip
|
||
zip64_magic_in_filename_2.zip
|
||
zip64_magic_in_filename_4.zip
|
||
zip64_magic_in_filename_5.zip
|
||
|
||
# These don’t unzip with the "unzip" tool, so they are difficult to audit, but
|
||
# their sizes are only from a few dozen to a few hundred bytes, so we assume
|
||
# their contents are very likely to be unproblematic.
|
||
deflate64_issue_25.zip
|
||
invalid_cde_number_of_files_allocation_greater_offset.zip
|
||
invalid_cde_number_of_files_allocation_smaller_offset.zip
|
||
invalid_offset2.zip
|
||
invalid_offset.zip
|
||
raw_deflate64_index_out_of_bounds.zip
|
||
zip64_magic_in_filename_3.zip
|
||
EOF
|
||
)"
|
||
|
||
|
||
print_help()
|
||
{
|
||
cat <<EOF
|
||
Usage: $1 VERSION
|
||
|
||
Generate a copy of the GitHub source archive for zip-rs/zip2, with everything
|
||
but the test/data/ subdirectory removed, and additionally, in which only test
|
||
data files that have been audited for obvious license issues are included. The
|
||
result will be named zip2-\${VERSION}-filtered.tar.gz and will be written into
|
||
the current working directory.
|
||
EOF
|
||
}
|
||
|
||
if [ "$#" != '1' ]
|
||
then
|
||
exec 1>&2
|
||
print_help "${0}"
|
||
exit 1
|
||
elif [ "${1-}" = '-h' ] || [ "${1-}" = '--help' ]
|
||
then
|
||
print_help "${0}"
|
||
exit 0
|
||
fi
|
||
|
||
VERSION="${1}"
|
||
SOURCE0="${FORGEURL}/archive/v${VERSION}/zip2-${VERSION}.tar.gz"
|
||
TARNAME="$(basename "${SOURCE0}")"
|
||
TARDIR="$(basename "${SOURCE0}" '.tar.gz')"
|
||
NEWTAR="${TARDIR}-filtered.tar.gz"
|
||
|
||
SAVEDIR="${PWD}"
|
||
XDIR="$(mktemp -d)"
|
||
trap "rm -rf '${XDIR}'" INT TERM EXIT
|
||
|
||
cd "${XDIR}"
|
||
curl -L -O "${SOURCE0}"
|
||
tar -xzf "${TARNAME}"
|
||
MTIME="$(stat -c '%Y' "${TARDIR}")"
|
||
# Remove everything but tests/data
|
||
find "${TARDIR}" -mindepth 1 -maxdepth 1 ! -name tests -execdir rm -r '{}' '+'
|
||
find "${TARDIR}/tests" -mindepth 1 -maxdepth 1 ! -name data \
|
||
-execdir rm -r '{}' '+'
|
||
# Empty tests/data (keeping the files in a duplicate tree), then move the files
|
||
# we want back in.
|
||
cp -lrp "${TARDIR}/tests/data" data_unfiltered
|
||
find "${TARDIR}/tests/data" -type f -delete
|
||
echo "${OK_FILES}" | while read -r fn
|
||
do
|
||
mv "data_unfiltered/${fn}" "${TARDIR}/tests/data/${fn}"
|
||
done
|
||
# https://www.gnu.org/software/tar/manual/html_section/Reproducibility.html
|
||
# We reset all mtimes to that of the top-level extracted directory; since git
|
||
# archives don’t have meaningful per-file mtimes, nothing useful is lost.
|
||
TZ=UTC LC_ALL=C tar \
|
||
--create --verbose \
|
||
--sort=name \
|
||
--format=posix \
|
||
--numeric-owner --owner=0 --group=0 \
|
||
--mode=go+u,go-w \
|
||
--pax-option='delete=atime,delete=ctime' \
|
||
--clamp-mtime --mtime="@${MTIME}" \
|
||
"${TARDIR}/" |
|
||
gzip -9 > "${NEWTAR}"
|
||
touch -d @"${MTIME}" "${NEWTAR}"
|
||
|
||
cd "${SAVEDIR}"
|
||
mv -v "${XDIR}/${NEWTAR}" .
|