diff --git a/chardetect.1 b/chardetect.1 index b5d3e15..614a0dd 100644 --- a/chardetect.1 +++ b/chardetect.1 @@ -1,4 +1,4 @@ -.TH CHARDETECT "1" "February 2026" "" "User Commands" +.TH CHARDETECT "1" "March 2026" "" "User Commands" .SH NAME .B chardetect \(en takes one or more file paths and reports their detected encodings @@ -7,39 +7,61 @@ .RB [ \-h ] .RB [ \-\-minimal ] .RB [ \-l ] -.RB [ \-e\ \fIENCODING_ERA ] +.RB [ \-e +.RB { modern_web , +.BR legacy_iso , +.BR legacy_mac , +.BR legacy_regional , +.BR dos , +.BR mainframe , +.BR all }] +.RB [ \-i\ \fIINCLUDE_ENCODINGS ] +.RB [ \-x\ \fIEXCLUDE_ENCODINGS ] +.RB [ \-\-no\-match\-encoding\ \fINO_MATCH_ENCODING ] +.RB [ \-\-empty\-input\-encoding\ \fIEMPTY_INPUT_ENCODING ] .RB [ \-\-version ] -.RI [ input \ ...] +.RI [ files \ ...] .SH OPTIONS .SS "POSITIONAL ARGUMENTS" .TP -.I input -File whose encoding we would like to determine. -(default: -.IR stdin ) +.I files +Files to detect encoding of .SS "OPTIONAL ARGUMENTS" .TP .B \-h\fR, \fB\-\-help -Show a help message and exit. +Show a help message and exit .TP .B \-\-minimal -Print only the encoding to standard output. +Output only the encoding name .TP -.B \-l\fR, \fB\-\-legacy -Rename legacy encodings to more modern ones. +.B \-l\fR, \fB\-\-language +Include detected language in output .TP -.B \-e\fR, \fB\-\-encoding\-era\ \fIENCODING_ERA -Which era of encodings to consider +.B \-e\fR, \fB\-\-encoding\-era +.RB { modern_web , +.BR legacy_iso , +.BR legacy_mac , +.BR legacy_regional , +.BR dos , +.BR mainframe , +.BR all } +Encoding era filter +.TP +.B \-i\fR, \fB\-\-include\-encodings\ \fIINCLUDE_ENCODINGS +Comma-separated list of encodings to consider +.TP +.B \-x\fR, \fB\-\-exclude\-encodings\ \fIEXCLUDE_ENCODINGS +Comma-separated list of encodings to exclude +.TP +.B \-\-no\-match\-encoding\ \fINO_MATCH_ENCODING +Encoding to return when detection is inconclusive (default: -.BR MODERN_WEB ). -Choices: -.BR MODERN_WEB , -.BR LEGACY_ISO , -.BR LEGACY_MAC , -.BR LEGACY_REGIONAL , -.BR DOS , -.BR MAINFRAME , -.BR ALL +.BR cp1252 ) +.TP +.B \-\-empty\-input\-encoding\ \fIEMPTY_INPUT_ENCODING +Encoding to return for empty input +(default: +.BR utf\-8 ) .TP .B \-\-version -Show program\(cqs version number and exit. +Show program\(cqs version number and exit diff --git a/get_source b/get_source deleted file mode 100755 index 59490a7..0000000 --- a/get_source +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -set -o errexit -set -o nounset - -if [ "$#" != '1' ] -then - cat 1>&2 < Downloading: ${URL}" 1>&2 -curl -L -O "${URL}" - -ARCHIVE="$(find . -mindepth 1 -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" -echo "--> Extracting: $(basename "${ARCHIVE}")" 1>&2 -tar -xzf "${ARCHIVE}" -echo '--> Removing tests due to licensing issues' 1>&2 -TARDIR="$(basename "${ARCHIVE}" '.tar.gz')" -MTIME="$(stat -c '%Y' "${TARDIR}")" -rm -rvf "${TARDIR}/tests/" -# Restore the original mtime even though we modified the base directory by -# removing the tests. -touch -d @"${MTIME}" "${TARDIR}" -FILTERED="$(basename "${ARCHIVE}" .tar.gz)-filtered.tar.zst" -echo "--> Re-archiving: ${FILTERED}" 1>&2 -# https://www.gnu.org/software/tar/manual/html_section/Reproducibility.html -TZ=UTC LC_ALL=C tar \ - --create --verbose \ - --sort=name \ - --format=posix \ - --numeric-owner --owner=0 --group=0 \ - --mode=go+u,go-w \ - --pax-option='delete=atime,delete=ctime' \ - "${TARDIR}/" | - zstdmt --ultra -22 > "${FILTERED}" -mv -v "${FILTERED}" "${OUTDIR}" -echo 'Done.' 1>&2 diff --git a/python-chardet.rpmlintrc b/python-chardet.rpmlintrc deleted file mode 100644 index ef558e4..0000000 --- a/python-chardet.rpmlintrc +++ /dev/null @@ -1,4 +0,0 @@ -# This Source cannot be a URL because we must filter out certain files. -addFilter(r" invalid-url Source[0-9]+: chardet-.*-filtered\.tar\.zst") -# Bogus; it is a perfectly fine zstandard file. -addFilter(r" inconsistent-file-extension chardet-.*-filtered\.tar\.zst$") diff --git a/python-chardet.spec b/python-chardet.spec index bec603e..ef9c5dc 100644 --- a/python-chardet.spec +++ b/python-chardet.spec @@ -1,52 +1,62 @@ -#global commit xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +# Using mypyc to produce a compiled extension resulted in a ~40% increase in +# throughput (files/s) on the upstream test corpus in upstream’s tests. +# +# However: +# - python-hatch-mypyc would need to be packaged +# - mypyc’s own documentation at +# https://mypyc.readthedocs.io/en/latest/introduction.html currently warns, +# “Mypyc is currently alpha software. It’s only recommended for production +# use cases with careful testing, and if you are willing to contribute fixes +# or to work around issues you will encounter.” +%bcond mypyc 0 Name: python-chardet -Version: 6.0.0.post1 +Version: 7.4.3 Release: %autorelease -Summary: Python character encoding detector +Summary: Universal character encoding detector -# The entire source, after tests/ are filtered, is LGPL-2.1-or-later. See the -# comment above Source0 for more details on tests/. -License: LGPL-2.1-or-later +# References: +# - https://github.com/chardet/chardet/issues/327 +# - https://github.com/chardet/chardet/issues/334 +# - https://github.com/chardet/chardet/issues/355 +# - https://lists.fedoraproject.org/archives/list/ +# legal@lists.fedoraproject.org/thread/ +# 67RIQBACH6FN6FGOTJPAXHX4YL5Q2HPG/#T6FCBBV7XQBEZO2QTTYRELKDC44BITIZ +# - https://dan-blanchard.github.io/blog/chardet-rewrite-controversy/ +License: 0BSD URL: https://github.com/chardet/chardet -# A filtered source tarball, obtained by (see Source1): -# -# ./get_source %%{version} -# or -# ./get_source %%{commit} -# -# is required because the contents of tests/ are under various undocumented -# licenses and are, for the most part, not freely redistributable. See: -# -# problematic licensing of /tests? -# https://github.com/chardet/chardet/issues/231 -%if %{undefined commit} -Source0: chardet-%{version}-filtered.tar.zst -%else -Source0: chardet-%{commit}-filtered.tar.zst -%endif -Source1: get_source +Source0: %{url}/archive/%{version}/chardet-%{version}.tar.gz # Hand-written for Fedora in groff_man(7) format based on --help output -Source2: chardetect.1 +Source1: chardetect.1 BuildSystem: pyproject -BuildOption(install): --assert-license chardet +# Compiling with mypyc produces a weird top-level module, which is apparently +# normal: “Having this module is part of how mypyc works.” +# https://github.com/hukkin/tomli/issues/268#issuecomment-4149757651 +# The actual name is something like: +# 0deeb2fec52624e647be__mypyc.cpython-314-x86_64-linux-gnu.so +BuildOption(install): --assert-license chardet %{?with_mypyc:'*_mypyc'} +%if %{without mypyc} BuildArch: noarch +%else +BuildRequires: %{py3_dist hatch-mypyc} +BuildRequires: gcc +BuildRequires: tomcli +%endif + +# See the “dev” dependency group, which also contains unwanted dependencies for +# linting, coverage analysis, and so on. +# https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/#_linters +# datasets: only for scripts/train.py +# prek: only for pre-commit checks +BuildRequires: %{py3_dist pytest} +# pytest-cov: only for coverage analysis +BuildRequires: %{py3_dist pytest-xdist} +# ruff: only for linting/formatting %global common_description %{expand: -Chardet: The Universal Character Encoding Detector - -Detects over 70 character encodings including: - - • All major Unicode encodings (UTF-8, UTF-16, UTF-32) - • Windows code pages (Windows-1250 through Windows-1258) - • ISO-8859 family (ISO-8859-1 through ISO-8859-16) - • CJK encodings (Big5, GB18030, EUC-JP, EUC-KR, Shift-JIS, and more) - • Cyrillic encodings (KOI8-R, KOI8-U, IBM866, and more) - • Mac encodings (MacRoman, MacCyrillic, and more) - • DOS/OEM code pages (CP437, CP850, CP866, and more) - • EBCDIC variants (CP037, CP500)} +%{summary}.} %description %{common_description} @@ -55,8 +65,16 @@ Detects over 70 character encodings including: %package -n python3-chardet Summary: %{summary} -%description -n python3-chardet -%{common_description} +%description -n python3-chardet %{common_description} + + +%if %{with mypyc} +%prep -a +# Upstream asks for no debugging symbols (-g0); mypyc defaults to -g1. Respect +# the distribution default of -g2. +tomcli set pyproject.toml str \ + tool.hatch.build.targets.wheel.hooks.mypyc.options.debug_level '0' +%endif %generate_buildrequires -p @@ -65,18 +83,31 @@ export SETUPTOOLS_SCM_PRETEND_VERSION='%{version}' %build -p export SETUPTOOLS_SCM_PRETEND_VERSION='%{version}' +%if %{with mypyc} +export HATCH_BUILD_HOOK_ENABLE_MYPYC='true' +%endif %install -a -install -t '%{buildroot}%{_mandir}/man1' -D -p -m 0644 '%{SOURCE2}' +install -t '%{buildroot}%{_mandir}/man1' -D -p -m 0644 '%{SOURCE1}' -# We cannot run the upstream tests because they would require data files with -# problematic license status. +%check -a +# These tests require test data from https://github.com/chardet/test-data. Many +# of the test data files appear to have problematic license status. +ignore="${ignore-} --ignore=tests/test_accuracy.py" +ignore="${ignore-} --deselect=tests/test_mime_type.py::test_none_none_files_have_correct_mime_types" +ignore="${ignore-} --deselect=tests/test_models.py::test_all_test_data_pairs_have_models" +# Don’t run benchmark (performance regression checking) tests. This package may +# be built on hardware with very different intrinsic performance from +# upstream’s development and CI hosts. +ignore="${ignore-} --ignore=tests/test_benchmark.py" + +%pytest ${ignore-} -v %files -n python3-chardet -f %{pyproject_files} -%doc README.rst +%doc README.md %{_bindir}/chardetect %{_mandir}/man1/chardetect.1* diff --git a/sources b/sources index dfc7f4b..434cae2 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (chardet-6.0.0.post1-filtered.tar.zst) = 6d40f1f19006555d8e6f7f0bea4fa1b2af7350c376d8a767a44266e1a7a501cde0fcac5408ee2ef808b626b1261dc9af774ddfe783cdc18617d2767d09cecfa6 +SHA512 (chardet-7.4.3.tar.gz) = 8d39e87ffe2dc0f24e24980e18b164629eafc962b60395ac3ad73bb41dc7461d38a66c2658c9341d06f6a178f50100edbd66a8238d5b1b7cd418783a6393efef