From afeeff893184937aadd1c2a13b32cfb3f5d83f9f Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 15 May 2025 15:19:22 -0400 Subject: [PATCH 1/6] Update pyo3 dependency to 0.25 --- Cargo.lock | 21 ++++++++++----------- Cargo.toml | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 45c1da51..52c0011b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,11 +544,10 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" +checksum = "f239d656363bcee73afef85277f1b281e8ac6212a1d42aa90e55b90ed43c47a4" dependencies = [ - "cfg-if", "indoc", "libc", "memoffset", @@ -562,9 +561,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" +checksum = "755ea671a1c34044fa165247aaf6f419ca39caa6003aee791a0df2713d8f1b6d" dependencies = [ "once_cell", "python3-dll-a", @@ -573,9 +572,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" +checksum = "fc95a2e67091e44791d4ea300ff744be5293f394f1bafd9f78c080814d35956e" dependencies = [ "libc", "pyo3-build-config", @@ -583,9 +582,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" +checksum = "a179641d1b93920829a62f15e87c0ed791b6c8db2271ba0fd7c2686090510214" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -595,9 +594,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" +checksum = "9dff85ebcaab8c441b0e3f7ae40a6963ecea8a9f5e74f647e33fcf5ec9a1e89e" dependencies = [ "heck 0.5.0", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index a2cd5389..d74705e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,11 +65,11 @@ wasm32-compat = ["libcramjam/wasm32-compat"] [dependencies] -pyo3 = { version = "^0.24", default-features = false, features = ["macros"] } +pyo3 = { version = "^0.25", default-features = false, features = ["macros"] } libcramjam = { version = "^0.7", default-features = false } [build-dependencies] -pyo3-build-config = "^0.24" +pyo3-build-config = "^0.25" [profile.release] strip = true From ea61683827df24b636fa4b4bb70119b1eb7a85e8 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Thu, 15 May 2025 15:55:01 -0400 Subject: [PATCH 2/6] use a mutable bytearray in reference cleanup test to avoid immortalization --- tests/test_buffer_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_buffer_view.py b/tests/test_buffer_view.py index 9776bdce..46cc2eda 100644 --- a/tests/test_buffer_view.py +++ b/tests/test_buffer_view.py @@ -99,7 +99,7 @@ def test_buffer_view_cleanup(): n_refs = 0 def get_buffer(): - data = b"bytes" + data = bytearray(b"bytes") buf = cramjam.Buffer(data, copy=False) nonlocal n_refs From aa54ac6678c7b18d53b40d0214f9d6c2604ead4d Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Sun, 18 May 2025 10:53:31 -0400 Subject: [PATCH 3/6] Use a bytearray in test that mutates a buffer --- tests/test_buffer_view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_buffer_view.py b/tests/test_buffer_view.py index 46cc2eda..bc0fb4df 100644 --- a/tests/test_buffer_view.py +++ b/tests/test_buffer_view.py @@ -37,7 +37,7 @@ def test_buffer_view_raises_when_writing_past_data_length_at_once(): @pytest.mark.skip_pypy def test_buffer_view_raises_when_writing_past_data_length_incrementally(): - data = b"bytes" + data = bytearray(b"bytes") buf = Buffer(data, copy=False) # This is okay, up to length of underlying buffer From becf091553eccbe5f778f387c8557233ca3638d2 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Sun, 18 May 2025 11:46:30 -0400 Subject: [PATCH 4/6] make tests pass on the free-threaded build --- src/io.rs | 28 ++++++++++++++++++---------- tests/conftest.py | 5 +++++ tests/test_buffer_view.py | 6 +++--- tests/test_rust_io.py | 4 ++-- tests/test_variants.py | 12 ++++++------ 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/io.rs b/src/io.rs index 2e772ddf..7d552dd0 100644 --- a/src/io.rs +++ b/src/io.rs @@ -14,6 +14,7 @@ use pyo3::exceptions::{self, PyBufferError}; use pyo3::ffi; use pyo3::prelude::*; use pyo3::types::PyBytes; +use pyo3::IntoPyObjectExt; use std::path::PathBuf; pub(crate) trait AsBytes { @@ -176,8 +177,8 @@ impl RustyFile { pub struct PythonBuffer { pub(crate) inner: std::pin::Pin>, pub(crate) pos: usize, - #[cfg(PyPy)] - pub(crate) owner: PyObject, + #[cfg(any(PyPy, Py_GIL_DISABLED))] + pub(crate) owner: Py, } // PyBuffer is thread-safe: the shape of the buffer is immutable while a Py_buffer exists. // Accessing the buffer contents is protected using the GIL. @@ -207,14 +208,23 @@ impl PythonBuffer { } /// Get the underlying buffer as a mutable slice of bytes pub fn as_slice_mut(&mut self) -> PyResult<&mut [u8]> { - #[cfg(PyPy)] + #[cfg(any(PyPy, Py_GIL_DISABLED))] { Python::with_gil(|py| { let is_memoryview = unsafe { ffi::PyMemoryView_Check(self.owner.as_ptr()) } == 1; if is_memoryview || self.owner.bind(py).is_instance_of::() { - Err(pyo3::exceptions::PyTypeError::new_err( - "With PyPy, an output of type `bytes` or `memoryview` does not work. See issue pypy/pypy#4918", - )) + #[cfg(PyPy)] + { + Err(pyo3::exceptions::PyTypeError::new_err( + "Cannot create mutable reference to `bytes` or `memoryview` on PyPy. See issue pypy/pypy#4918", + )) + } + #[cfg(Py_GIL_DISABLED)] + { + Err(pyo3::exceptions::PyTypeError::new_err( + "Cannot create mutable reference to `bytes` or `memoryview` on the free-threaded build. See issue milesgranger/cramjam#213" + )) + } } else { Ok(()) } @@ -274,8 +284,8 @@ impl<'a, 'py> TryFrom<&'a Bound<'py, PyAny>> for PythonBuffer { let buf = Self { inner: std::pin::Pin::from(buf), pos: 0, - #[cfg(PyPy)] - owner: Python::with_gil(|py| obj.to_object(py)), + #[cfg(any(PyPy, Py_GIL_DISABLED))] + owner: Python::with_gil(|py| obj.into_py_any(py).unwrap()), }; // sanity checks if buf.inner.shape.is_null() { @@ -412,7 +422,6 @@ impl RustyBuffer { pub fn __init__(py: Python, mut data: Option>, copy: Option) -> PyResult { if let Some(maybe_bytestype) = data.as_mut() { let mut bytestype = maybe_bytestype.extract::>(py)?; - if copy.unwrap_or(true) { let mut buf = vec![]; bytestype.read_to_end(&mut buf)?; @@ -648,7 +657,6 @@ impl RustyBuffer { (*view).buf = bytes.as_ptr() as *mut std::os::raw::c_void; (*view).len = bytes.len() as isize; (*view).readonly = 0; - (*view).itemsize = 1; (*view).format = std::ptr::null_mut(); if (flags & ffi::PyBUF_FORMAT) == ffi::PyBUF_FORMAT { diff --git a/tests/conftest.py b/tests/conftest.py index 408bb15b..32674325 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import platform +import sysconfig import pytest @@ -8,6 +9,10 @@ def is_pypy(): impl = platform.python_implementation() return impl.lower() == "pypy" +@pytest.fixture(scope="session") +def is_free_threaded(): + return bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + def pytest_configure(config): config.addinivalue_line("markers", "skip_pypy: skip this test on PyPy") diff --git a/tests/test_buffer_view.py b/tests/test_buffer_view.py index bc0fb4df..dc499895 100644 --- a/tests/test_buffer_view.py +++ b/tests/test_buffer_view.py @@ -12,7 +12,7 @@ def test_buffer_view(copy): if copy is not None: kwargs["copy"] = copy - data = b"bytes" + data = bytearray(b"bytes") buf = Buffer(data, **kwargs) buf.write(b"0") @@ -26,7 +26,7 @@ def test_buffer_view(copy): @pytest.mark.skip_pypy def test_buffer_view_raises_when_writing_past_data_length_at_once(): - data = b"bytes" + data = bytearray(b"bytes") buf = Buffer(data, copy=False) # Won't write pasted underlying buffer if passed data all at once @@ -74,7 +74,7 @@ def test_buffer_view_raises_when_truncating(): @pytest.mark.skip_pypy @pytest.mark.parametrize("whence", (0, 1, 2)) def test_buffer_view_raises_when_write_after_bad_seek(whence): - buf = Buffer(b"bytes", copy=False) + buf = Buffer(bytearray(b"bytes"), copy=False) buf.seek(2, whence=0) # Seek forward 2 from start, also okay buf.seek(2, whence=1) # Seek forward 2 from current position, okay diff --git a/tests/test_rust_io.py b/tests/test_rust_io.py index 6737262e..72b5e3e1 100644 --- a/tests/test_rust_io.py +++ b/tests/test_rust_io.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize("Obj", (File, Buffer)) -def test_obj_api(tmpdir, Obj, is_pypy): +def test_obj_api(tmpdir, Obj, is_pypy, is_free_threaded): if isinstance(Obj, File): buf = File(str(tmpdir.join("file.txt"))) else: @@ -30,7 +30,7 @@ def test_obj_api(tmpdir, Obj, is_pypy): ): buf.seek(0) - if isinstance(out, bytes) and is_pypy: + if isinstance(out, bytes) and (is_pypy or is_free_threaded): with pytest.raises(OSError): buf.readinto(out) continue diff --git a/tests/test_variants.py b/tests/test_variants.py index 1afa2c5b..740913f4 100644 --- a/tests/test_variants.py +++ b/tests/test_variants.py @@ -106,7 +106,7 @@ def test_variants_raise_exception(variant_str): @pytest.mark.parametrize("variant_str", VARIANTS) @given(raw_data=st.binary()) def test_variants_compress_into( - variant_str, input_type, output_type, raw_data, tmp_path_factory, is_pypy + variant_str, input_type, output_type, raw_data, tmp_path_factory, is_pypy, is_free_threaded ): # TODO: Fix segfault when using blosc2 compress_into cramjam.File # decompress_into appears to work fine. @@ -148,9 +148,9 @@ def test_variants_compress_into( else: output = output_type(b"0" * compressed_len) - if is_pypy and isinstance(output, (bytes, memoryview)): + if (is_pypy or is_free_threaded) and isinstance(output, (bytes, memoryview)): pytest.xfail( - reason="PyPy de/compress_into w/ bytes or memoryview is a bit flaky behavior" + reason="Decompressing into immutable objects is not supported on PyPy or the free-threaded build" ) n_bytes = variant.compress_into(input, output) @@ -176,7 +176,7 @@ def test_variants_compress_into( @pytest.mark.parametrize("variant_str", VARIANTS) @given(raw_data=st.binary()) def test_variants_decompress_into( - variant_str, input_type, output_type, tmp_path_factory, raw_data, is_pypy + variant_str, input_type, output_type, tmp_path_factory, raw_data, is_pypy, is_free_threaded ): if variant_str == "izlib" and output_type == "memoryview": pytest.skip("See issue https://github.com/milesgranger/cramjam/issues/193") @@ -213,9 +213,9 @@ def test_variants_decompress_into( else: output = output_type(b"0" * len(raw_data)) - if is_pypy and isinstance(output, (bytes, memoryview)): + if (is_pypy or is_free_threaded) and isinstance(output, (bytes, memoryview)): pytest.xfail( - reason="PyPy de/compress_into w/ bytes or memoryview is a bit flaky behavior" + reason="Decompressing into immutable objects is not supported on PyPy or the free-threaded build" ) n_bytes = variant.decompress_into(input, output) From 828b616c4ae34dd3dad0eac266316cf7d79b584c Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 20 May 2025 13:20:43 -0400 Subject: [PATCH 5/6] test only fails on 3.14 --- tests/test_rust_io.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_rust_io.py b/tests/test_rust_io.py index 72b5e3e1..ae329691 100644 --- a/tests/test_rust_io.py +++ b/tests/test_rust_io.py @@ -1,3 +1,5 @@ +import sys + import pytest from cramjam import File, Buffer @@ -30,7 +32,8 @@ def test_obj_api(tmpdir, Obj, is_pypy, is_free_threaded): ): buf.seek(0) - if isinstance(out, bytes) and (is_pypy or is_free_threaded): + if isinstance(out, bytes) and ( + is_pypy or (is_free_threaded and sys.version_info >= (3, 14))): with pytest.raises(OSError): buf.readinto(out) continue From c89e7e010fe038d290cdd12cc8c0135992bce7db Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 26 May 2025 11:22:25 -0600 Subject: [PATCH 6/6] relax hypothesis pin --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0584e0bd..7e77e5be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,5 +31,5 @@ dev = [ "pytest>=5.30", "pytest-xdist", "pytest-benchmark", - "hypothesis<6.123.0" + "hypothesis" ]