71 lines
2.8 KiB
Diff
71 lines
2.8 KiB
Diff
From 83c6cdfa63ad61accec35575f503b629f94dc4af Mon Sep 17 00:00:00 2001
|
|
From: Cary Phillips <cary@ilm.com>
|
|
Date: Thu, 19 Mar 2026 14:22:40 -0700
|
|
Subject: [PATCH] Fix B44/B44A integer overflow: use uint64_t for row offset
|
|
(#2312)
|
|
|
|
The B44 and B44A decoder and encoder use channel width (`nx`) and
|
|
height (`ny`) in row pointer math. `nx` and `ny` are `int`; the
|
|
scratch buffer is correctly sized with `(uint64_t)ny * (uint64_t)nx *
|
|
bytes_per_element`, but row bases were computed as:
|
|
|
|
```
|
|
row0 = (uint16_t*)scratch;
|
|
row0 += y * nx; // int * int -> signed overflow when y*nx > INT_MAX
|
|
```
|
|
|
|
For large `nx` (e.g. 268435456), `y*nx` overflows, so `row0`/`row1`/`row2`/`row3`
|
|
point before the scratch buffer.
|
|
|
|
Fix: compute the row offset in `uint64_t` before pointer arithmetic in both
|
|
`uncompress_b44_impl` (decoder) and `compress_b44_impl` (encoder).
|
|
|
|
Analysis and solution with the help of Curor / Claude Opus 4.5
|
|
|
|
Signed-off-by: Cary Phillips <cary@ilm.com>
|
|
---
|
|
.../OpenEXR/OpenEXRCore/internal_b44.c | 23 ++++++++++---------
|
|
1 file changed, 12 insertions(+), 11 deletions(-)
|
|
|
|
diff --git a/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c b/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c
|
|
index 8d0c257e658..93279ff2ed5 100644
|
|
--- a/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c
|
|
+++ b/pxr/imaging/plugin/hioOpenEXR/OpenEXR/OpenEXRCore/internal_b44.c
|
|
@@ -390,13 +390,13 @@ compress_b44_impl (exr_encode_pipeline_t* encode, int flat_field)
|
|
// rightmost column and the bottom row.
|
|
//
|
|
uint16_t *row0, *row1, *row2, *row3;
|
|
+ /* row offset in elements: use uint64_t so y*nx cannot overflow int */
|
|
+ uint64_t row_off = (uint64_t) (y) * (uint64_t) (nx);
|
|
|
|
- row0 = (uint16_t*) scratch;
|
|
- row0 += y * nx;
|
|
-
|
|
- row1 = row0 + nx;
|
|
- row2 = row1 + nx;
|
|
- row3 = row2 + nx;
|
|
+ row0 = (uint16_t*) scratch + row_off;
|
|
+ row1 = row0 + (uint64_t) nx;
|
|
+ row2 = row1 + (uint64_t) nx;
|
|
+ row3 = row2 + (uint64_t) nx;
|
|
|
|
if (y + 3 >= ny)
|
|
{
|
|
@@ -512,11 +512,12 @@ uncompress_b44_impl (
|
|
|
|
for (int y = 0; y < ny; y += 4)
|
|
{
|
|
- row0 = (uint16_t*) scratch;
|
|
- row0 += y * nx;
|
|
- row1 = row0 + nx;
|
|
- row2 = row1 + nx;
|
|
- row3 = row2 + nx;
|
|
+ /* row offset in elements: use uint64_t so y*nx cannot overflow int */
|
|
+ uint64_t row_off = (uint64_t) (y) * (uint64_t) (nx);
|
|
+ row0 = (uint16_t*) scratch + row_off;
|
|
+ row1 = row0 + (uint64_t) nx;
|
|
+ row2 = row1 + (uint64_t) nx;
|
|
+ row3 = row2 + (uint64_t) nx;
|
|
for (int x = 0; x < nx; x += 4)
|
|
{
|
|
if (bIn + 3 > comp_buf_size) return EXR_ERR_OUT_OF_MEMORY;
|