From 8e0d7668161707216cab8f2201017ccc9bbaba81 Mon Sep 17 00:00:00 2001 From: mdw Date: Wed, 25 Mar 2026 17:28:44 -0400 Subject: [PATCH 1/9] Do not allow zero chunk size If nbodies is smaller than 8, chunk size can be assigned a zero value. This will result in an HDF5 library error and termination. --- src/Component.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Component.cc b/src/Component.cc index dfcf394e1..a5e51b8ef 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2505,7 +2505,7 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) int chunk = H5chunk; // Sanity - if (H5chunk >= nbodies) { + if (H5chunk >= nbodies and nbodies >= 8) { chunk = nbodies/8; } @@ -2641,7 +2641,7 @@ void Component::write_H5(H5::Group& group) if (H5chunk) { // Sanity int chunk = H5chunk; - if (H5chunk >= nbodies) { + if (H5chunk >= nbodies and nbodies >= 8) { chunk = nbodies/8; } hsize_t chunk_dims[1] = {static_cast(chunk)}; From 0f783fe02d16fe1c783c370fcc982216acec6ae1 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:46:01 +0000 Subject: [PATCH 2/9] Initial plan From 739a998bfb83c0d1dc9f09c1199cad4dace5b6ef Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Wed, 25 Mar 2026 17:46:17 -0400 Subject: [PATCH 3/9] Update src/Component.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Component.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Component.cc b/src/Component.cc index a5e51b8ef..92c98ae8b 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2644,6 +2644,13 @@ void Component::write_H5(H5::Group& group) if (H5chunk >= nbodies and nbodies >= 8) { chunk = nbodies/8; } + // Ensure chunk size is valid for the dataspace: 1 <= chunk <= nbodies + if (chunk > nbodies) { + chunk = nbodies; + } + if (chunk < 1) { + chunk = 1; + } hsize_t chunk_dims[1] = {static_cast(chunk)}; dcpl.setChunk(1, chunk_dims); } From 4de6ff2db944c6cf190992d3940decd14a95c34d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 21:47:25 +0000 Subject: [PATCH 4/9] Clamp chunk to [1, nbodies] to prevent HDF5 chunk dims exceeding dataset extent Co-authored-by: The9Cat <25960766+The9Cat@users.noreply.github.com> Agent-Logs-Url: https://github.com/EXP-code/EXP/sessions/50498858-ea92-42e1-8f2b-1e11e7a0f23c --- src/Component.cc | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Component.cc b/src/Component.cc index a5e51b8ef..304b41844 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2504,10 +2504,12 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) if (H5compress or H5chunk) { int chunk = H5chunk; - // Sanity - if (H5chunk >= nbodies and nbodies >= 8) { + // Clamp chunk to [1, nbodies]: use nbodies/8 as a downsize when + // H5chunk would exceed the dataset extent, then ensure at least 1 + if (H5chunk >= nbodies) { chunk = nbodies/8; } + chunk = std::max(1, std::min(chunk, nbodies)); dcpl1.add(HighFive::Chunking(chunk)); if (H5shuffle) dcpl1.add(HighFive::Shuffle()); @@ -2639,11 +2641,13 @@ void Component::write_H5(H5::Group& group) if (H5compress or H5chunk) { // Set chunking if (H5chunk) { - // Sanity + // Clamp chunk to [1, nbodies]: use nbodies/8 as a downsize when + // H5chunk would exceed the dataset extent, then ensure at least 1 int chunk = H5chunk; - if (H5chunk >= nbodies and nbodies >= 8) { + if (H5chunk >= nbodies) { chunk = nbodies/8; } + chunk = std::max(1, std::min(chunk, nbodies)); hsize_t chunk_dims[1] = {static_cast(chunk)}; dcpl.setChunk(1, chunk_dims); } From 80c633026131414058ad791b38adb21a9f0c4c5e Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Wed, 25 Mar 2026 17:58:13 -0400 Subject: [PATCH 5/9] Update src/Component.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Component.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Component.cc b/src/Component.cc index 304b41844..a72624568 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2509,7 +2509,11 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) if (H5chunk >= nbodies) { chunk = nbodies/8; } - chunk = std::max(1, std::min(chunk, nbodies)); + if (chunk < 1) { + chunk = 1; + } else if (static_cast(chunk) > nbodies) { + chunk = static_cast(nbodies); + } dcpl1.add(HighFive::Chunking(chunk)); if (H5shuffle) dcpl1.add(HighFive::Shuffle()); From 89cfc9cefbc6e98b07898a827bea3a56a4669afa Mon Sep 17 00:00:00 2001 From: Martin Weinberg Date: Wed, 25 Mar 2026 17:58:34 -0400 Subject: [PATCH 6/9] Update src/Component.cc Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/Component.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Component.cc b/src/Component.cc index a72624568..5daccbaeb 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2651,7 +2651,7 @@ void Component::write_H5(H5::Group& group) if (H5chunk >= nbodies) { chunk = nbodies/8; } - chunk = std::max(1, std::min(chunk, nbodies)); + chunk = std::clamp(chunk, 1, static_cast(nbodies)); hsize_t chunk_dims[1] = {static_cast(chunk)}; dcpl.setChunk(1, chunk_dims); } From c19c4ce9277424cd3784ff17238956d9fc53fa89 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 22:00:08 +0000 Subject: [PATCH 7/9] Initial plan From 4a86266c88bbece5d507141970a7a4d61edc8f91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 25 Mar 2026 22:04:33 +0000 Subject: [PATCH 8/9] Fix nbodies==0 chunk dims exceeding zero-length HDF5 dataspace Co-authored-by: The9Cat <25960766+The9Cat@users.noreply.github.com> Agent-Logs-Url: https://github.com/EXP-code/EXP/sessions/010b5b10-88a6-4331-b7ed-e0d62fae5309 --- src/Component.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Component.cc b/src/Component.cc index 5daccbaeb..56e92ab26 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2501,7 +2501,7 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) auto dcplI = HighFive::DataSetCreateProps{}; auto dcplD = HighFive::DataSetCreateProps{}; - if (H5compress or H5chunk) { + if ((H5compress or H5chunk) and nbodies > 0) { int chunk = H5chunk; // Clamp chunk to [1, nbodies]: use nbodies/8 as a downsize when @@ -2642,7 +2642,7 @@ void Component::write_H5(H5::Group& group) // This could be generalized by registering a user filter, like // blosc. Right now, we're using the default (which is gzip) - if (H5compress or H5chunk) { + if ((H5compress or H5chunk) and h5_particles.size() > 0) { // Set chunking if (H5chunk) { // Clamp chunk to [1, nbodies]: use nbodies/8 as a downsize when From 1ba31df52dbe953f4aae582ddcb4b6484ab4c278 Mon Sep 17 00:00:00 2001 From: "Martin D. Weinberg" Date: Wed, 25 Mar 2026 18:22:35 -0400 Subject: [PATCH 9/9] Added some additional comments only [no ci] --- src/Component.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Component.cc b/src/Component.cc index 56e92ab26..c50884ce6 100644 --- a/src/Component.cc +++ b/src/Component.cc @@ -2501,6 +2501,10 @@ void Component::write_HDF5(HighFive::Group& group, bool masses, bool IDs) auto dcplI = HighFive::DataSetCreateProps{}; auto dcplD = HighFive::DataSetCreateProps{}; + // Compression and chunking. Do not set chunk size larger than + // nbodies. Turn off compression altogether if nbodies = 0 to avoid + // HDF5 errors. + // if ((H5compress or H5chunk) and nbodies > 0) { int chunk = H5chunk; @@ -2642,6 +2646,10 @@ void Component::write_H5(H5::Group& group) // This could be generalized by registering a user filter, like // blosc. Right now, we're using the default (which is gzip) + // + // Do not set chunk size larger than number of particles. If the + // particle number is zero, do not compress. + // if ((H5compress or H5chunk) and h5_particles.size() > 0) { // Set chunking if (H5chunk) {