From 9e84829e8d035669a9779e95c7773c70b2245e4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Savi=C4=87?= Date: Thu, 20 Aug 2026 09:45:12 +0000 Subject: [PATCH 1/2] Allow theta sketches with lg_k = 4 to match Java The C++ theta implementation used a single constant, theta_constants::MIN_LG_K = 5, for two distinct roles: the floor on the user-requested nominal size (K) and the floor on the internal hash table (cache) size. Java keeps these separate -- ThetaUtil.MIN_LG_NOM_LONGS = 4 and MIN_LG_ARR_LONGS = 5 -- so Java accepts lg_k = 4 (nominal 16) while C++ rejected it in theta_base_builder:: set_lg_k. As a result, C++ could not *construct* a theta/tuple update sketch or a union at lg_k = 4, even though Java can. (Reading was unaffected: the compact serialization format does not carry lg_nom, so C++ already deserializes Java-produced lg = 4 compact sketches -- this only concerns building them.) Split the constant: MIN_LG_K becomes 4 (the nominal floor validated by set_lg_k, shared by update/union/tuple builders), and a new MIN_LG_ARR = 5 keeps the hash table floored at 32 slots via starting_sub_multiple(). A lg_k = 4 sketch therefore starts with lg_cur_size (5) > lg_nom_size (4) and only ever rebuilds down to the nominal 16, which the existing update/rebuild path already handles. Add tests: building at the minimum lg_k (and rejecting below it), estimation + trim + serialization round trip for an update sketch, and a union built at the minimum lg_k. Co-authored-by: Isaac --- theta/include/theta_constants.hpp | 8 +++-- .../include/theta_update_sketch_base_impl.hpp | 4 +-- theta/test/theta_sketch_test.cpp | 30 +++++++++++++++++++ theta/test/theta_union_test.cpp | 23 ++++++++++++++ 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/theta/include/theta_constants.hpp b/theta/include/theta_constants.hpp index fc97e35a..cdf2001f 100644 --- a/theta/include/theta_constants.hpp +++ b/theta/include/theta_constants.hpp @@ -34,10 +34,14 @@ namespace theta_constants { /// max theta - signed max for compatibility with Java const uint64_t MAX_THETA = LLONG_MAX; - /// min log2 of K - const uint8_t MIN_LG_K = 5; + /// min log2 of nominal entries (K) + const uint8_t MIN_LG_K = 4; /// max log2 of K const uint8_t MAX_LG_K = 26; + /// min log2 of the internal hash table (cache) size. The hash table needs headroom above the + /// nominal size to hold transient entries before rebuild, so its floor is higher than MIN_LG_K. + /// Matches Java's ThetaUtil.MIN_LG_ARR_LONGS. + const uint8_t MIN_LG_ARR = 5; /// default log2 of K const uint8_t DEFAULT_LG_K = 12; } diff --git a/theta/include/theta_update_sketch_base_impl.hpp b/theta/include/theta_update_sketch_base_impl.hpp index 2a4b0297..e734cae3 100644 --- a/theta/include/theta_update_sketch_base_impl.hpp +++ b/theta/include/theta_update_sketch_base_impl.hpp @@ -265,7 +265,7 @@ void theta_update_sketch_base::reset() { } } const uint8_t starting_lg_size = theta_build_helper::starting_sub_multiple( - lg_nom_size_ + 1, theta_constants::MIN_LG_K, static_cast(rf_)); + lg_nom_size_ + 1, theta_constants::MIN_LG_ARR, static_cast(rf_)); if (starting_lg_size != lg_cur_size_) { allocator_.deallocate(entries_, cur_size); lg_cur_size_ = starting_lg_size; @@ -346,7 +346,7 @@ uint64_t theta_base_builder::starting_theta() const { template uint8_t theta_base_builder::starting_lg_size() const { - return theta_build_helper::starting_sub_multiple(lg_k_ + 1, theta_constants::MIN_LG_K, static_cast(rf_)); + return theta_build_helper::starting_sub_multiple(lg_k_ + 1, theta_constants::MIN_LG_ARR, static_cast(rf_)); } // iterator diff --git a/theta/test/theta_sketch_test.cpp b/theta/test/theta_sketch_test.cpp index 97c4f14e..0dfa6985 100644 --- a/theta/test/theta_sketch_test.cpp +++ b/theta/test/theta_sketch_test.cpp @@ -57,6 +57,36 @@ TEST_CASE("theta sketch: empty", "[theta_sketch]") { REQUIRE(update_sketch.compact(false).is_ordered()); } +TEST_CASE("theta sketch: min lg_k", "[theta_sketch]") { + // lg_k = 4 (nominal 16) is the smallest allowed nominal size, matching Java's + // ThetaUtil.MIN_LG_NOM_LONGS. Anything below it must throw. + REQUIRE(theta_constants::MIN_LG_K == 4); + REQUIRE_THROWS_AS(update_theta_sketch::builder().set_lg_k(theta_constants::MIN_LG_K - 1), + std::invalid_argument); + update_theta_sketch min_sketch = update_theta_sketch::builder().set_lg_k(theta_constants::MIN_LG_K).build(); + REQUIRE(min_sketch.get_lg_k() == theta_constants::MIN_LG_K); + + // update well past the nominal size to force estimation mode and exercise the rebuild path. + // The internal hash table floor stays at MIN_LG_ARR (5, i.e. 32 slots), one above the nominal. + const int n = 10000; + for (int i = 0; i < n; ++i) min_sketch.update(i); + REQUIRE(min_sketch.is_estimation_mode()); + REQUIRE(min_sketch.get_theta() < 1.0); + // the true count is bracketed by the 2-standard-deviation confidence bounds + REQUIRE(min_sketch.get_lower_bound(2) <= n); + REQUIRE(min_sketch.get_upper_bound(2) >= n); + + // trimming reduces the sketch to exactly the nominal number of entries (2^4 = 16) + min_sketch.trim(); + REQUIRE(min_sketch.get_num_retained() == (1 << theta_constants::MIN_LG_K)); + + // a compacted min-size sketch round trips through serialization + auto bytes = min_sketch.compact().serialize(); + compact_theta_sketch deserialized = compact_theta_sketch::deserialize(bytes.data(), bytes.size()); + REQUIRE(deserialized.get_num_retained() == min_sketch.get_num_retained()); + REQUIRE(deserialized.get_estimate() == min_sketch.get_estimate()); +} + TEST_CASE("theta sketch: non empty no retained keys", "[theta_sketch]") { update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001f).build(); update_sketch.update(1); diff --git a/theta/test/theta_union_test.cpp b/theta/test/theta_union_test.cpp index aef1c312..09a1c6ec 100644 --- a/theta/test/theta_union_test.cpp +++ b/theta/test/theta_union_test.cpp @@ -76,6 +76,29 @@ TEST_CASE("theta union: exact mode half overlap", "[theta_union]") { REQUIRE_FALSE(sketch3.is_estimation_mode()); } +TEST_CASE("theta union: min lg_k", "[theta_union]") { + // A union built with the minimum lg_k = 4 (nominal 16) must build and produce a valid result, + // matching Java. Requesting a smaller lg_k must throw. + REQUIRE_THROWS_AS(theta_union::builder().set_lg_k(theta_constants::MIN_LG_K - 1), + std::invalid_argument); + + auto sketch1 = update_theta_sketch::builder().build(); + for (int i = 0; i < 10000; i++) sketch1.update(i); + auto sketch2 = update_theta_sketch::builder().build(); + for (int i = 5000; i < 15000; i++) sketch2.update(i); + + auto u = theta_union::builder().set_lg_k(theta_constants::MIN_LG_K).build(); + u.update(sketch1); + u.update(sketch2); + auto result = u.get_result(); + REQUIRE(result.is_estimation_mode()); + REQUIRE(result.get_theta() < 1.0); + // the true union cardinality (15000 distinct values) is bracketed by the confidence bounds + const int distinct = 15000; + REQUIRE(result.get_lower_bound(2) <= distinct); + REQUIRE(result.get_upper_bound(2) >= distinct); +} + TEST_CASE("theta union: exact mode half overlap wrapped compact", "[theta_union]") { auto sketch1 = update_theta_sketch::builder().build(); int value = 0; From 689ffa075ece03c8387a9f65092be49d7d0d65ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Savi=C4=87?= Date: Thu, 20 Aug 2026 12:12:11 +0000 Subject: [PATCH 2/2] Ajmo testovi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Savić --- theta/include/theta_constants.hpp | 4 +-- theta/test/theta_sketch_test.cpp | 18 ++++++++++--- theta/test/theta_union_test.cpp | 23 ---------------- tuple/test/tuple_sketch_test.cpp | 44 +++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/theta/include/theta_constants.hpp b/theta/include/theta_constants.hpp index cdf2001f..2e9b2e35 100644 --- a/theta/include/theta_constants.hpp +++ b/theta/include/theta_constants.hpp @@ -38,9 +38,7 @@ namespace theta_constants { const uint8_t MIN_LG_K = 4; /// max log2 of K const uint8_t MAX_LG_K = 26; - /// min log2 of the internal hash table (cache) size. The hash table needs headroom above the - /// nominal size to hold transient entries before rebuild, so its floor is higher than MIN_LG_K. - /// Matches Java's ThetaUtil.MIN_LG_ARR_LONGS. + /// min log2 of cache size const uint8_t MIN_LG_ARR = 5; /// default log2 of K const uint8_t DEFAULT_LG_K = 12; diff --git a/theta/test/theta_sketch_test.cpp b/theta/test/theta_sketch_test.cpp index 0dfa6985..95df9a92 100644 --- a/theta/test/theta_sketch_test.cpp +++ b/theta/test/theta_sketch_test.cpp @@ -66,12 +66,24 @@ TEST_CASE("theta sketch: min lg_k", "[theta_sketch]") { update_theta_sketch min_sketch = update_theta_sketch::builder().set_lg_k(theta_constants::MIN_LG_K).build(); REQUIRE(min_sketch.get_lg_k() == theta_constants::MIN_LG_K); - // update well past the nominal size to force estimation mode and exercise the rebuild path. - // The internal hash table floor stays at MIN_LG_ARR (5, i.e. 32 slots), one above the nominal. + // update well past the nominal size to force estimation mode and exercise the rebuild path, + // tracking the peak number of retained entries seen between rebuilds. const int n = 10000; - for (int i = 0; i < n; ++i) min_sketch.update(i); + uint32_t max_retained = 0; + for (int i = 0; i < n; ++i) { + min_sketch.update(i); + max_retained = std::max(max_retained, min_sketch.get_num_retained()); + } REQUIRE(min_sketch.is_estimation_mode()); REQUIRE(min_sketch.get_theta() < 1.0); + + // The internal hash table is floored at MIN_LG_ARR (5, i.e. 32 slots), one lg above the + // nominal size. This is exactly what MIN_LG_ARR guarantees: between rebuilds the sketch holds + // more than the nominal 2^MIN_LG_K (16) entries, but never more than the 2^MIN_LG_ARR (32) + // slots of the table. Were the table sized to the nominal 16, it could not retain more than 16. + REQUIRE(max_retained > (1 << theta_constants::MIN_LG_K)); + REQUIRE(max_retained <= (1 << theta_constants::MIN_LG_ARR)); + // the true count is bracketed by the 2-standard-deviation confidence bounds REQUIRE(min_sketch.get_lower_bound(2) <= n); REQUIRE(min_sketch.get_upper_bound(2) >= n); diff --git a/theta/test/theta_union_test.cpp b/theta/test/theta_union_test.cpp index 09a1c6ec..aef1c312 100644 --- a/theta/test/theta_union_test.cpp +++ b/theta/test/theta_union_test.cpp @@ -76,29 +76,6 @@ TEST_CASE("theta union: exact mode half overlap", "[theta_union]") { REQUIRE_FALSE(sketch3.is_estimation_mode()); } -TEST_CASE("theta union: min lg_k", "[theta_union]") { - // A union built with the minimum lg_k = 4 (nominal 16) must build and produce a valid result, - // matching Java. Requesting a smaller lg_k must throw. - REQUIRE_THROWS_AS(theta_union::builder().set_lg_k(theta_constants::MIN_LG_K - 1), - std::invalid_argument); - - auto sketch1 = update_theta_sketch::builder().build(); - for (int i = 0; i < 10000; i++) sketch1.update(i); - auto sketch2 = update_theta_sketch::builder().build(); - for (int i = 5000; i < 15000; i++) sketch2.update(i); - - auto u = theta_union::builder().set_lg_k(theta_constants::MIN_LG_K).build(); - u.update(sketch1); - u.update(sketch2); - auto result = u.get_result(); - REQUIRE(result.is_estimation_mode()); - REQUIRE(result.get_theta() < 1.0); - // the true union cardinality (15000 distinct values) is bracketed by the confidence bounds - const int distinct = 15000; - REQUIRE(result.get_lower_bound(2) <= distinct); - REQUIRE(result.get_upper_bound(2) >= distinct); -} - TEST_CASE("theta union: exact mode half overlap wrapped compact", "[theta_union]") { auto sketch1 = update_theta_sketch::builder().build(); int value = 0; diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp index 6e44e284..1870fec3 100644 --- a/tuple/test/tuple_sketch_test.cpp +++ b/tuple/test/tuple_sketch_test.cpp @@ -19,6 +19,7 @@ #include #include +#include namespace datasketches { @@ -49,6 +50,49 @@ TEST_CASE("tuple sketch float: builder", "[tuple_sketch]") { REQUIRE(sketch.get_theta() == 0.5); // theta = p } +TEST_CASE("tuple sketch: min lg_k", "[tuple_sketch]") { + // Tuple sketches reuse theta's builder and hash table, so the same nominal floor + // (MIN_LG_K = 4, matching Java's ThetaUtil.MIN_LG_NOM_LONGS) and cache floor (MIN_LG_ARR = 5) + // apply. lg_k = 4 (nominal 16) is the smallest allowed nominal size; below it must throw. + REQUIRE(theta_constants::MIN_LG_K == 4); + REQUIRE_THROWS_AS(update_tuple_sketch::builder().set_lg_k(theta_constants::MIN_LG_K - 1), + std::invalid_argument); + auto min_sketch = update_tuple_sketch::builder().set_lg_k(theta_constants::MIN_LG_K).build(); + REQUIRE(min_sketch.get_lg_k() == theta_constants::MIN_LG_K); + + // update well past the nominal size to force estimation mode and exercise the rebuild path, + // tracking the peak number of retained entries seen between rebuilds. + const int n = 10000; + uint32_t max_retained = 0; + for (int i = 0; i < n; ++i) { + min_sketch.update(i, 1.0f); + if (min_sketch.get_num_retained() > max_retained) max_retained = min_sketch.get_num_retained(); + } + REQUIRE(min_sketch.is_estimation_mode()); + REQUIRE(min_sketch.get_theta() < 1.0); + + // The internal hash table is floored at MIN_LG_ARR (5, i.e. 32 slots), one lg above the + // nominal size. This is exactly what MIN_LG_ARR guarantees: between rebuilds the sketch holds + // more than the nominal 2^MIN_LG_K (16) entries, but never more than the 2^MIN_LG_ARR (32) + // slots of the table. Were the table sized to the nominal 16, it could not retain more than 16. + REQUIRE(max_retained > (1 << theta_constants::MIN_LG_K)); + REQUIRE(max_retained <= (1 << theta_constants::MIN_LG_ARR)); + + // the true count is bracketed by the 2-standard-deviation confidence bounds + REQUIRE(min_sketch.get_lower_bound(2) <= n); + REQUIRE(min_sketch.get_upper_bound(2) >= n); + + // trimming reduces the sketch to exactly the nominal number of entries (2^4 = 16) + min_sketch.trim(); + REQUIRE(min_sketch.get_num_retained() == (1 << theta_constants::MIN_LG_K)); + + // a compacted min-size sketch round trips through serialization + auto bytes = min_sketch.compact().serialize(); + auto deserialized = compact_tuple_sketch::deserialize(bytes.data(), bytes.size()); + REQUIRE(deserialized.get_num_retained() == min_sketch.get_num_retained()); + REQUIRE(deserialized.get_estimate() == min_sketch.get_estimate()); +} + TEST_CASE("tuple sketch float: empty", "[tuple_sketch]") { auto update_sketch = update_tuple_sketch::builder().build(); std::cout << "sizeof(update_tuple_sketch)=" << sizeof(update_sketch) << std::endl;