diff --git a/theta/include/theta_constants.hpp b/theta/include/theta_constants.hpp index fc97e35a..2e9b2e35 100644 --- a/theta/include/theta_constants.hpp +++ b/theta/include/theta_constants.hpp @@ -34,10 +34,12 @@ namespace theta_constants { /// max theta - signed max for compatibility with Java const uint64_t MAX_THETA = LLONG_MAX; - /// min log2 of K - const uint8_t MIN_LG_K = 5; + /// min log2 of nominal entries (K) + const uint8_t MIN_LG_K = 4; /// max log2 of K const uint8_t MAX_LG_K = 26; + /// min log2 of cache size + const uint8_t MIN_LG_ARR = 5; /// default log2 of K const uint8_t DEFAULT_LG_K = 12; } diff --git a/theta/include/theta_update_sketch_base_impl.hpp b/theta/include/theta_update_sketch_base_impl.hpp index 2a4b0297..e734cae3 100644 --- a/theta/include/theta_update_sketch_base_impl.hpp +++ b/theta/include/theta_update_sketch_base_impl.hpp @@ -265,7 +265,7 @@ void theta_update_sketch_base::reset() { } } const uint8_t starting_lg_size = theta_build_helper::starting_sub_multiple( - lg_nom_size_ + 1, theta_constants::MIN_LG_K, static_cast(rf_)); + lg_nom_size_ + 1, theta_constants::MIN_LG_ARR, static_cast(rf_)); if (starting_lg_size != lg_cur_size_) { allocator_.deallocate(entries_, cur_size); lg_cur_size_ = starting_lg_size; @@ -346,7 +346,7 @@ uint64_t theta_base_builder::starting_theta() const { template uint8_t theta_base_builder::starting_lg_size() const { - return theta_build_helper::starting_sub_multiple(lg_k_ + 1, theta_constants::MIN_LG_K, static_cast(rf_)); + return theta_build_helper::starting_sub_multiple(lg_k_ + 1, theta_constants::MIN_LG_ARR, static_cast(rf_)); } // iterator diff --git a/theta/test/theta_sketch_test.cpp b/theta/test/theta_sketch_test.cpp index 97c4f14e..95df9a92 100644 --- a/theta/test/theta_sketch_test.cpp +++ b/theta/test/theta_sketch_test.cpp @@ -57,6 +57,48 @@ TEST_CASE("theta sketch: empty", "[theta_sketch]") { REQUIRE(update_sketch.compact(false).is_ordered()); } +TEST_CASE("theta sketch: min lg_k", "[theta_sketch]") { + // lg_k = 4 (nominal 16) is the smallest allowed nominal size, matching Java's + // ThetaUtil.MIN_LG_NOM_LONGS. Anything below it must throw. + REQUIRE(theta_constants::MIN_LG_K == 4); + REQUIRE_THROWS_AS(update_theta_sketch::builder().set_lg_k(theta_constants::MIN_LG_K - 1), + std::invalid_argument); + update_theta_sketch min_sketch = update_theta_sketch::builder().set_lg_k(theta_constants::MIN_LG_K).build(); + REQUIRE(min_sketch.get_lg_k() == theta_constants::MIN_LG_K); + + // update well past the nominal size to force estimation mode and exercise the rebuild path, + // tracking the peak number of retained entries seen between rebuilds. + const int n = 10000; + uint32_t max_retained = 0; + for (int i = 0; i < n; ++i) { + min_sketch.update(i); + max_retained = std::max(max_retained, min_sketch.get_num_retained()); + } + REQUIRE(min_sketch.is_estimation_mode()); + REQUIRE(min_sketch.get_theta() < 1.0); + + // The internal hash table is floored at MIN_LG_ARR (5, i.e. 32 slots), one lg above the + // nominal size. This is exactly what MIN_LG_ARR guarantees: between rebuilds the sketch holds + // more than the nominal 2^MIN_LG_K (16) entries, but never more than the 2^MIN_LG_ARR (32) + // slots of the table. Were the table sized to the nominal 16, it could not retain more than 16. + REQUIRE(max_retained > (1 << theta_constants::MIN_LG_K)); + REQUIRE(max_retained <= (1 << theta_constants::MIN_LG_ARR)); + + // the true count is bracketed by the 2-standard-deviation confidence bounds + REQUIRE(min_sketch.get_lower_bound(2) <= n); + REQUIRE(min_sketch.get_upper_bound(2) >= n); + + // trimming reduces the sketch to exactly the nominal number of entries (2^4 = 16) + min_sketch.trim(); + REQUIRE(min_sketch.get_num_retained() == (1 << theta_constants::MIN_LG_K)); + + // a compacted min-size sketch round trips through serialization + auto bytes = min_sketch.compact().serialize(); + compact_theta_sketch deserialized = compact_theta_sketch::deserialize(bytes.data(), bytes.size()); + REQUIRE(deserialized.get_num_retained() == min_sketch.get_num_retained()); + REQUIRE(deserialized.get_estimate() == min_sketch.get_estimate()); +} + TEST_CASE("theta sketch: non empty no retained keys", "[theta_sketch]") { update_theta_sketch update_sketch = update_theta_sketch::builder().set_p(0.001f).build(); update_sketch.update(1); diff --git a/tuple/test/tuple_sketch_test.cpp b/tuple/test/tuple_sketch_test.cpp index 6e44e284..1870fec3 100644 --- a/tuple/test/tuple_sketch_test.cpp +++ b/tuple/test/tuple_sketch_test.cpp @@ -19,6 +19,7 @@ #include #include +#include namespace datasketches { @@ -49,6 +50,49 @@ TEST_CASE("tuple sketch float: builder", "[tuple_sketch]") { REQUIRE(sketch.get_theta() == 0.5); // theta = p } +TEST_CASE("tuple sketch: min lg_k", "[tuple_sketch]") { + // Tuple sketches reuse theta's builder and hash table, so the same nominal floor + // (MIN_LG_K = 4, matching Java's ThetaUtil.MIN_LG_NOM_LONGS) and cache floor (MIN_LG_ARR = 5) + // apply. lg_k = 4 (nominal 16) is the smallest allowed nominal size; below it must throw. + REQUIRE(theta_constants::MIN_LG_K == 4); + REQUIRE_THROWS_AS(update_tuple_sketch::builder().set_lg_k(theta_constants::MIN_LG_K - 1), + std::invalid_argument); + auto min_sketch = update_tuple_sketch::builder().set_lg_k(theta_constants::MIN_LG_K).build(); + REQUIRE(min_sketch.get_lg_k() == theta_constants::MIN_LG_K); + + // update well past the nominal size to force estimation mode and exercise the rebuild path, + // tracking the peak number of retained entries seen between rebuilds. + const int n = 10000; + uint32_t max_retained = 0; + for (int i = 0; i < n; ++i) { + min_sketch.update(i, 1.0f); + if (min_sketch.get_num_retained() > max_retained) max_retained = min_sketch.get_num_retained(); + } + REQUIRE(min_sketch.is_estimation_mode()); + REQUIRE(min_sketch.get_theta() < 1.0); + + // The internal hash table is floored at MIN_LG_ARR (5, i.e. 32 slots), one lg above the + // nominal size. This is exactly what MIN_LG_ARR guarantees: between rebuilds the sketch holds + // more than the nominal 2^MIN_LG_K (16) entries, but never more than the 2^MIN_LG_ARR (32) + // slots of the table. Were the table sized to the nominal 16, it could not retain more than 16. + REQUIRE(max_retained > (1 << theta_constants::MIN_LG_K)); + REQUIRE(max_retained <= (1 << theta_constants::MIN_LG_ARR)); + + // the true count is bracketed by the 2-standard-deviation confidence bounds + REQUIRE(min_sketch.get_lower_bound(2) <= n); + REQUIRE(min_sketch.get_upper_bound(2) >= n); + + // trimming reduces the sketch to exactly the nominal number of entries (2^4 = 16) + min_sketch.trim(); + REQUIRE(min_sketch.get_num_retained() == (1 << theta_constants::MIN_LG_K)); + + // a compacted min-size sketch round trips through serialization + auto bytes = min_sketch.compact().serialize(); + auto deserialized = compact_tuple_sketch::deserialize(bytes.data(), bytes.size()); + REQUIRE(deserialized.get_num_retained() == min_sketch.get_num_retained()); + REQUIRE(deserialized.get_estimate() == min_sketch.get_estimate()); +} + TEST_CASE("tuple sketch float: empty", "[tuple_sketch]") { auto update_sketch = update_tuple_sketch::builder().build(); std::cout << "sizeof(update_tuple_sketch)=" << sizeof(update_sketch) << std::endl;