From 1060819573764b439cd6e80e7f2f17f07ab2caca Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Wed, 22 Jul 2026 10:15:24 +0800 Subject: [PATCH] feat(rest): support rest catalog for databases, tables and snapshots Add a REST catalog selected via the "metastore=rest" option: - RestHttpClient: blocking libcurl client with exponential-backoff retries. 429/503 are retried for every method, a transient transport error only for an idempotent one; a server closing the connection without responding is not retried. A Retry-After header is honored in both the delta-seconds and the HTTP-date form with a locale-independent parse, and is never shortened: the overall retry budget bounds it while the per-sleep cap bounds only the backoff. Easy handles are pooled, so a request reuses the previous connection instead of paying a handshake every time; TLS below 1.2 is refused and requests carry a default user agent. Every transported request is debug-logged with the server request id. - RestApi: HTTP + JSON protocol layer. "/v1/config" merges the server options into the client ones (overrides > client options > defaults, an override sent as null unsets the client option), "header." options are sent as request headers and a listing follows the page tokens to the last page. A failed request maps to a Status by the code of the parsed error body, falling back to the http status, and carries a RestErrorDetail with that code so callers can tell e.g. an authentication failure from another IO error. A response body is never echoed into an error, since it may carry credentials, so a body that is not an error object is only reported as unparsable. - RestCatalog: database, table and snapshot operations, with "table-default." option defaults and a schema conversion computing the highest field id (ids at or above the new SpecialFieldIds::SYSTEM_FIELD_ID_START are excluded, duplicated ids are rejected at any nesting level). A branch stays in the object name so the server resolves it and returns the branch's own schema, the default branch "main" being matched ignoring case and addressed as the bare table. The "sys" database serves the local global system tables like FileSystemCatalog. - Bear token authentication provider ("bear" is the protocol's historical spelling of "bearer"). RestAuthParameter::Create url-encodes the query parameter values a signing provider signs. - CatalogOptions: public catalog-level option keys (metastore, uri, token, token.provider, "table-default." prefix), kept separate from the table-level CoreOptions. - CatalogUtils: system database, system table and branch checks shared by FileSystemCatalog and RestCatalog. - RestUtil::ExtractRequestId: the request id lookup shared by the debug log line and the error message, falling back to any header whose name carries a request id since a gateway may report it under its own name. The pieces overlapping with the object store file systems move to common/utils: UrlUtils carries both URL encoding flavors (form-urlencoded for the REST api, RFC 3986 for S3), the generic HTTP client moves there from common/fs, and the new SensitiveConfigUtils redacts credentials for every surface that may expose them - the sys.catalog_options table masks the value of a credential-carrying option key, keeping at most a short tail of an identifier-like value, and a server error message carrying a secret marker is redacted as a whole. libcurl is used from the system, so the new PAIMON_ENABLE_REST option defaults to OFF like the other optional components with external requirements; its CMake detection is a single find_package guarded by PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST. Co-Authored-By: Claude Opus 5 (1M context) --- CMakeLists.txt | 11 +- ci/scripts/build_paimon.sh | 1 + docs/source/building.rst | 1 + docs/source/user_guide/catalog.rst | 64 +- include/paimon/catalog/catalog.h | 4 +- include/paimon/catalog_options.h | 45 + include/paimon/utils/special_field_ids.h | 4 + src/paimon/CMakeLists.txt | 59 +- src/paimon/common/catalog_options.cpp | 27 + .../common/{fs => utils}/http_client.cpp | 44 +- src/paimon/common/{fs => utils}/http_client.h | 8 + src/paimon/common/utils/http_client_test.cpp | 47 + .../common/utils/sensitive_config_utils.cpp | 130 ++ .../common/utils/sensitive_config_utils.h | 60 + .../utils/sensitive_config_utils_test.cpp | 107 ++ src/paimon/common/utils/url_utils.cpp | 132 +++ src/paimon/common/utils/url_utils.h | 54 + src/paimon/common/utils/url_utils_test.cpp | 84 ++ src/paimon/core/catalog/catalog.cpp | 23 + src/paimon/core/catalog/catalog_utils.cpp | 71 ++ src/paimon/core/catalog/catalog_utils.h | 46 + .../core/catalog/file_system_catalog.cpp | 82 +- src/paimon/core/catalog/file_system_catalog.h | 1 - .../core/catalog/file_system_catalog_test.cpp | 76 +- src/paimon/core/schema/table_schema.cpp | 62 + src/paimon/core/schema/table_schema.h | 7 + src/paimon/core/schema/table_schema_test.cpp | 39 + src/paimon/core/snapshot.cpp | 23 + src/paimon/core/snapshot.h | 3 + src/paimon/core/snapshot_test.cpp | 32 + .../table/system/global_system_tables.cpp | 11 +- src/paimon/fs/s3/s3_file_system.cpp | 55 +- src/paimon/fs/s3/s3_file_system.h | 2 +- src/paimon/rest/mock_rest_server.cpp | 218 ++++ src/paimon/rest/mock_rest_server.h | 91 ++ src/paimon/rest/resource_paths.cpp | 64 + src/paimon/rest/resource_paths.h | 45 + src/paimon/rest/resource_paths_test.cpp | 40 + src/paimon/rest/rest_api.cpp | 280 +++++ src/paimon/rest/rest_api.h | 138 +++ src/paimon/rest/rest_auth.cpp | 71 ++ src/paimon/rest/rest_auth.h | 73 ++ src/paimon/rest/rest_catalog.cpp | 442 +++++++ src/paimon/rest/rest_catalog.h | 101 ++ src/paimon/rest/rest_catalog_test.cpp | 1047 +++++++++++++++++ src/paimon/rest/rest_http_client.cpp | 462 ++++++++ src/paimon/rest/rest_http_client.h | 149 +++ src/paimon/rest/rest_http_client_test.cpp | 565 +++++++++ src/paimon/rest/rest_messages.cpp | 386 ++++++ src/paimon/rest/rest_messages.h | 389 ++++++ src/paimon/rest/rest_messages_test.cpp | 267 +++++ src/paimon/rest/rest_util.cpp | 74 ++ src/paimon/rest/rest_util.h | 61 + src/paimon/rest/rest_util_test.cpp | 47 + test/inte/read_inte_test.cpp | 29 +- 55 files changed, 6288 insertions(+), 166 deletions(-) create mode 100644 include/paimon/catalog_options.h create mode 100644 src/paimon/common/catalog_options.cpp rename src/paimon/common/{fs => utils}/http_client.cpp (91%) rename src/paimon/common/{fs => utils}/http_client.h (81%) create mode 100644 src/paimon/common/utils/http_client_test.cpp create mode 100644 src/paimon/common/utils/sensitive_config_utils.cpp create mode 100644 src/paimon/common/utils/sensitive_config_utils.h create mode 100644 src/paimon/common/utils/sensitive_config_utils_test.cpp create mode 100644 src/paimon/common/utils/url_utils.cpp create mode 100644 src/paimon/common/utils/url_utils.h create mode 100644 src/paimon/common/utils/url_utils_test.cpp create mode 100644 src/paimon/core/catalog/catalog_utils.cpp create mode 100644 src/paimon/core/catalog/catalog_utils.h create mode 100644 src/paimon/rest/mock_rest_server.cpp create mode 100644 src/paimon/rest/mock_rest_server.h create mode 100644 src/paimon/rest/resource_paths.cpp create mode 100644 src/paimon/rest/resource_paths.h create mode 100644 src/paimon/rest/resource_paths_test.cpp create mode 100644 src/paimon/rest/rest_api.cpp create mode 100644 src/paimon/rest/rest_api.h create mode 100644 src/paimon/rest/rest_auth.cpp create mode 100644 src/paimon/rest/rest_auth.h create mode 100644 src/paimon/rest/rest_catalog.cpp create mode 100644 src/paimon/rest/rest_catalog.h create mode 100644 src/paimon/rest/rest_catalog_test.cpp create mode 100644 src/paimon/rest/rest_http_client.cpp create mode 100644 src/paimon/rest/rest_http_client.h create mode 100644 src/paimon/rest/rest_http_client_test.cpp create mode 100644 src/paimon/rest/rest_messages.cpp create mode 100644 src/paimon/rest/rest_messages.h create mode 100644 src/paimon/rest/rest_messages_test.cpp create mode 100644 src/paimon/rest/rest_util.cpp create mode 100644 src/paimon/rest/rest_util.h create mode 100644 src/paimon/rest/rest_util_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e17bdda..e99034dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,9 +68,17 @@ option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF) option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF) option(PAIMON_ENABLE_TANTIVY "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF) +option(PAIMON_ENABLE_REST "Whether to enable the rest catalog (requires libcurl)" OFF) if(PAIMON_ENABLE_ORC) add_definitions(-DPAIMON_ENABLE_ORC) endif() +if(PAIMON_ENABLE_REST) + add_definitions(-DPAIMON_ENABLE_REST) +endif() +# libcurl backs the HTTP client shared by the S3 file system and the rest catalog. +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + find_package(CURL REQUIRED) +endif() if(PAIMON_ENABLE_AVRO) add_definitions(-DPAIMON_ENABLE_AVRO) endif() @@ -498,9 +506,6 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake" config_summary_message() -if(PAIMON_ENABLE_S3) - find_package(CURL REQUIRED) -endif() add_subdirectory(src/paimon) add_subdirectory(src/paimon/fs/local) if(PAIMON_ENABLE_JINDO) diff --git a/ci/scripts/build_paimon.sh b/ci/scripts/build_paimon.sh index 3df51435..c55a8a6f 100755 --- a/ci/scripts/build_paimon.sh +++ b/ci/scripts/build_paimon.sh @@ -146,6 +146,7 @@ CMAKE_ARGS=( "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}" "-DPAIMON_ENABLE_LUCENE=ON" "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}" + "-DPAIMON_ENABLE_REST=ON" "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}" ) diff --git a/docs/source/building.rst b/docs/source/building.rst index 2877ed9a..0c5ece36 100644 --- a/docs/source/building.rst +++ b/docs/source/building.rst @@ -126,6 +126,7 @@ boolean flags to ``cmake``. * ``-DPAIMON_ENABLE_LUMINA=ON``: Support for the Lumina vector index. * ``-DPAIMON_ENABLE_LUCENE=ON``: Support for Lucene full-text search indexes * ``-DPAIMON_ENABLE_TANTIVY=ON``: Enable the experimental Tantivy full-text index Rust FFI. +* ``-DPAIMON_ENABLE_REST=ON``: Support for the REST catalog (``metastore=rest``), requires the libcurl development package. Third-party dependency source ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index 5ff90536..c57c50c1 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -23,15 +23,63 @@ Paimon C++ provides a :ref:`Catalog abstraction ` to manage the abstraction provides a series of ways to help you better integrate with computing engines. We always recommend that you use Catalog to access the Paimon table. +Paimon C++ supports two metastores, selected with the catalog option +``metastore``: the filesystem metastore (default) and the REST metastore. + Filesystem Catalog ~~~~~~~~~~~~~~~~~~ -Paimon C++ catalog currently support one types of metastores filesystem metastore (default), -which stores both metadata and table files in filesystems. +The filesystem metastore (``metastore=filesystem``, the default) stores both +metadata and table files in filesystems. The ``root_path`` argument of +``Catalog::Create`` is the warehouse directory holding the databases and tables. + +REST Catalog +~~~~~~~~~~~~ +With the REST metastore (``metastore=rest``), catalog metadata is managed by a +remote catalog server exposed through a REST API; table data itself is still read +and written through the table paths returned by the server. See `Java Paimon REST +Catalog `_ for the concept +and the server-side protocol. + +REST catalog support is an optional build component: configure the build with +``-DPAIMON_ENABLE_REST=ON`` (see :ref:`cpp_build_optional_components`). + +When ``metastore=rest``, the ``root_path`` argument of ``Catalog::Create`` is not +a filesystem path but the warehouse (instance) name under which the tables are +registered on the REST server. The catalog is configured through the +``CatalogOptions`` keys: + +* ``metastore``: must be ``rest`` to select the REST catalog. +* ``uri``: server url of the REST catalog server. +* ``token.provider``: authentication provider of the REST catalog; currently only + ``bear`` is supported (the protocol's historical spelling of "bearer"). +* ``token``: token of the ``bear`` token provider. +* ``table-default.``: table option defaults applied when a created table + left ```` unset. +* ``header.``: sent as the ```` http header on every request to the + server. The server may configure headers of its own through the ``/v1/config`` + endpoint, which are merged with these as any other option is. + +.. code-block:: cpp + + std::map options = { + {"metastore", "rest"}, + {"uri", "http://127.0.0.1:8080"}, + {"token.provider", "bear"}, + {"token", ""}, + }; + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr catalog, + paimon::Catalog::Create(/*root_path=*/"my_instance", options)); + +On creation the catalog queries the server's ``/v1/config`` endpoint and merges +its response with the options above: the server's overrides win over the client +options, which in turn win over the server's defaults. -.. note:: +Databases and tables are then created, listed, loaded, renamed and dropped +through the regular ``Catalog`` API, and table snapshots can be listed through +``Catalog::ListSnapshots``. - Current Paimon C++ only supports filesystem catalog. In the future, we will - support REST catalog. - By using the Paimon REST catalog, changes to the catalog will be directly stored - in a remote catalog server which exposed through REST API. See `Java Paimon REST - Catalog `_. +The C++ REST catalog covers the database, table and snapshot operations of the +``Catalog`` API. The parts of the Java REST catalog that have no C++ counterpart +yet — altering a database or a table, views, functions, partitions, tags, branch +management and consumers — are not supported, and neither is the ``dlf`` token +provider. diff --git a/include/paimon/catalog/catalog.h b/include/paimon/catalog/catalog.h index b5d6bddd..9213bde9 100644 --- a/include/paimon/catalog/catalog.h +++ b/include/paimon/catalog/catalog.h @@ -58,7 +58,9 @@ class PAIMON_EXPORT Catalog { /// %Factory method for creating a `Catalog` instance. /// - /// @param root_path Path to the root directory where the catalog is located. + /// @param root_path Path to the root directory where the catalog is located. For the + /// REST catalog (`CatalogOptions::METASTORE` set to "rest") this is + /// instead the warehouse (instance) name registered on the server. /// @param options Configuration options for catalog initialization. /// @param file_system Specifies the file system for file operations. /// If not set, use default file system (configured in diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h new file mode 100644 index 00000000..f58a876c --- /dev/null +++ b/include/paimon/catalog_options.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "paimon/visibility.h" + +namespace paimon { + +/// Catalog-level configuration option keys; table-level keys live in `Options`. +struct PAIMON_EXPORT CatalogOptions { + /// "metastore" - Metastore of the paimon catalog. + /// Supported values are "filesystem" (default) and "rest". + static const char METASTORE[]; + + /// "uri" - Server url of the REST catalog. Only used when METASTORE is "rest". + static const char URI[]; + + /// "token" - Token of the "bear" token provider of the REST catalog. + static const char TOKEN[]; + + /// "token.provider" - Authentication provider of the REST catalog. Only "bear" is + /// supported ("bear" is the protocol's historical spelling of "bearer", do not "fix" it). + static const char TOKEN_PROVIDER[]; + + /// "table-default." - Prefix of the catalog options that provide table option + /// defaults: "table-default.=" applies "=" to a created + /// table when the caller left "" unset. + static const char TABLE_DEFAULT_OPTION_PREFIX[]; +}; + +} // namespace paimon diff --git a/include/paimon/utils/special_field_ids.h b/include/paimon/utils/special_field_ids.h index c4e03f00..829f2988 100644 --- a/include/paimon/utils/special_field_ids.h +++ b/include/paimon/utils/special_field_ids.h @@ -42,6 +42,10 @@ class SpecialFieldIds { /// Special field ID reserved for index score. Value: CPP_FIELD_ID_END - 1 inline static constexpr int32_t INDEX_SCORE = CPP_FIELD_ID_END - 1; + + /// Lowest field ID reserved for system fields; IDs at or above it are excluded from the + /// highest field ID of a schema. Value: INT32_MAX / 2 + inline static constexpr int32_t SYSTEM_FIELD_ID_START = std::numeric_limits::max() / 2; }; } // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index ce2f5ddf..552f772d 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -15,6 +15,7 @@ # limitations under the License. set(PAIMON_COMMON_SRCS + common/catalog_options.cpp common/compression/block_compression_factory.cpp common/compression/block_compressor.cpp common/compression/block_decompressor.cpp @@ -179,13 +180,20 @@ set(PAIMON_COMMON_SRCS common/utils/roaring_bitmap32.cpp common/utils/roaring_bitmap64.cpp common/utils/row_range_index.cpp + common/utils/sensitive_config_utils.cpp common/utils/status.cpp - common/utils/string_utils.cpp) + common/utils/string_utils.cpp + common/utils/url_utils.cpp) +# The shared HTTP client is used by both the object store file systems and the +# rest catalog. +set(PAIMON_CURL_LINK_LIBS) +if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + list(APPEND PAIMON_COMMON_SRCS common/utils/http_client.cpp) + set(PAIMON_CURL_LINK_LIBS CURL::libcurl) +endif() if(PAIMON_ENABLE_S3) - list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp - common/fs/object_store_file_system.cpp) - set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl) + list(APPEND PAIMON_COMMON_SRCS common/fs/object_store_file_system.cpp) endif() set(PAIMON_CORE_SRCS @@ -228,6 +236,7 @@ set(PAIMON_CORE_SRCS core/casting/timestamp_to_timestamp_cast_executor.cpp core/casting/casting_utils.cpp core/catalog/catalog.cpp + core/catalog/catalog_utils.cpp core/catalog/file_system_catalog.cpp core/catalog/identifier.cpp core/core_options.cpp @@ -405,6 +414,18 @@ set(PAIMON_CORE_SRCS core/utils/snapshot_manager.cpp core/utils/tag_manager.cpp) +if(PAIMON_ENABLE_REST) + list(APPEND + PAIMON_CORE_SRCS + rest/rest_http_client.cpp + rest/resource_paths.cpp + rest/rest_api.cpp + rest/rest_auth.cpp + rest/rest_catalog.cpp + rest/rest_messages.cpp + rest/rest_util.cpp) +endif() + add_paimon_lib(paimon SOURCES ${PAIMON_COMMON_SRCS} @@ -418,7 +439,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} STATIC_LINK_LIBS arrow tbb @@ -428,7 +449,7 @@ add_paimon_lib(paimon xxhash Threads::Threads RapidJSON - ${PAIMON_OBJECT_STORE_LINK_LIBS} + ${PAIMON_CURL_LINK_LIBS} SHARED_LINK_FLAGS ${PAIMON_VERSION_SCRIPT_FLAGS}) @@ -436,6 +457,13 @@ add_subdirectory(common/file_index) add_subdirectory(common/global_index) if(PAIMON_BUILD_TESTS) + # The shared HTTP client is compiled only for the components that need libcurl, + # so its test follows the same gate. + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS) + if(PAIMON_ENABLE_S3 OR PAIMON_ENABLE_REST) + set(PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS common/utils/http_client_test.cpp) + endif() + add_paimon_test(memory_test SOURCES common/memory/memory_pool_test.cpp @@ -593,10 +621,13 @@ if(PAIMON_BUILD_TESTS) common/io/cache/lru_cache_test.cpp common/utils/byte_range_combiner_test.cpp common/utils/scope_guard_test.cpp + common/utils/sensitive_config_utils_test.cpp common/utils/serialization_utils_test.cpp common/utils/status_test.cpp common/utils/stream_utils_test.cpp common/utils/string_utils_test.cpp + common/utils/url_utils_test.cpp + ${PAIMON_COMMON_HTTP_CLIENT_TEST_SRCS} common/utils/range_test.cpp common/utils/uuid_test.cpp common/utils/decimal_utils_test.cpp @@ -874,4 +905,20 @@ if(PAIMON_BUILD_TESTS) EXTRA_INCLUDES ${JINDOSDK_INCLUDE_DIR}) + if(PAIMON_ENABLE_REST) + add_paimon_test(rest_test + SOURCES + rest/rest_http_client_test.cpp + rest/mock_rest_server.cpp + rest/resource_paths_test.cpp + rest/rest_catalog_test.cpp + rest/rest_messages_test.cpp + rest/rest_util_test.cpp + STATIC_LINK_LIBS + paimon_shared + test_utils_static + ${TEST_STATIC_LINK_LIBS} + ${GTEST_LINK_TOOLCHAIN}) + endif() + endif() diff --git a/src/paimon/common/catalog_options.cpp b/src/paimon/common/catalog_options.cpp new file mode 100644 index 00000000..6e89e80a --- /dev/null +++ b/src/paimon/common/catalog_options.cpp @@ -0,0 +1,27 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/catalog_options.h" + +namespace paimon { + +const char CatalogOptions::METASTORE[] = "metastore"; +const char CatalogOptions::URI[] = "uri"; +const char CatalogOptions::TOKEN[] = "token"; +const char CatalogOptions::TOKEN_PROVIDER[] = "token.provider"; +const char CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX[] = "table-default."; + +} // namespace paimon diff --git a/src/paimon/common/fs/http_client.cpp b/src/paimon/common/utils/http_client.cpp similarity index 91% rename from src/paimon/common/fs/http_client.cpp rename to src/paimon/common/utils/http_client.cpp index 3ff589b4..e61c7faa 100644 --- a/src/paimon/common/fs/http_client.cpp +++ b/src/paimon/common/utils/http_client.cpp @@ -17,7 +17,7 @@ * under the License. */ -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include @@ -47,11 +47,6 @@ class CurlGlobalGuard { } }; -std::shared_ptr GetCurlGlobalGuard() { - static auto guard = std::make_shared(); - return guard; -} - void TrimHttpWhitespace(std::string* value) { constexpr char kHttpWhitespace[] = " \t\r\n"; size_t begin = value->find_first_not_of(kHttpWhitespace); @@ -95,16 +90,7 @@ size_t WriteCallback(char* data, size_t size, size_t count, void* user_data) { size_t HeaderCallback(char* data, size_t size, size_t count, void* user_data) { auto* context = static_cast(user_data); size_t bytes = size * count; - std::string line(data, bytes); - size_t colon = line.find(':'); - if (colon != std::string::npos) { - std::string name = line.substr(0, colon); - TrimHttpWhitespace(&name); - name = StringUtils::ToLowerCase(name); - std::string value = line.substr(colon + 1); - TrimHttpWhitespace(&value); - context->response.headers[name] = std::move(value); - } + ParseHttpHeaderLine(data, bytes, &context->response.headers); return bytes; } @@ -119,9 +105,31 @@ bool IsRetryable(CURLcode code, int64_t status_code) { } // namespace +std::shared_ptr EnsureCurlGlobalInit() { + static auto guard = std::make_shared(); + return guard; +} + +void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers) { + std::string line(data, size); + size_t colon = line.find(':'); + if (colon == std::string::npos) { + return; + } + std::string name = line.substr(0, colon); + TrimHttpWhitespace(&name); + if (name.empty()) { + return; + } + name = StringUtils::ToLowerCase(name); + std::string value = line.substr(colon + 1); + TrimHttpWhitespace(&value); + (*headers)[name] = std::move(value); +} + class CurlHttpClient::Impl { public: - Impl() : guard_(GetCurlGlobalGuard()) {} + Impl() : guard_(EnsureCurlGlobalInit()) {} ~Impl() { for (CURL* handle : handles_) { @@ -146,7 +154,7 @@ class CurlHttpClient::Impl { } private: - std::shared_ptr guard_; + std::shared_ptr guard_; mutable std::mutex mutex_; mutable std::vector handles_; }; diff --git a/src/paimon/common/fs/http_client.h b/src/paimon/common/utils/http_client.h similarity index 81% rename from src/paimon/common/fs/http_client.h rename to src/paimon/common/utils/http_client.h index 5dd8d029..0dd3e742 100644 --- a/src/paimon/common/fs/http_client.h +++ b/src/paimon/common/utils/http_client.h @@ -34,6 +34,14 @@ enum class HttpMethod { HEAD, GET }; using HttpHeaders = std::map; using HttpBodyConsumer = std::function; +/// Ensures libcurl's global state is initialized; the returned guard keeps it alive. +PAIMON_EXPORT std::shared_ptr EnsureCurlGlobalInit(); + +/// Parses one raw HTTP header line into `headers`, lower-casing the name and trimming +/// HTTP whitespace around the name and value; lines without a ':' or with an empty +/// name are ignored. +PAIMON_EXPORT void ParseHttpHeaderLine(const char* data, size_t size, HttpHeaders* headers); + struct HttpRequest { HttpMethod method = HttpMethod::GET; std::string url; diff --git a/src/paimon/common/utils/http_client_test.cpp b/src/paimon/common/utils/http_client_test.cpp new file mode 100644 index 00000000..5bf3ce23 --- /dev/null +++ b/src/paimon/common/utils/http_client_test.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/http_client.h" + +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(HttpClientUtilTest, ParseHttpHeaderLine) { + HttpHeaders headers; + const std::string line = "Content-Type: application/json\r\n"; + ParseHttpHeaderLine(line.data(), line.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "application/json"}}), headers); + + const std::string overwrite = "content-type: text/PLAIN \r\n"; + ParseHttpHeaderLine(overwrite.data(), overwrite.size(), &headers); + ASSERT_EQ((HttpHeaders{{"content-type", "text/PLAIN"}}), headers); + + // Lines without a colon (like the status line) and empty names are ignored. + const std::string status_line = "HTTP/1.1 200 OK\r\n"; + ParseHttpHeaderLine(status_line.data(), status_line.size(), &headers); + const std::string empty_name = ": value\r\n"; + ParseHttpHeaderLine(empty_name.data(), empty_name.size(), &headers); + ASSERT_EQ(1, headers.size()); + + const std::string empty_value = "x-empty:\r\n"; + ParseHttpHeaderLine(empty_value.data(), empty_value.size(), &headers); + ASSERT_EQ("", headers.at("x-empty")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/utils/sensitive_config_utils.cpp b/src/paimon/common/utils/sensitive_config_utils.cpp new file mode 100644 index 00000000..28740412 --- /dev/null +++ b/src/paimon/common/utils/sensitive_config_utils.cpp @@ -0,0 +1,130 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/sensitive_config_utils.h" + +#include "paimon/common/utils/string_utils.h" + +namespace paimon { + +namespace { + +// Substring markers of a credential, matched against the normalized form so that the +// separators of a key or a message do not matter. +constexpr const char* kSensitiveMarkers[] = { + "password", "secret", "token", "credential", "accesskey", "accountkey", + "encryptionkey", "authorization", "privatekey", "apikey", "sas"}; + +// Keys whose value is masked as a whole rather than keeping a trailing hint. Every true +// secret is here; only an identifier-like key keeps a tail. "accessKeySecret" normalizes +// to contain "secret" and is masked as a whole, while "accessKeyId" hits only the +// "accesskey" marker of `kSensitiveMarkers` and keeps its tail. This mirrors AWS/Azure, +// where the access key id is loggable but the secret, the token and the SAS are not. +constexpr const char* kFullMaskMarkers[] = {"password", "token", "authorization", "secret", + "credential", "privatekey", "encryptionkey", "apikey", + "accountkey", "sas"}; + +// A value shorter than this reveals too much of itself through a four character tail, so +// it is masked as a whole. +constexpr size_t kMinLengthForTail = 12; +constexpr size_t kTailLength = 4; + +// Free-form text is scanned for one more marker: in a message "signature" names a +// credential, while in an option key it names a request signature algorithm. +constexpr const char kTextOnlyMarker[] = "signature"; + +// The Azure SAS "sig" would be ambiguous once separators are removed ("sig" is a +// substring of many words), so it is matched literally instead. +constexpr const char* kLiteralTextMarkers[] = {"sig=", "\"sig\"", "'sig'"}; + +// Drops every character of an already lower-cased string that is not a letter or a +// digit, so "access-key", "access.key" and "accessKey" all normalize to "accesskey". +std::string StripSeparators(const std::string& lowered) { + std::string normalized; + normalized.reserve(lowered.size()); + for (char c : lowered) { + if ((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) { + normalized.push_back(c); + } + } + return normalized; +} + +template +bool ContainsMarker(const std::string& normalized, const char* const (&markers)[N]) { + for (const char* marker : markers) { + if (normalized.find(marker) != std::string::npos) { + return true; + } + } + return false; +} + +std::string NormalizeKey(const std::string& key) { + return StripSeparators(StringUtils::ToLowerCase(key)); +} + +} // namespace + +bool SensitiveConfigUtils::IsSensitiveKey(const std::string& key) { + if (key.empty()) { + return false; + } + return ContainsMarker(NormalizeKey(key), kSensitiveMarkers); +} + +std::string SensitiveConfigUtils::RedactValue(const std::string& key, const std::string& value) { + if (key.empty()) { + return value; + } + std::string normalized = NormalizeKey(key); + if (!ContainsMarker(normalized, kSensitiveMarkers)) { + return value; + } + if (!ContainsMarker(normalized, kFullMaskMarkers) && value.size() >= kMinLengthForTail) { + return "****" + value.substr(value.size() - kTailLength); + } + return kRedacted; +} + +std::map SensitiveConfigUtils::RedactMap( + const std::map& options) { + std::map redacted; + for (const auto& [key, value] : options) { + redacted.emplace(key, RedactValue(key, value)); + } + return redacted; +} + +std::string SensitiveConfigUtils::RedactText(const std::string& text) { + if (text.empty()) { + return text; + } + std::string lowered = StringUtils::ToLowerCase(text); + for (const char* marker : kLiteralTextMarkers) { + if (lowered.find(marker) != std::string::npos) { + return kRedacted; + } + } + std::string normalized = StripSeparators(lowered); + if (ContainsMarker(normalized, kSensitiveMarkers) || + normalized.find(kTextOnlyMarker) != std::string::npos) { + return kRedacted; + } + return text; +} + +} // namespace paimon diff --git a/src/paimon/common/utils/sensitive_config_utils.h b/src/paimon/common/utils/sensitive_config_utils.h new file mode 100644 index 00000000..71715609 --- /dev/null +++ b/src/paimon/common/utils/sensitive_config_utils.h @@ -0,0 +1,60 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/visibility.h" + +namespace paimon { + +/// Redacts credentials (passwords, secrets, tokens, access keys) before they reach a log +/// line, an error message or a user visible table. +class PAIMON_EXPORT SensitiveConfigUtils { + public: + SensitiveConfigUtils() = delete; + ~SensitiveConfigUtils() = delete; + + /// Replaces a redacted option value, and the whole text of a redacted message. + static constexpr const char* kRedacted = "******"; + + /// Returns whether `key` names a credential. The key is matched lower-cased and + /// stripped of separators, so "dlf.access-key-secret", "fs.s3a.access.key", + /// "fs.azure.account-key.store" and "accessKeySecret" all hit a marker. + static bool IsSensitiveKey(const std::string& key); + + /// Returns `value` masked when `key` names a credential, and `value` unchanged + /// otherwise. A key naming a true secret (a token, a password, an account key, ...) + /// is masked as a whole, while an identifier-like key ("dlf.access-key-id" hits only + /// the "accesskey" marker) keeps the last four characters of a long enough value: + /// enough to tell two credentials apart in a support case, not enough to use either. + static std::string RedactValue(const std::string& key, const std::string& value); + + /// Returns a copy of `options` with the value of every sensitive key redacted by + /// `RedactValue`. Every surface exposing catalog or table options to users (e.g. the + /// `sys.catalog_options` table) must pass them through this filter. + static std::map RedactMap( + const std::map& options); + + /// Redacts free-form text such as a server error message: arbitrary text cannot be + /// masked per-secret reliably, so the whole text becomes `kRedacted` as soon as any + /// marker of a sensitive value ("password", "token", "sig=", ...) appears. + static std::string RedactText(const std::string& text); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/sensitive_config_utils_test.cpp b/src/paimon/common/utils/sensitive_config_utils_test.cpp new file mode 100644 index 00000000..5e8b5350 --- /dev/null +++ b/src/paimon/common/utils/sensitive_config_utils_test.cpp @@ -0,0 +1,107 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/sensitive_config_utils.h" + +#include +#include + +#include "gtest/gtest.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(SensitiveConfigUtilsTest, IsSensitiveKey) { + // the key is matched lower-cased and stripped of separators, so the same marker hits + // whichever separator style a key uses + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("token")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("dlf.access-key-secret")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("fs.s3a.access.key")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("fs.azure.account-key.store1")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("accessKeySecret")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("client.credential")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("fs.azure.sas.container")); + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("HTTP.Authorization")); + + // the match is a substring one, so a key merely naming a credential is masked too + ASSERT_TRUE(SensitiveConfigUtils::IsSensitiveKey("token.provider")); + + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("")); + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("uri")); + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("file.format")); + // "signature" marks free-form text, not an option key: in a key it names the signing + // algorithm rather than a stored credential + ASSERT_FALSE(SensitiveConfigUtils::IsSensitiveKey("dlf.signing-algorithm")); +} + +TEST(SensitiveConfigUtilsTest, RedactValue) { + // a key naming no credential keeps its value + ASSERT_EQ("orc", SensitiveConfigUtils::RedactValue("file.format", "orc")); + ASSERT_EQ("v", SensitiveConfigUtils::RedactValue("", "v")); + + // a key naming a true secret is masked as a whole, however long the value is + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactValue("token", "bearer-credential-1")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactValue("dlf.access-key-secret", "secret-value-1")); + + // an identifier-like key keeps the last four characters of a long enough value, so + // two credentials can be told apart without disclosing either + ASSERT_EQ("****k-id", SensitiveConfigUtils::RedactValue("dlf.access-key-id", "an-access-k-id")); + // a short value would reveal too much of itself through the tail + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactValue("dlf.access-key-id", "short-id")); +} + +TEST(SensitiveConfigUtilsTest, RedactMap) { + const std::map options = { + {"uri", "http://127.0.0.1:8080"}, + {"token", "bearer-credential"}, + {"dlf.access-key-secret", "ak-secret"}, + {"file.format", "orc"}, + }; + std::map redacted = SensitiveConfigUtils::RedactMap(options); + // every key stays listed, only the credential-carrying values are replaced + ASSERT_EQ(options.size(), redacted.size()); + ASSERT_EQ("http://127.0.0.1:8080", redacted.at("uri")); + ASSERT_EQ("orc", redacted.at("file.format")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, redacted.at("token")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, redacted.at("dlf.access-key-secret")); + + ASSERT_TRUE(SensitiveConfigUtils::RedactMap({}).empty()); +} + +TEST(SensitiveConfigUtilsTest, RedactText) { + // a marker anywhere in the text redacts all of it: arbitrary text cannot be masked + // per-secret reliably + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("invalid password=abc123")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("bad ACCESS-KEY provided")); + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("url?X-Amz-Signature=deadbeef")); + // the Azure SAS "sig" is matched literally, since it is ambiguous once the separators + // are stripped + ASSERT_EQ(SensitiveConfigUtils::kRedacted, + SensitiveConfigUtils::RedactText("url?sig=deadbeef")); + + ASSERT_EQ("table t1 not found", SensitiveConfigUtils::RedactText("table t1 not found")); + // a word merely containing "sig" is not a marker + ASSERT_EQ("design is invalid", SensitiveConfigUtils::RedactText("design is invalid")); + ASSERT_EQ("", SensitiveConfigUtils::RedactText("")); +} + +} // namespace paimon::test diff --git a/src/paimon/common/utils/url_utils.cpp b/src/paimon/common/utils/url_utils.cpp new file mode 100644 index 00000000..883a4b77 --- /dev/null +++ b/src/paimon/common/utils/url_utils.cpp @@ -0,0 +1,132 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include + +namespace paimon { + +namespace { + +bool IsAlphaNumeric(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9'); +} + +bool IsFormUnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '.' || c == '-' || c == '*' || c == '_'; +} + +bool IsRfc3986UnreservedChar(char c) { + return IsAlphaNumeric(c) || c == '-' || c == '.' || c == '_' || c == '~'; +} + +void AppendPercentEncoded(char c, std::string* out) { + char buf[4]; + std::snprintf(buf, sizeof(buf), "%%%02X", static_cast(c)); + out->append(buf); +} + +int32_t HexValue(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +} // namespace + +std::string UrlUtils::EncodeString(const std::string& input) { + std::string encoded; + encoded.reserve(input.size()); + for (char c : input) { + if (IsFormUnreservedChar(c)) { + encoded.push_back(c); + } else if (c == ' ') { + encoded.push_back('+'); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +std::string UrlUtils::DecodeString(const std::string& input) { + std::string decoded; + decoded.reserve(input.size()); + for (size_t i = 0; i < input.size(); i++) { + char c = input[i]; + if (c == '+') { + decoded.push_back(' '); + } else if (c == '%' && i + 2 < input.size()) { + int32_t high = HexValue(input[i + 1]); + int32_t low = HexValue(input[i + 2]); + if (high >= 0 && low >= 0) { + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } else { + decoded.push_back(c); + } + } + return decoded; +} + +std::string UrlUtils::PercentEncode(std::string_view value, bool preserve_slash) { + std::string encoded; + encoded.reserve(value.size()); + for (char c : value) { + if (IsRfc3986UnreservedChar(c) || (preserve_slash && c == '/')) { + encoded.push_back(c); + } else { + AppendPercentEncoded(c, &encoded); + } + } + return encoded; +} + +Result UrlUtils::PercentDecode(std::string_view value) { + std::string decoded; + decoded.reserve(value.size()); + for (size_t i = 0; i < value.size(); i++) { + char c = value[i]; + if (c == '%') { + if (i + 2 >= value.size()) { + return Status::IOError("invalid percent encoding in URL component"); + } + int32_t high = HexValue(value[i + 1]); + int32_t low = HexValue(value[i + 2]); + if (high < 0 || low < 0) { + return Status::IOError("invalid percent encoding in URL component"); + } + decoded.push_back(static_cast((high << 4) | low)); + i += 2; + } else { + decoded.push_back(c); + } + } + return decoded; +} + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils.h b/src/paimon/common/utils/url_utils.h new file mode 100644 index 00000000..a045dc06 --- /dev/null +++ b/src/paimon/common/utils/url_utils.h @@ -0,0 +1,54 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/result.h" +#include "paimon/visibility.h" + +namespace paimon { + +/// URL encoding and decoding shared by the REST catalog and the object store file +/// systems. +class PAIMON_EXPORT UrlUtils { + public: + UrlUtils() = delete; + ~UrlUtils() = delete; + + /// URL-encode a string in the `application/x-www-form-urlencoded` flavor (UTF-8): + /// alphanumeric characters and ".", "-", "*", "_" are kept, a space becomes "+", all + /// other bytes become percent-encoded "%XX". This is what the REST catalog server + /// expects; unlike RFC 3986 it also escapes "~". + static std::string EncodeString(const std::string& input); + + /// Decodes `EncodeString` output ('+' back to space, "%XX" to the byte); malformed + /// escape sequences are kept as-is instead of failing. + static std::string DecodeString(const std::string& input); + + /// Percent-encode a URL component with RFC 3986 rules: alphanumeric characters and + /// "-", ".", "_", "~" are kept, every other byte (including a space) becomes "%XX". + /// With `preserve_slash`, '/' is kept as-is so an object key keeps its path shape. + static std::string PercentEncode(std::string_view value, bool preserve_slash = false); + + /// Strict inverse of `PercentEncode`: "%XX" becomes the byte, '+' is kept as-is, and + /// a '%' not followed by two hex digits is an error. + static Result PercentDecode(std::string_view value); +}; + +} // namespace paimon diff --git a/src/paimon/common/utils/url_utils_test.cpp b/src/paimon/common/utils/url_utils_test.cpp new file mode 100644 index 00000000..ebdc2bdc --- /dev/null +++ b/src/paimon/common/utils/url_utils_test.cpp @@ -0,0 +1,84 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/common/utils/url_utils.h" + +#include "gtest/gtest.h" +#include "paimon/status.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +TEST(UrlUtilsTest, EncodeString) { + ASSERT_EQ("abcDEF012.-*_", UrlUtils::EncodeString("abcDEF012.-*_")); + ASSERT_EQ("a+b", UrlUtils::EncodeString("a b")); + ASSERT_EQ("a%2Fb", UrlUtils::EncodeString("a/b")); + ASSERT_EQ("a%3Db%26c", UrlUtils::EncodeString("a=b&c")); + // '~' is the character distinguishing the two encoding flavors: EncodeString + // (form-urlencoded) escapes it while PercentEncode (RFC 3986) keeps it. + ASSERT_EQ("%7E", UrlUtils::EncodeString("~")); + ASSERT_EQ("a%2Bb", UrlUtils::EncodeString("a+b")); + ASSERT_EQ("%E4%B8%AD", UrlUtils::EncodeString("中")); +} + +TEST(UrlUtilsTest, DecodeString) { + ASSERT_EQ("a/b", UrlUtils::DecodeString("a%2Fb")); + ASSERT_EQ("a b", UrlUtils::DecodeString("a+b")); + ASSERT_EQ("中", UrlUtils::DecodeString("%E4%B8%AD")); + ASSERT_EQ("a/", UrlUtils::DecodeString("a%2F")); + // Malformed escapes are kept as-is. + ASSERT_EQ("a%", UrlUtils::DecodeString("a%")); + ASSERT_EQ("%2", UrlUtils::DecodeString("%2")); + ASSERT_EQ("%ZZ", UrlUtils::DecodeString("%ZZ")); + ASSERT_EQ("100%", UrlUtils::DecodeString("100%")); +} + +TEST(UrlUtilsTest, PercentEncode) { + ASSERT_EQ("abcDEF012-._~", UrlUtils::PercentEncode("abcDEF012-._~")); + ASSERT_EQ("a%20b", UrlUtils::PercentEncode("a b")); + ASSERT_EQ("a%2Ab", UrlUtils::PercentEncode("a*b")); + ASSERT_EQ("a%2Fb", UrlUtils::PercentEncode("a/b")); + ASSERT_EQ("a/b%3Dc", UrlUtils::PercentEncode("a/b=c", /*preserve_slash=*/true)); + // a literal '%' is encoded whether or not '/' is kept, so an input already reading + // "%2F" cannot come out as a slash + ASSERT_EQ("a%252Fb", UrlUtils::PercentEncode("a%2Fb", /*preserve_slash=*/true)); + ASSERT_EQ("%E4%B8%AD", UrlUtils::PercentEncode("中")); +} + +TEST(UrlUtilsTest, PercentDecode) { + ASSERT_OK_AND_ASSIGN(std::string decoded, UrlUtils::PercentDecode("a%2Fb%20c")); + ASSERT_EQ("a/b c", decoded); + // '+' is form encoding only; percent decoding keeps it. + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("a+b")); + ASSERT_EQ("a+b", decoded); + ASSERT_OK_AND_ASSIGN(decoded, UrlUtils::PercentDecode("%e4%b8%ad")); + ASSERT_EQ("中", decoded); + // Malformed escapes are an error. + ASSERT_NOK(UrlUtils::PercentDecode("a%").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%2").status()); + ASSERT_NOK(UrlUtils::PercentDecode("%ZZ").status()); + ASSERT_NOK(UrlUtils::PercentDecode("100%").status()); +} + +TEST(UrlUtilsTest, RoundTrip) { + const std::string input = "db 1/table$branch_b1=中%"; + ASSERT_EQ(input, UrlUtils::DecodeString(UrlUtils::EncodeString(input))); + ASSERT_OK_AND_ASSIGN(std::string decoded, + UrlUtils::PercentDecode(UrlUtils::PercentEncode(input))); + ASSERT_EQ(input, decoded); +} + +} // namespace paimon::test diff --git a/src/paimon/core/catalog/catalog.cpp b/src/paimon/core/catalog/catalog.cpp index ab26ce0c..97f13afa 100644 --- a/src/paimon/core/catalog/catalog.cpp +++ b/src/paimon/core/catalog/catalog.cpp @@ -20,8 +20,13 @@ #include +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/catalog/file_system_catalog.h" #include "paimon/core/core_options.h" +#ifdef PAIMON_ENABLE_REST +#include "paimon/rest/rest_catalog.h" +#endif namespace paimon { @@ -33,6 +38,24 @@ const char Catalog::DB_LOCATION_PROP[] = "location"; Result> Catalog::Create(const std::string& root_path, const std::map& options, const std::shared_ptr& file_system) { + std::string metastore = "filesystem"; + auto metastore_iter = options.find(CatalogOptions::METASTORE); + if (metastore_iter != options.end()) { + // Matched leniently in lower case; the Java catalog factory looks the metastore + // up by its exact identifier, so only the exact spelling is portable. + metastore = StringUtils::ToLowerCase(metastore_iter->second); + } + if (metastore == "rest") { +#ifdef PAIMON_ENABLE_REST + return RestCatalog::Create(root_path, options, file_system); +#else + return Status::NotImplemented( + "the rest catalog requires building paimon with PAIMON_ENABLE_REST=ON"); +#endif + } + if (metastore != "filesystem") { + return Status::Invalid("unsupported metastore: ", metastore); + } PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options, file_system)); return std::make_unique(core_options.GetFileSystem(), root_path, options); } diff --git a/src/paimon/core/catalog/catalog_utils.cpp b/src/paimon/core/catalog/catalog_utils.cpp new file mode 100644 index 00000000..7a6768ab --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/core/catalog/catalog_utils.h" + +#include + +#include "fmt/format.h" +#include "paimon/catalog/catalog.h" +#include "paimon/result.h" + +namespace paimon { + +namespace { + +Status SystemTableError(const Identifier& identifier, const std::string& action) { + return Status::Invalid(fmt::format("Cannot '{}' for system table '{}', please use data table.", + action, identifier.ToString())); +} + +} // namespace + +bool CatalogUtils::IsSystemDatabase(const std::string& db_name) { + return db_name == Catalog::SYSTEM_DATABASE_NAME; +} + +Status CatalogUtils::CheckNotSystemDatabase(const std::string& db_name, const std::string& action) { + if (IsSystemDatabase(db_name)) { + return Status::Invalid( + fmt::format("Cannot '{}' for system database '{}'.", action, db_name)); + } + return Status::OK(); +} + +Status CatalogUtils::CheckNotSystemTable(const Identifier& identifier, const std::string& action) { + // The system database is checked first so that an identifier of "sys" is rejected + // without being parsed as a table name. + if (IsSystemDatabase(identifier.GetDatabaseName())) { + return SystemTableError(identifier, action); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + return SystemTableError(identifier, action); + } + return Status::OK(); +} + +Status CatalogUtils::CheckNotBranch(const Identifier& identifier, const std::string& action) { + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + if (branch) { + return Status::Invalid(fmt::format( + "Cannot '{}' for branch table '{}', please modify the table with the default branch.", + action, identifier.ToString())); + } + return Status::OK(); +} + +} // namespace paimon diff --git a/src/paimon/core/catalog/catalog_utils.h b/src/paimon/core/catalog/catalog_utils.h new file mode 100644 index 00000000..b9a54ac1 --- /dev/null +++ b/src/paimon/core/catalog/catalog_utils.h @@ -0,0 +1,46 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/status.h" + +namespace paimon { + +/// Checks shared by the catalog implementations. Every check takes an `action` naming +/// the rejected operation in the error message, e.g. "dropTable". +class CatalogUtils { + public: + CatalogUtils() = delete; + ~CatalogUtils() = delete; + + /// Returns whether `db_name` is the reserved system database "sys". + static bool IsSystemDatabase(const std::string& db_name); + + /// Fails when `db_name` is the system database. + static Status CheckNotSystemDatabase(const std::string& db_name, const std::string& action); + + /// Fails when `identifier` denotes a system table or any table of the system database. + static Status CheckNotSystemTable(const Identifier& identifier, const std::string& action); + + /// Fails when `identifier` carries a "$branch_" suffix. + static Status CheckNotBranch(const Identifier& identifier, const std::string& action); +}; + +} // namespace paimon diff --git a/src/paimon/core/catalog/file_system_catalog.cpp b/src/paimon/core/catalog/file_system_catalog.cpp index 292af2e8..a036c080 100644 --- a/src/paimon/core/catalog/file_system_catalog.cpp +++ b/src/paimon/core/catalog/file_system_catalog.cpp @@ -31,6 +31,7 @@ #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/system/global_system_tables.h" @@ -60,10 +61,7 @@ FileSystemCatalog::FileSystemCatalog(const std::shared_ptr& fs, Status FileSystemCatalog::CreateDatabase(const std::string& db_name, const std::map& options, bool ignore_if_exists) { - if (IsSystemDatabase(db_name)) { - return Status::Invalid( - fmt::format("Cannot create database for system database {}.", db_name)); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(db_name, "createDatabase")); PAIMON_ASSIGN_OR_RAISE(bool exist, DatabaseExists(db_name)); if (exist) { if (ignore_if_exists) { @@ -94,7 +92,7 @@ Status FileSystemCatalog::CreateDatabaseImpl(const std::string& db_name, } Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const { - if (IsSystemDatabase(db_name)) { + if (CatalogUtils::IsSystemDatabase(db_name)) { return true; } return fs_->Exists(NewDatabasePath(warehouse_, db_name)); @@ -102,7 +100,7 @@ Result FileSystemCatalog::DatabaseExists(const std::string& db_name) const Result FileSystemCatalog::TableExists(const Identifier& identifier) const { // Handle sys database global tables - if (IsSystemDatabase(identifier.GetDatabaseName())) { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), catalog_options_); } PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); @@ -136,12 +134,8 @@ Status FileSystemCatalog::CreateTable(const Identifier& identifier, ArrowSchema* const std::vector& primary_keys, const std::map& options, bool ignore_if_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid( - fmt::format("Cannot create table for system table {}, please use data table.", - identifier.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "createTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "createTable")); PAIMON_ASSIGN_OR_RAISE(bool db_exist, DatabaseExists(identifier.GetDatabaseName())); if (!db_exist) { return Status::Invalid( @@ -198,16 +192,12 @@ const std::map& FileSystemCatalog::GetOptions() const return catalog_options_; } -bool FileSystemCatalog::IsSystemDatabase(const std::string& db_name) { - return db_name == SYSTEM_DATABASE_NAME; -} - Result FileSystemCatalog::IsSpecifiedSystemTable(const Identifier& identifier) { return identifier.IsSystemTable(); } Result FileSystemCatalog::IsSystemTable(const Identifier& identifier) { - if (IsSystemDatabase(identifier.GetDatabaseName())) { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { return true; } return IsSpecifiedSystemTable(identifier); @@ -241,7 +231,7 @@ Result> FileSystemCatalog::ListDatabases() const { } Result> FileSystemCatalog::ListTables(const std::string& db_name) const { - if (IsSystemDatabase(db_name)) { + if (CatalogUtils::IsSystemDatabase(db_name)) { return GlobalSystemTableLoader::GetSupportedTableNames(catalog_options_); } std::string database_path = NewDatabasePath(warehouse_, db_name); @@ -276,7 +266,7 @@ Result FileSystemCatalog::TableExistsInFileSystem(const std::string& table Result> FileSystemCatalog::LoadTableSchema( const Identifier& identifier) const { // Handle sys database global tables - if (IsSystemDatabase(identifier.GetDatabaseName())) { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { PAIMON_ASSIGN_OR_RAISE(bool supported, GlobalSystemTableLoader::IsSupported( identifier.GetTableName(), catalog_options_)); if (!supported) { @@ -341,10 +331,7 @@ Result> FileSystemCatalog::GetTable(const Identifier& ide Status FileSystemCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { - if (IsSystemDatabase(name)) { - return Status::Invalid(fmt::format("Cannot drop system database {}.", name)); - } - + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "dropDatabase")); PAIMON_ASSIGN_OR_RAISE(bool exist, DatabaseExists(name)); if (!exist) { if (ignore_if_not_exists) { @@ -427,10 +414,10 @@ Status FileSystemCatalog::DropTableImpl(const Identifier& identifier, } Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_system_table, IsSystemTable(identifier)); - if (is_system_table) { - return Status::Invalid(fmt::format("Cannot drop system table {}.", identifier.ToString())); - } + // A branch identifier resolves to the main table directory, so without this check + // dropping "t$branch_b" would delete the whole table "t". + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "dropTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "dropTable")); PAIMON_ASSIGN_OR_RAISE(std::string table_path, GetTableLocation(identifier)); PAIMON_ASSIGN_OR_RAISE(bool exist, fs_->Exists(table_path)); if (!exist) { @@ -490,12 +477,10 @@ Status FileSystemCatalog::DropTable(const Identifier& identifier, bool ignore_if Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, bool ignore_if_not_exists) { - PAIMON_ASSIGN_OR_RAISE(bool is_from_system_table, IsSystemTable(from_table)); - PAIMON_ASSIGN_OR_RAISE(bool is_to_system_table, IsSystemTable(to_table)); - if (is_from_system_table || is_to_system_table) { - return Status::Invalid(fmt::format("Cannot rename system table {} or {}.", - from_table.ToString(), to_table.ToString())); - } + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(to_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "renameTable")); if (from_table.GetDatabaseName() != to_table.GetDatabaseName()) { return Status::Invalid( @@ -523,24 +508,6 @@ Status FileSystemCatalog::RenameTable(const Identifier& from_table, const Identi return Status::OK(); } -namespace { -SnapshotInfo::CommitKind ConvertCommitKind(Snapshot::CommitKind internal) { - if (internal == Snapshot::CommitKind::Append()) { - return SnapshotInfo::CommitKind::APPEND; - } - if (internal == Snapshot::CommitKind::Compact()) { - return SnapshotInfo::CommitKind::COMPACT; - } - if (internal == Snapshot::CommitKind::Overwrite()) { - return SnapshotInfo::CommitKind::OVERWRITE; - } - if (internal == Snapshot::CommitKind::Analyze()) { - return SnapshotInfo::CommitKind::ANALYZE; - } - return SnapshotInfo::CommitKind::UNKNOWN; -} -} // namespace - Result> FileSystemCatalog::ListSnapshots( const Identifier& identifier, const std::string& branch) const { PAIMON_ASSIGN_OR_RAISE(bool exists, TableExists(identifier)); @@ -555,20 +522,9 @@ Result> FileSystemCatalog::ListSnapshots( std::vector result; result.reserve(snapshots.size()); - for (const auto& snap : snapshots) { - SnapshotInfo info; - info.snapshot_id = snap.Id(); - info.schema_id = snap.SchemaId(); - info.commit_user = snap.CommitUser(); - info.commit_kind = ConvertCommitKind(snap.GetCommitKind()); - info.time_millis = snap.TimeMillis(); - info.total_record_count = snap.TotalRecordCount(); - info.delta_record_count = snap.DeltaRecordCount(); - info.watermark = snap.Watermark(); - result.push_back(std::move(info)); + result.push_back(snap.ToSnapshotInfo()); } - return result; } diff --git a/src/paimon/core/catalog/file_system_catalog.h b/src/paimon/core/catalog/file_system_catalog.h index 3a464ae8..3925aff8 100644 --- a/src/paimon/core/catalog/file_system_catalog.h +++ b/src/paimon/core/catalog/file_system_catalog.h @@ -73,7 +73,6 @@ class FileSystemCatalog : public Catalog { static std::string NewDatabasePath(const std::string& warehouse, const std::string& db_name); static Result NewDataTablePath(const std::string& warehouse, const Identifier& identifier); - static bool IsSystemDatabase(const std::string& db_name); static Result IsSpecifiedSystemTable(const Identifier& identifier); static Result IsSystemTable(const Identifier& identifier); Result>> TableSchemaExists( diff --git a/src/paimon/core/catalog/file_system_catalog_test.cpp b/src/paimon/core/catalog/file_system_catalog_test.cpp index 2a07b262..9f6a568f 100644 --- a/src/paimon/core/catalog/file_system_catalog_test.cpp +++ b/src/paimon/core/catalog/file_system_catalog_test.cpp @@ -93,7 +93,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); ASSERT_NOK_WITH_MSG(catalog.CreateDatabase(Catalog::SYSTEM_DATABASE_NAME, options, /*ignore_if_exists=*/true), - "Cannot create database for system database"); + "Cannot 'createDatabase' for system database 'sys'."); } /// Do not support create system table. { @@ -116,7 +116,7 @@ TEST(FileSystemCatalogTest, TestCreateSystemDatabaseAndTable) { ASSERT_TRUE(arrow::ExportSchema(typed_schema, &schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(Identifier("db1", "ta$ble"), &schema, {"f1"}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&schema); } } @@ -208,11 +208,51 @@ TEST(FileSystemCatalogTest, TestOptionsSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(options_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(options_identifier, false), + "Cannot 'dropTable' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(options_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); +} + +TEST(FileSystemCatalogTest, TestBranchIdentifierRejectedForTableOperations) { + std::map options; + options[Options::FILE_SYSTEM] = "local"; + options[Options::FILE_FORMAT] = "orc"; + ASSERT_OK_AND_ASSIGN(auto core_options, CoreOptions::FromMap(options)); + auto dir = UniqueTestDirectory::Create(); + ASSERT_TRUE(dir); + FileSystemCatalog catalog(core_options.GetFileSystem(), dir->Str(), options); + ASSERT_OK(catalog.CreateDatabase("db1", options, /*ignore_if_exists=*/true)); + + auto typed_schema = + arrow::schema({arrow::field("f0", arrow::int32()), arrow::field("f1", arrow::utf8())}); + ::ArrowSchema schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &schema).ok()); + ASSERT_OK(catalog.CreateTable(Identifier("db1", "tbl1"), &schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, options, + /*ignore_if_exists=*/false)); + ArrowSchemaRelease(&schema); + + // A branch identifier resolves to the main table directory, so create, drop and rename + // must reject it: dropping "tbl1$branch_b1" would otherwise delete the whole "tbl1". + Identifier branch_identifier("db1", "tbl1$branch_b1"); + ::ArrowSchema branch_create_schema; + ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &branch_create_schema).ok()); + ASSERT_NOK_WITH_MSG( + catalog.CreateTable(branch_identifier, &branch_create_schema, {}, {}, options, false), + "Cannot 'createTable' for branch table"); + ArrowSchemaRelease(&branch_create_schema); + ASSERT_NOK_WITH_MSG(catalog.DropTable(branch_identifier, false), + "Cannot 'dropTable' for branch table"); + ASSERT_NOK_WITH_MSG(catalog.RenameTable(branch_identifier, Identifier("db1", "tbl2"), false), + "Cannot 'renameTable' for branch table"); + ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("db1", "tbl1"), branch_identifier, false), + "Cannot 'renameTable' for branch table"); + // the table itself is untouched by the rejected operations + ASSERT_OK_AND_ASSIGN(bool exists, catalog.TableExists(Identifier("db1", "tbl1"))); + ASSERT_TRUE(exists); } TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { @@ -279,11 +319,12 @@ TEST(FileSystemCatalogTest, TestAuditLogAndBinlogSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(audit_log_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(binlog_identifier, false), + "Cannot 'dropTable' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(audit_log_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); } TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { @@ -422,11 +463,12 @@ TEST(FileSystemCatalogTest, TestMetadataSystemTableCatalog) { ASSERT_TRUE(arrow::ExportSchema(*typed_schema, &system_create_schema).ok()); ASSERT_NOK_WITH_MSG( catalog.CreateTable(snapshots_identifier, &system_create_schema, {}, {}, options, false), - "Cannot create table for system table"); + "Cannot 'createTable' for system table"); ArrowSchemaRelease(&system_create_schema); - ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), "Cannot drop system table"); + ASSERT_NOK_WITH_MSG(catalog.DropTable(snapshots_identifier, false), + "Cannot 'dropTable' for system table"); ASSERT_NOK_WITH_MSG(catalog.RenameTable(snapshots_identifier, Identifier("db1", "tbl2"), false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); } TEST(FileSystemCatalogTest, TestCreateTableWithBlob) { @@ -748,7 +790,7 @@ TEST(FileSystemCatalogTest, TestDropDatabase) { ASSERT_NOK_WITH_MSG(catalog.DropDatabase(Catalog::SYSTEM_DATABASE_NAME, /*ignore_if_not_exists=*/false, /*cascade=*/false), - "Cannot drop system database sys."); + "Cannot 'dropDatabase' for system database 'sys'."); ArrowSchemaRelease(&schema); } @@ -787,10 +829,10 @@ TEST(FileSystemCatalogTest, TestDropTable) { ASSERT_FALSE(exist); /// Test 4: Drop system table. - ASSERT_NOK_WITH_MSG( - catalog.DropTable(Identifier("test_db", "tbl$system"), - /*ignore_if_not_exists=*/false), - "Cannot drop system table Identifier{database='test_db', table='tbl$system'}."); + ASSERT_NOK_WITH_MSG(catalog.DropTable(Identifier("test_db", "tbl$system"), + /*ignore_if_not_exists=*/false), + "Cannot 'dropTable' for system table 'Identifier{database='test_db', " + "table='tbl$system'}', please use data table."); ArrowSchemaRelease(&schema); } @@ -856,7 +898,7 @@ TEST(FileSystemCatalogTest, TestRenameTable) { ASSERT_NOK_WITH_MSG(catalog.RenameTable(Identifier("test_db", "tbl$system"), Identifier("test_db", "new_system_tbl"), /*ignore_if_not_exists=*/false), - "Cannot rename system table"); + "Cannot 'renameTable' for system table"); ArrowSchemaRelease(&schema1); ArrowSchemaRelease(&schema2); diff --git a/src/paimon/core/schema/table_schema.cpp b/src/paimon/core/schema/table_schema.cpp index 2beff595..e5139478 100644 --- a/src/paimon/core/schema/table_schema.cpp +++ b/src/paimon/core/schema/table_schema.cpp @@ -39,6 +39,7 @@ #include "paimon/core/schema/arrow_schema_validator.h" #include "paimon/defs.h" #include "paimon/status.h" +#include "paimon/utils/special_field_ids.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" #include "rapidjson/rapidjson.h" @@ -240,6 +241,67 @@ Result> TableSchema::CreateFromJson(const std::stri table_schema.options_, table_schema.comment_, table_schema.time_millis_); } +namespace { + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id); + +Status CollectTypeFieldIds(const rapidjson::Value& type, std::set* seen_ids, + int32_t* max_id) { + if (!type.IsObject()) { + return Status::OK(); + } + // ROW nests its fields under "fields", ARRAY/MULTISET under "element", MAP under "key"/"value". + if (type.HasMember("fields") && type["fields"].IsArray()) { + PAIMON_RETURN_NOT_OK(CollectFieldIds(type["fields"], seen_ids, max_id)); + } + if (type.HasMember("element")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["element"], seen_ids, max_id)); + } + if (type.HasMember("key")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["key"], seen_ids, max_id)); + } + if (type.HasMember("value")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(type["value"], seen_ids, max_id)); + } + return Status::OK(); +} + +Status CollectFieldIds(const rapidjson::Value& fields, std::set* seen_ids, + int32_t* max_id) { + for (const auto& field : fields.GetArray()) { + if (!field.IsObject()) { + return Status::Invalid("Broken schema, a field must be an object."); + } + if (!field.HasMember("id") || !field["id"].IsInt()) { + return Status::Invalid("Broken schema, a field misses an integer id."); + } + int32_t id = field["id"].GetInt(); + if (!seen_ids->insert(id).second) { + return Status::Invalid(fmt::format("Broken schema, field id {} is duplicated.", id)); + } + if (id < SpecialFieldIds::SYSTEM_FIELD_ID_START) { + *max_id = std::max(*max_id, id); + } + if (field.HasMember("type")) { + PAIMON_RETURN_NOT_OK(CollectTypeFieldIds(field["type"], seen_ids, max_id)); + } + } + return Status::OK(); +} + +} // namespace + +Result TableSchema::ComputeHighestFieldId(const rapidjson::Value& fields) { + if (!fields.IsArray()) { + return Status::Invalid("Broken schema, 'fields' must be an array."); + } + int32_t max_id = -1; + std::set seen_ids; + PAIMON_RETURN_NOT_OK(CollectFieldIds(fields, &seen_ids, &max_id)); + return max_id; +} + Result> TableSchema::InitSchema( int64_t schema_id, const std::vector& fields, int32_t highest_field_id, const std::vector& partition_keys, const std::vector& primary_keys, diff --git a/src/paimon/core/schema/table_schema.h b/src/paimon/core/schema/table_schema.h index bc0c94e4..eab109cf 100644 --- a/src/paimon/core/schema/table_schema.h +++ b/src/paimon/core/schema/table_schema.h @@ -54,6 +54,13 @@ class TableSchema : public DataSchema, public Jsonizable { static Result> CreateFromJson(const std::string& json_str); + /// Computes the highest non-system field id across all nesting levels of a "fields" JSON + /// array, or -1 when there is none; ids at or above `SpecialFieldIds::SYSTEM_FIELD_ID_START` + /// are excluded. A non-object field or a missing, non-integer or duplicated field id fails + /// the computation. Used when the schema source (e.g. a rest catalog server) does not + /// report the "highestFieldId" itself. + static Result ComputeHighestFieldId(const rapidjson::Value& fields); + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const noexcept(false) override; diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index 988f17f9..fd0ac422 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -30,6 +30,8 @@ #include "paimon/fs/local/local_file_system.h" #include "paimon/status.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/special_field_ids.h" +#include "rapidjson/document.h" namespace paimon::test { @@ -151,6 +153,43 @@ TEST_F(TableSchemaTest, TestGetFieldTypeForVariant) { ASSERT_EQ(variant_type, FieldType::VARIANT); } +TEST_F(TableSchemaTest, TestComputeHighestFieldId) { + auto compute = [](const std::string& fields_json) -> Result { + rapidjson::Document doc; + doc.Parse(fields_json.c_str()); + EXPECT_FALSE(doc.HasParseError()) << fields_json; + return TableSchema::ComputeHighestFieldId(doc); + }; + // ids inside nested rows are included + ASSERT_OK_AND_ASSIGN(int32_t highest, compute(R"([ + {"id": 0, "name": "f0", "type": "INT"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 5, "name": "inner", "type": "INT"}]}} + ])")); + ASSERT_EQ(5, highest); + // system field ids are reserved and excluded from the highest field id + ASSERT_OK_AND_ASSIGN(highest, compute(R"([{"id": 3, "name": "f0", "type": "INT"}, + {"id": )" + + std::to_string(SpecialFieldIds::SEQUENCE_NUMBER) + + R"(, "name": "_SEQUENCE_NUMBER", "type": "BIGINT"}])")); + ASSERT_EQ(3, highest); + ASSERT_OK_AND_ASSIGN(highest, compute("[]")); + ASSERT_EQ(-1, highest); + // broken schemas fail instead of being silently skipped + ASSERT_NOK_WITH_MSG(compute(R"([{"id": 0, "name": "a", "type": "INT"}, + {"id": 0, "name": "b", "type": "INT"}])") + .status(), + "duplicated"); + ASSERT_NOK_WITH_MSG(compute(R"([{"name": "a", "type": "INT"}])").status(), "integer id"); + ASSERT_NOK_WITH_MSG(compute(R"([{"id": "0", "name": "a", "type": "INT"}])").status(), + "integer id"); + ASSERT_NOK_WITH_MSG(compute("[1]").status(), "must be an object"); + rapidjson::Document not_an_array; + not_an_array.Parse("{}"); + ASSERT_NOK_WITH_MSG(TableSchema::ComputeHighestFieldId(not_an_array).status(), + "must be an array"); +} + TEST_F(TableSchemaTest, TestInvalidCreate) { // partial fields have field id arrow::FieldVector fields = MakeArrowField(3); diff --git a/src/paimon/core/snapshot.cpp b/src/paimon/core/snapshot.cpp index d3081c91..cd5f933f 100644 --- a/src/paimon/core/snapshot.cpp +++ b/src/paimon/core/snapshot.cpp @@ -315,4 +315,27 @@ Result Snapshot::FromPath(const std::shared_ptr& fs, return snapshot; } +SnapshotInfo Snapshot::ToSnapshotInfo() const { + SnapshotInfo info; + info.snapshot_id = Id(); + info.schema_id = SchemaId(); + info.commit_user = CommitUser(); + if (commit_kind_ == CommitKind::Append()) { + info.commit_kind = SnapshotInfo::CommitKind::APPEND; + } else if (commit_kind_ == CommitKind::Compact()) { + info.commit_kind = SnapshotInfo::CommitKind::COMPACT; + } else if (commit_kind_ == CommitKind::Overwrite()) { + info.commit_kind = SnapshotInfo::CommitKind::OVERWRITE; + } else if (commit_kind_ == CommitKind::Analyze()) { + info.commit_kind = SnapshotInfo::CommitKind::ANALYZE; + } else { + info.commit_kind = SnapshotInfo::CommitKind::UNKNOWN; + } + info.time_millis = TimeMillis(); + info.total_record_count = TotalRecordCount(); + info.delta_record_count = DeltaRecordCount(); + info.watermark = Watermark(); + return info; +} + } // namespace paimon diff --git a/src/paimon/core/snapshot.h b/src/paimon/core/snapshot.h index 1b7f6593..2d2cd061 100644 --- a/src/paimon/core/snapshot.h +++ b/src/paimon/core/snapshot.h @@ -27,6 +27,7 @@ #include "paimon/common/utils/jsonizable.h" #include "paimon/result.h" +#include "paimon/snapshot/snapshot_info.h" #include "paimon/type_fwd.h" #include "rapidjson/allocators.h" #include "rapidjson/document.h" @@ -131,6 +132,8 @@ class Snapshot : public Jsonizable { bool operator==(const Snapshot& other) const; bool TEST_Equal(const Snapshot& other) const; + SnapshotInfo ToSnapshotInfo() const; + public: static constexpr int64_t FIRST_SNAPSHOT_ID = 1; static constexpr int32_t TABLE_STORE_02_VERSION = 1; diff --git a/src/paimon/core/snapshot_test.cpp b/src/paimon/core/snapshot_test.cpp index 606e3200..7562f38d 100644 --- a/src/paimon/core/snapshot_test.cpp +++ b/src/paimon/core/snapshot_test.cpp @@ -329,6 +329,38 @@ TEST_F(SnapshotTest, TestSnapshotInfoCommitKindToString) { ASSERT_EQ("UNKNOWN", SnapshotInfo::CommitKindToString(SnapshotInfo::CommitKind::UNKNOWN)); } +TEST_F(SnapshotTest, TestToSnapshotInfo) { + auto make_snapshot = [](const Snapshot::CommitKind& kind) { + return Snapshot( + /*id=*/10, /*schema_id=*/15, /*base_manifest_list=*/"bml", + /*base_manifest_list_size=*/10, + /*delta_manifest_list=*/"dml", /*delta_manifest_list_size=*/20, + /*changelog_manifest_list=*/std::nullopt, /*changelog_manifest_list_size=*/std::nullopt, + /*index_manifest=*/std::nullopt, /*commit_user=*/"user1", /*commit_identifier=*/20, + kind, /*time_millis=*/1234, /*total_record_count=*/35, /*delta_record_count=*/40, + /*changelog_record_count=*/std::nullopt, /*watermark=*/50, + /*statistics=*/std::nullopt, /*properties=*/std::nullopt, + /*next_row_id=*/std::nullopt); + }; + SnapshotInfo info = make_snapshot(Snapshot::CommitKind::Append()).ToSnapshotInfo(); + ASSERT_EQ(10, info.snapshot_id); + ASSERT_EQ(15, info.schema_id); + ASSERT_EQ("user1", info.commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(1234, info.time_millis); + ASSERT_EQ(35, info.total_record_count.value()); + ASSERT_EQ(40, info.delta_record_count.value()); + ASSERT_EQ(50, info.watermark.value()); + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, + make_snapshot(Snapshot::CommitKind::Compact()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::OVERWRITE, + make_snapshot(Snapshot::CommitKind::Overwrite()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::ANALYZE, + make_snapshot(Snapshot::CommitKind::Analyze()).ToSnapshotInfo().commit_kind); + ASSERT_EQ(SnapshotInfo::CommitKind::UNKNOWN, + make_snapshot(Snapshot::CommitKind::Unknown()).ToSnapshotInfo().commit_kind); +} + TEST_F(SnapshotTest, TestChangelogManifestListSerialization) { // Test with changelog_manifest_list set to a non-null value { diff --git a/src/paimon/core/table/system/global_system_tables.cpp b/src/paimon/core/table/system/global_system_tables.cpp index bdfab0f0..dd1b4e60 100644 --- a/src/paimon/core/table/system/global_system_tables.cpp +++ b/src/paimon/core/table/system/global_system_tables.cpp @@ -20,6 +20,7 @@ #include "paimon/core/table/system/global_system_tables.h" #include +#include #include #include #include @@ -33,6 +34,7 @@ #include "paimon/common/data/generic_row.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" #include "paimon/core/io/data_file_meta.h" @@ -336,9 +338,14 @@ Result> CatalogOptionsSystemTable::ArrowSchema() Result> CatalogOptionsSystemTable::BuildRows() const { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, ArrowSchema()); + // Catalog options may carry credentials (e.g. the REST token), which must never reach + // anyone able to query this table; a secret is masked as a whole, an identifier-like + // value keeps at most a short tail. + const std::map masked_options = + SensitiveConfigUtils::RedactMap(context_.catalog_options); std::vector rows; - rows.reserve(context_.catalog_options.size()); - for (const auto& [key, value] : context_.catalog_options) { + rows.reserve(masked_options.size()); + for (const auto& [key, value] : masked_options) { GenericRow row(schema->num_fields()); row.SetField(0, StringValue(key)); row.SetField(1, StringValue(value)); diff --git a/src/paimon/fs/s3/s3_file_system.cpp b/src/paimon/fs/s3/s3_file_system.cpp index 4d896680..49668b70 100644 --- a/src/paimon/fs/s3/s3_file_system.cpp +++ b/src/paimon/fs/s3/s3_file_system.cpp @@ -45,52 +45,22 @@ #include #include "fmt/format.h" -#include "paimon/common/fs/http_client.h" +#include "paimon/common/utils/http_client.h" #include "paimon/common/utils/options_utils.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" #include "paimon/executor.h" namespace paimon::s3 { namespace { -Result PercentEncode(std::string_view value, bool preserve_slash) { - if (value.size() > std::numeric_limits::max()) { - return Status::IOError("S3 URL component is too large to encode"); - } - char* encoded = curl_easy_escape(nullptr, value.data(), static_cast(value.size())); - if (encoded == nullptr) { - return Status::IOError("failed to URL encode S3 component"); - } - ScopeGuard free_encoded([encoded] { curl_free(encoded); }); - std::string result(encoded); - if (preserve_slash) { - result = StringUtils::Replace(result, "%2F", "/"); - } - return result; -} - Result PercentDecode(std::string_view value, const std::string& field) { - for (size_t position = 0; position < value.size(); ++position) { - if (value[position] == '%' && - (position + 2 >= value.size() || - !std::isxdigit(static_cast(value[position + 1])) || - !std::isxdigit(static_cast(value[position + 2])))) { - return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); - } - } - if (value.size() > std::numeric_limits::max()) { - return Status::IOError(fmt::format("S3 {} is too large to URL decode", field)); + Result decoded = UrlUtils::PercentDecode(value); + if (!decoded.ok()) { + return Status::IOError(fmt::format("invalid URL encoding in S3 {}", field)); } - int decoded_size = 0; - char* decoded = - curl_easy_unescape(nullptr, value.data(), static_cast(value.size()), &decoded_size); - if (decoded == nullptr) { - return Status::IOError(fmt::format("failed to URL decode S3 {}", field)); - } - ScopeGuard free_decoded([decoded] { curl_free(decoded); }); - std::string result(decoded, decoded_size); - return result; + return decoded; } Result ParseNonNegativeInt64(const std::string& value, const std::string& field) { @@ -707,13 +677,10 @@ class S3ObjectStoreClient : public ObjectStoreClient, int32_t max_keys) const override { std::string query = "list-type=2&delimiter=%2F&encoding-type=url"; if (!path.key.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_prefix, PercentEncode(path.key, false)); - query += "&prefix=" + encoded_prefix; + query += "&prefix=" + UrlUtils::PercentEncode(path.key); } if (!continuation_token.empty()) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_token, - PercentEncode(continuation_token, false)); - query += "&continuation-token=" + encoded_token; + query += "&continuation-token=" + UrlUtils::PercentEncode(continuation_token); } if (max_keys > 0) { query += "&max-keys=" + std::to_string(max_keys); @@ -850,13 +817,11 @@ class S3ObjectStoreClient : public ObjectStoreClient, : endpoint_.scheme != "http" || IsIpAddressAuthority(endpoint_.authority) || !IsVirtualHostableS3Bucket(object.bucket, true)); if (use_path_style) { - PAIMON_ASSIGN_OR_RAISE(std::string encoded_bucket, PercentEncode(object.bucket, false)); - request_path += encoded_bucket + "/"; + request_path += UrlUtils::PercentEncode(object.bucket) + "/"; } else { authority = object.bucket + "." + authority; } - PAIMON_ASSIGN_OR_RAISE(std::string encoded_key, PercentEncode(object.key, true)); - request_path += encoded_key; + request_path += UrlUtils::PercentEncode(object.key, /*preserve_slash=*/true); if (!query.empty()) { request_path += "?" + query; } diff --git a/src/paimon/fs/s3/s3_file_system.h b/src/paimon/fs/s3/s3_file_system.h index 12caf8b7..8c4eebbc 100644 --- a/src/paimon/fs/s3/s3_file_system.h +++ b/src/paimon/fs/s3/s3_file_system.h @@ -23,8 +23,8 @@ #include #include -#include "paimon/common/fs/http_client.h" #include "paimon/common/fs/object_store_file_system.h" +#include "paimon/common/utils/http_client.h" namespace paimon::s3 { diff --git a/src/paimon/rest/mock_rest_server.cpp b/src/paimon/rest/mock_rest_server.cpp new file mode 100644 index 00000000..0bf4ec01 --- /dev/null +++ b/src/paimon/rest/mock_rest_server.cpp @@ -0,0 +1,218 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/mock_rest_server.h" + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { + +bool ReceiveAll(int32_t fd, std::string* buffer, size_t min_size) { + char chunk[4096]; + while (buffer->size() < min_size) { + ssize_t received = ::recv(fd, chunk, sizeof(chunk), 0); + if (received <= 0) { + return false; + } + buffer->append(chunk, static_cast(received)); + } + return true; +} + +std::map ParseQuery(const std::string& query) { + std::map params; + for (const std::string& pair : StringUtils::Split(query, "&", /*ignore_empty=*/true)) { + size_t eq = pair.find('='); + if (eq == std::string::npos) { + params[UrlUtils::DecodeString(pair)] = ""; + } else { + params[UrlUtils::DecodeString(pair.substr(0, eq))] = + UrlUtils::DecodeString(pair.substr(eq + 1)); + } + } + return params; +} + +} // namespace + +MockRestServer::MockRestServer(Handler handler, int32_t listen_fd, int32_t port) + : handler_(std::move(handler)), listen_fd_(listen_fd), port_(port) { + accept_thread_ = std::thread([this] { AcceptLoop(); }); +} + +Result> MockRestServer::Start(Handler handler) { + int32_t listen_fd = ::socket(AF_INET, SOCK_STREAM, 0); + if (listen_fd < 0) { + return Status::IOError("mock rest server: failed to create socket: ", std::strerror(errno)); + } + int32_t reuse = 1; + ::setsockopt(listen_fd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + struct sockaddr_in address; + std::memset(&address, 0, sizeof(address)); + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + address.sin_port = 0; + if (::bind(listen_fd, reinterpret_cast(&address), sizeof(address)) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to bind: ", std::strerror(errno)); + } + if (::listen(listen_fd, 16) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to listen: ", std::strerror(errno)); + } + socklen_t address_len = sizeof(address); + if (::getsockname(listen_fd, reinterpret_cast(&address), &address_len) < 0) { + ::close(listen_fd); + return Status::IOError("mock rest server: failed to get port: ", std::strerror(errno)); + } + int32_t port = ntohs(address.sin_port); + return std::unique_ptr(new MockRestServer(std::move(handler), listen_fd, port)); +} + +MockRestServer::~MockRestServer() { + Stop(); +} + +void MockRestServer::Stop() { + if (stopped_.exchange(true)) { + return; + } + // `shutdown` wakes the blocked `accept`; the fd is closed only after the accept + // thread has joined, so it cannot be recycled by another thread while the accept + // thread might still use it. + ::shutdown(listen_fd_, SHUT_RDWR); + if (accept_thread_.joinable()) { + accept_thread_.join(); + } + ::close(listen_fd_); +} + +std::string MockRestServer::GetBaseUri() const { + return fmt::format("http://127.0.0.1:{}", port_); +} + +void MockRestServer::AcceptLoop() { + while (!stopped_.load()) { + int32_t connection_fd = ::accept(listen_fd_, nullptr, nullptr); + if (connection_fd < 0) { + if (stopped_.load()) { + return; + } + continue; + } + // The connection is handled on the accept thread; the socket timeouts keep a + // wedged peer from blocking `Stop()` indefinitely. + struct timeval timeout; + timeout.tv_sec = 30; + timeout.tv_usec = 0; + ::setsockopt(connection_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + ::setsockopt(connection_fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + HandleConnection(connection_fd); + ::close(connection_fd); + } +} + +void MockRestServer::HandleConnection(int32_t connection_fd) { + std::string buffer; + size_t header_end; + while ((header_end = buffer.find("\r\n\r\n")) == std::string::npos) { + if (!ReceiveAll(connection_fd, &buffer, buffer.size() + 1)) { + return; + } + } + + Request request; + std::string header_part = buffer.substr(0, header_end); + std::vector lines = StringUtils::Split(header_part, "\r\n", + /*ignore_empty=*/true); + if (lines.empty()) { + return; + } + std::vector request_line = StringUtils::Split(lines[0], " ", + /*ignore_empty=*/true); + if (request_line.size() < 2) { + return; + } + request.method = request_line[0]; + std::string target = request_line[1]; + size_t question = target.find('?'); + if (question == std::string::npos) { + request.path = UrlUtils::DecodeString(target); + } else { + request.path = UrlUtils::DecodeString(target.substr(0, question)); + request.query_params = ParseQuery(target.substr(question + 1)); + } + size_t content_length = 0; + for (size_t i = 1; i < lines.size(); i++) { + size_t colon = lines[i].find(':'); + if (colon == std::string::npos) { + continue; + } + std::string name = lines[i].substr(0, colon); + std::string value = lines[i].substr(colon + 1); + StringUtils::Trim(&name); + StringUtils::Trim(&value); + request.headers[StringUtils::ToLowerCase(name)] = value; + } + auto length_iter = request.headers.find("content-length"); + if (length_iter != request.headers.end()) { + content_length = + static_cast(std::strtoul(length_iter->second.c_str(), nullptr, 10)); + } + size_t body_begin = header_end + 4; + if (!ReceiveAll(connection_fd, &buffer, body_begin + content_length)) { + return; + } + request.body = buffer.substr(body_begin, content_length); + + Response response = handler_(request); + if (response.close_without_response) { + return; + } + std::string extra_headers; + for (const auto& [name, value] : response.headers) { + extra_headers += fmt::format("{}: {}\r\n", name, value); + } + std::string payload = fmt::format( + "HTTP/1.1 {} MOCK\r\nContent-Type: {}\r\nContent-Length: {}\r\n{}Connection: " + "close\r\n\r\n{}", + response.code, response.content_type, response.body.size() + response.missing_body_bytes, + extra_headers, response.body); + size_t sent = 0; + while (sent < payload.size()) { + ssize_t written = ::send(connection_fd, payload.data() + sent, payload.size() - sent, 0); + if (written <= 0) { + return; + } + sent += static_cast(written); + } +} + +} // namespace paimon diff --git a/src/paimon/rest/mock_rest_server.h b/src/paimon/rest/mock_rest_server.h new file mode 100644 index 00000000..240f661e --- /dev/null +++ b/src/paimon/rest/mock_rest_server.h @@ -0,0 +1,91 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// A minimal blocking HTTP/1.1 server for REST catalog unit tests. It listens on a +/// random port of 127.0.0.1, parses one request per connection and answers it with the +/// response returned by the registered handler. Test only, not production code. +class MockRestServer { + public: + struct Request { + std::string method; + /// Url-decoded path, e.g. "/v1/config". + std::string path; + /// Url-decoded query parameters. + std::map query_params; + /// Headers with lower-cased names. + std::map headers; + std::string body; + }; + + struct Response { + int32_t code = 200; + std::string body; + std::string content_type = "application/json"; + std::map headers; + /// Close without answering; the client observes CURLE_GOT_NOTHING, a transport + /// error that is never retried. + bool close_without_response = false; + /// Advertise this many bytes beyond the actual body in `Content-Length`, then + /// close: the client sees the response cut off mid-body (CURLE_PARTIAL_FILE), a + /// retriable transport error. + size_t missing_body_bytes = 0; + }; + + using Handler = std::function; + + /// Starts the server; `handler` runs on the accept thread, so any state it touches + /// must be thread-safe. + static Result> Start(Handler handler); + + ~MockRestServer(); + + void Stop(); + + int32_t GetPort() const { + return port_; + } + + /// "http://127.0.0.1:" + std::string GetBaseUri() const; + + private: + MockRestServer(Handler handler, int32_t listen_fd, int32_t port); + + void AcceptLoop(); + void HandleConnection(int32_t connection_fd); + + Handler handler_; + int32_t listen_fd_ = -1; + int32_t port_ = 0; + std::atomic stopped_{false}; + std::thread accept_thread_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.cpp b/src/paimon/rest/resource_paths.cpp new file mode 100644 index 00000000..48448237 --- /dev/null +++ b/src/paimon/rest/resource_paths.cpp @@ -0,0 +1,64 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/resource_paths.h" + +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +namespace { +constexpr const char kV1[] = "/v1"; +} // namespace + +ResourcePaths::ResourcePaths(const std::string& prefix) { + base_ = kV1; + if (!prefix.empty()) { + base_ += "/" + UrlUtils::EncodeString(prefix); + } +} + +std::string ResourcePaths::Config() { + return std::string(kV1) + "/config"; +} + +std::string ResourcePaths::Databases() const { + return base_ + "/databases"; +} + +std::string ResourcePaths::Database(const std::string& database_name) const { + return Databases() + "/" + UrlUtils::EncodeString(database_name); +} + +std::string ResourcePaths::Tables(const std::string& database_name) const { + return Database(database_name) + "/tables"; +} + +std::string ResourcePaths::Table(const std::string& database_name, + const std::string& table_name) const { + return Tables(database_name) + "/" + UrlUtils::EncodeString(table_name); +} + +std::string ResourcePaths::RenameTable() const { + return base_ + "/tables/rename"; +} + +std::string ResourcePaths::Snapshots(const std::string& database_name, + const std::string& table_name) const { + return Table(database_name, table_name) + "/snapshots"; +} + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths.h b/src/paimon/rest/resource_paths.h new file mode 100644 index 00000000..ea36bf48 --- /dev/null +++ b/src/paimon/rest/resource_paths.h @@ -0,0 +1,45 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +namespace paimon { + +/// Builds resource paths of the REST catalog server. All path segments are +/// url-encoded; `prefix` (usually pushed down by the server through `/v1/config`) may +/// be empty, in which case it is skipped. +class ResourcePaths { + public: + explicit ResourcePaths(const std::string& prefix); + + /// "/v1/config", the only path that does not carry the prefix. + static std::string Config(); + + std::string Databases() const; + std::string Database(const std::string& database_name) const; + std::string Tables(const std::string& database_name) const; + std::string Table(const std::string& database_name, const std::string& table_name) const; + std::string RenameTable() const; + std::string Snapshots(const std::string& database_name, const std::string& table_name) const; + + private: + /// "/v1" or "/v1/{encoded prefix}". + std::string base_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/resource_paths_test.cpp b/src/paimon/rest/resource_paths_test.cpp new file mode 100644 index 00000000..782f3f69 --- /dev/null +++ b/src/paimon/rest/resource_paths_test.cpp @@ -0,0 +1,40 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/resource_paths.h" + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(ResourcePathsTest, WithPrefix) { + ResourcePaths paths("my prefix"); + ASSERT_EQ("/v1/config", ResourcePaths::Config()); + ASSERT_EQ("/v1/my+prefix/databases", paths.Databases()); + ASSERT_EQ("/v1/my+prefix/databases/db%231", paths.Database("db#1")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables", paths.Tables("db")); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1", paths.Table("db", "t1")); + ASSERT_EQ("/v1/my+prefix/tables/rename", paths.RenameTable()); + ASSERT_EQ("/v1/my+prefix/databases/db/tables/t1/snapshots", paths.Snapshots("db", "t1")); +} + +TEST(ResourcePathsTest, WithoutPrefix) { + ResourcePaths paths(""); + ASSERT_EQ("/v1/databases", paths.Databases()); + ASSERT_EQ("/v1/tables/rename", paths.RenameTable()); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_api.cpp b/src/paimon/rest/rest_api.cpp new file mode 100644 index 00000000..393a0938 --- /dev/null +++ b/src/paimon/rest/rest_api.cpp @@ -0,0 +1,280 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_api.h" + +#include +#include +#include + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/sensitive_config_utils.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { +// The `PAIMON_ASSIGN_OR_RAISE` macro cannot take a declaration type containing a comma. +using StringMap = std::map; + +// A successful response body may carry credentials (e.g. a token response), so neither it +// nor the parse failure quoting it is echoed into the error message; only the request path +// is reported. +template +Status ParseResponseBody(const std::string& body, const std::string& path, ResponseT* entity) { + if (!RapidJsonUtil::FromJsonString(body, entity).ok()) { + return Status::Invalid( + fmt::format("failed to deserialize the response of {} from the rest server", path)); + } + return Status::OK(); +} +} // namespace + +RestApi::RestApi(std::unique_ptr client, + std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths) + : client_(std::move(client)), + auth_provider_(std::move(auth_provider)), + base_headers_(base_headers), + options_(options), + resource_paths_(paths) {} + +Result> RestApi::Create(const std::map& options, + const std::string& warehouse, bool config_required, + const RestHttpClient::Config& http_config) { + auto uri_iter = options.find(CatalogOptions::URI); + if (uri_iter == options.end() || uri_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::URI)); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr client, + RestHttpClient::Create(uri_iter->second, http_config)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr auth_provider, + AuthProvider::Create(options)); + + std::map merged_options = options; + std::map base_headers = + RestUtil::ExtractPrefixMap(options, kHeaderOptionPrefix); + if (config_required) { + std::map query_params; + if (!warehouse.empty()) { + query_params[kQueryParamWarehouse] = warehouse; + } + RestAuthParameter auth_parameter = + RestAuthParameter::Create("GET", ResourcePaths::Config(), query_params, ""); + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider->MergeAuthHeader(base_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE( + RestHttpClient::Response response, + client->Execute("GET", ResourcePaths::Config(), query_params, headers, "")); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + ConfigResponse config; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, ResourcePaths::Config(), &config)); + merged_options = config.Merge(options); + for (const auto& [key, value] : + RestUtil::ExtractPrefixMap(merged_options, kHeaderOptionPrefix)) { + base_headers[key] = value; + } + std::shared_ptr logger = Logger::GetLogger("RestApi"); + PAIMON_LOG_DEBUG(logger, + "merged %zu client options with the /v1/config response into %zu options", + options.size(), merged_options.size()); + } + std::string prefix; + auto prefix_iter = merged_options.find(kOptionUrlPrefix); + if (prefix_iter != merged_options.end()) { + prefix = prefix_iter->second; + } + return std::unique_ptr(new RestApi(std::move(client), std::move(auth_provider), + base_headers, merged_options, + ResourcePaths(prefix))); +} + +Status RestApi::ErrorToStatus(const RestHttpClient::Response& response) { + // The code of the parsed error body takes precedence over the http status, which + // a gateway may have rewritten. + int64_t code = response.code; + std::string message; + std::string resource_info; + bool body_parsed = false; + if (!response.body.empty()) { + ErrorResponse error; + if (RapidJsonUtil::FromJsonString(response.body, &error).ok()) { + body_parsed = true; + // The message may embed secrets (e.g. "password=..."), so it is redacted. + message = SensitiveConfigUtils::RedactText(error.GetMessage()); + if (!error.GetResourceType().empty()) { + resource_info = fmt::format(" (resource type: {}, resource name: {})", + error.GetResourceType(), error.GetResourceName()); + } + if (error.GetCode() != 0) { + code = error.GetCode(); + } + } + } + if (message.empty()) { + // The body is never echoed (it may carry credentials), so a server answering with + // something other than an error object is told apart by the message alone. + message = body_parsed ? fmt::format("empty error message (http status {})", response.code) + : fmt::format("unparsable error response body (http status {})", + response.code); + } + message += resource_info; + std::string request_id = RestUtil::ExtractRequestId(response.headers); + if (request_id != RestUtil::kUnknownRequestId) { + message += fmt::format(" requestId:{}", request_id); + } + Status status; + switch (code) { + case HttpStatus::kBadRequest: + status = Status::Invalid(message); + break; + case HttpStatus::kUnauthorized: + status = Status::IOError("not authorized: ", message); + break; + case HttpStatus::kForbidden: + status = Status::IOError("forbidden: ", message); + break; + case HttpStatus::kNotFound: + status = Status::NotExist(message); + break; + case HttpStatus::kConflict: + status = Status::Exist(message); + break; + case HttpStatus::kInternalServerError: + status = Status::IOError("server error: ", message); + break; + case HttpStatus::kNotImplemented: + status = Status::NotImplemented(message); + break; + case HttpStatus::kServiceUnavailable: + status = Status::IOError("service unavailable: ", message); + break; + default: + status = + Status::IOError(fmt::format("rest request failed with code {}: {}", code, message)); + break; + } + return status.WithDetail(std::make_shared(code)); +} + +Result RestApi::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, const std::string& body) const { + RestAuthParameter auth_parameter = RestAuthParameter::Create(method, path, query_params, body); + StringMap request_headers = base_headers_; + if (!body.empty()) { + // Set before merging the auth headers so a provider that signs headers covers it. + request_headers["Content-Type"] = "application/json"; + } + PAIMON_ASSIGN_OR_RAISE(StringMap headers, + auth_provider_->MergeAuthHeader(request_headers, auth_parameter)); + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + client_->Execute(method, path, query_params, headers, body)); + if (!response.IsSuccessful()) { + return ErrorToStatus(response); + } + return response; +} + +template +Result RestApi::GetEntity(const std::string& path, + const std::map& query_params) const { + PAIMON_ASSIGN_OR_RAISE(RestHttpClient::Response response, + Execute("GET", path, query_params, "")); + ResponseT entity; + PAIMON_RETURN_NOT_OK(ParseResponseBody(response.body, path, &entity)); + return entity; +} + +template +Result> RestApi::ListAllPages( + const std::string& path) const { + std::vector items; + std::map query_params; + while (true) { + PAIMON_ASSIGN_OR_RAISE(ResponseT response, GetEntity(path, query_params)); + const auto& data = response.Data(); + items.insert(items.end(), data.begin(), data.end()); + const std::optional& next_page_token = response.NextPageToken(); + if (!next_page_token || next_page_token.value().empty() || data.empty()) { + return items; + } + query_params[kQueryParamPageToken] = next_page_token.value(); + } +} + +Result> RestApi::ListDatabases() const { + return ListAllPages(resource_paths_.Databases()); +} + +Status RestApi::CreateDatabase(const std::string& name, + const std::map& options) const { + CreateDatabaseRequest request(name, options); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Databases(), {}, body).status(); +} + +Result RestApi::GetDatabase(const std::string& name) const { + return GetEntity(resource_paths_.Database(name), {}); +} + +Status RestApi::DropDatabase(const std::string& name) const { + return Execute("DELETE", resource_paths_.Database(name), {}, "").status(); +} + +Result> RestApi::ListTables(const std::string& database_name) const { + return ListAllPages(resource_paths_.Tables(database_name)); +} + +Result RestApi::GetTable(const Identifier& identifier) const { + return GetEntity( + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), {}); +} + +Status RestApi::CreateTable(const Identifier& identifier, const std::string& schema_json) const { + CreateTableRequest request(identifier.GetDatabaseName(), identifier.GetTableName(), + schema_json); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.Tables(identifier.GetDatabaseName()), {}, body).status(); +} + +Status RestApi::DropTable(const Identifier& identifier) const { + return Execute("DELETE", + resource_paths_.Table(identifier.GetDatabaseName(), identifier.GetTableName()), + {}, "") + .status(); +} + +Status RestApi::RenameTable(const Identifier& from_table, const Identifier& to_table) const { + RenameTableRequest request(from_table.GetDatabaseName(), from_table.GetTableName(), + to_table.GetDatabaseName(), to_table.GetTableName()); + PAIMON_ASSIGN_OR_RAISE(std::string body, request.ToJsonString()); + return Execute("POST", resource_paths_.RenameTable(), {}, body).status(); +} + +Result> RestApi::ListSnapshots(const Identifier& identifier) const { + return ListAllPages( + resource_paths_.Snapshots(identifier.GetDatabaseName(), identifier.GetTableName())); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_api.h b/src/paimon/rest/rest_api.h new file mode 100644 index 00000000..c2394ced --- /dev/null +++ b/src/paimon/rest/rest_api.h @@ -0,0 +1,138 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/catalog/identifier.h" +#include "paimon/core/snapshot.h" +#include "paimon/rest/resource_paths.h" +#include "paimon/rest/rest_auth.h" +#include "paimon/rest/rest_http_client.h" +#include "paimon/rest/rest_messages.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Carries the code of a failed rest request (the parsed error body's code, falling back +/// to the http status) so callers can tell e.g. an authentication failure (401/403) from +/// another IO error. +class RestErrorDetail : public StatusDetail { + public: + static constexpr const char* kTypeId = "rest-error-detail"; + + explicit RestErrorDetail(int64_t code) : code_(code) {} + + const char* type_id() const override { + return kTypeId; + } + + std::string ToString() const override { + return "rest error code " + std::to_string(code_); + } + + int64_t GetCode() const { + return code_; + } + + private: + int64_t code_; +}; + +/// The client of the REST catalog server. This layer only talks HTTP + JSON and never +/// touches the file system. +class RestApi { + public: + static constexpr const char* kQueryParamPageToken = "pageToken"; + static constexpr const char* kQueryParamWarehouse = "warehouse"; + /// Option key of the url path prefix inserted after "/v1", usually pushed down by + /// "/v1/config". + static constexpr const char* kOptionUrlPrefix = "prefix"; + /// Options with this prefix are sent as http headers (with the prefix stripped). + static constexpr const char* kHeaderOptionPrefix = "header."; + + /// Creates the api client. + /// + /// @param options Client side options; `CatalogOptions::URI` and + /// `CatalogOptions::TOKEN_PROVIDER` are required. + /// @param warehouse Warehouse sent as query parameter of "/v1/config"; may be empty. + /// @param config_required When true, fetch "/v1/config" and merge the server + /// defaults/overrides into `options` with the precedence + /// overrides > client options > defaults. + /// @param http_config Transport level settings, mainly overridable for tests. + static Result> Create( + const std::map& options, const std::string& warehouse, + bool config_required, const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + /// Options merged with the server side config. + const std::map& GetMergedOptions() const { + return options_; + } + + Result> ListDatabases() const; + Status CreateDatabase(const std::string& name, + const std::map& options) const; + Result GetDatabase(const std::string& name) const; + Status DropDatabase(const std::string& name) const; + + Result> ListTables(const std::string& database_name) const; + Result GetTable(const Identifier& identifier) const; + /// `schema_json` uses the schema JSON layout of the protocol + /// (fields/partitionKeys/primaryKeys/options/comment). + Status CreateTable(const Identifier& identifier, const std::string& schema_json) const; + Status DropTable(const Identifier& identifier) const; + Status RenameTable(const Identifier& from_table, const Identifier& to_table) const; + + Result> ListSnapshots(const Identifier& identifier) const; + + /// Maps a non-successful http response to a status: 404 becomes `NotExist`, 409 + /// becomes `Exist`, 400 becomes `Invalid`, 501 becomes `NotImplemented` and the + /// other codes become `IOError`. The status carries a `RestErrorDetail` with the + /// mapped code. + static Status ErrorToStatus(const RestHttpClient::Response& response); + + private: + RestApi(std::unique_ptr client, std::unique_ptr auth_provider, + const std::map& base_headers, + const std::map& options, const ResourcePaths& paths); + + /// Executes one request with the authentication headers merged in, mapping a + /// non-successful response to an error status. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::string& body) const; + + template + Result GetEntity(const std::string& path, + const std::map& query_params) const; + + /// Fetches all pages, stopping at an empty page or a missing/empty next page token. + template + Result> ListAllPages(const std::string& path) const; + + std::unique_ptr client_; + std::unique_ptr auth_provider_; + std::map base_headers_; + std::map options_; + ResourcePaths resource_paths_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.cpp b/src/paimon/rest/rest_auth.cpp new file mode 100644 index 00000000..65d6d755 --- /dev/null +++ b/src/paimon/rest/rest_auth.cpp @@ -0,0 +1,71 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_auth.h" + +#include "fmt/format.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" + +namespace paimon { + +RestAuthParameter RestAuthParameter::Create(const std::string& method, + const std::string& resource_path, + const std::map& query_params, + const std::string& data) { + RestAuthParameter parameter; + parameter.method = method; + parameter.resource_path = resource_path; + for (const auto& [key, value] : query_params) { + parameter.parameters[key] = UrlUtils::EncodeString(value); + } + parameter.data = data; + return parameter; +} + +Result> BearTokenAuthProvider::MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const { + std::map headers = base_header; + headers["Authorization"] = "Bearer " + token_; + return headers; +} + +Result> AuthProvider::Create( + const std::map& options) { + auto provider_iter = options.find(CatalogOptions::TOKEN_PROVIDER); + if (provider_iter == options.end() || provider_iter->second.empty()) { + return Status::Invalid(fmt::format("option '{}' must be configured for the rest catalog", + CatalogOptions::TOKEN_PROVIDER)); + } + // Matched leniently in lower case; other clients may match the provider name + // case-sensitively, so only the exact "bear" spelling is portable. + std::string provider = StringUtils::ToLowerCase(provider_iter->second); + if (provider == "bear") { + auto token_iter = options.find(CatalogOptions::TOKEN); + if (token_iter == options.end() || token_iter->second.empty()) { + return Status::Invalid( + fmt::format("option '{}' must be configured for the bear token provider", + CatalogOptions::TOKEN)); + } + return std::make_unique(token_iter->second); + } + return Status::NotImplemented( + fmt::format("unsupported token provider: {}, only 'bear' is supported for now", provider)); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_auth.h b/src/paimon/rest/rest_auth.h new file mode 100644 index 00000000..22da08d9 --- /dev/null +++ b/src/paimon/rest/rest_auth.h @@ -0,0 +1,73 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// Input of one request signature. +struct RestAuthParameter { + std::string method; + std::string resource_path; + /// Query parameters with url-encoded values. + std::map parameters; + /// Request body, empty when the request carries none. + std::string data; + + /// Builds the parameter of one request. `query_params` are stored url-encoded, since + /// a signing provider signs the query string as it is sent; building the parameter + /// only through here keeps a request from being signed over unencoded values. + static RestAuthParameter Create(const std::string& method, const std::string& resource_path, + const std::map& query_params, + const std::string& data); +}; + +/// Generates authentication headers for REST catalog requests. +class AuthProvider { + public: + virtual ~AuthProvider() = default; + + /// Returns `base_header` merged with the authentication headers of this provider. + virtual Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const = 0; + + /// Creates the provider configured by `CatalogOptions::TOKEN_PROVIDER`. + static Result> Create( + const std::map& options); +}; + +/// Adds `Authorization: Bearer `. +class BearTokenAuthProvider : public AuthProvider { + public: + explicit BearTokenAuthProvider(const std::string& token) : token_(token) {} + + Result> MergeAuthHeader( + const std::map& base_header, + const RestAuthParameter& parameter) const override; + + private: + std::string token_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.cpp b/src/paimon/rest/rest_catalog.cpp new file mode 100644 index 00000000..6706bb50 --- /dev/null +++ b/src/paimon/rest/rest_catalog.cpp @@ -0,0 +1,442 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include + +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/catalog/catalog_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/system/global_system_tables.h" +#include "paimon/core/table/system/system_table.h" +#include "paimon/core/table/system/system_table_schema.h" +#include "paimon/defs.h" +#include "paimon/fs/file_system.h" +#include "paimon/rest/rest_util.h" +#include "rapidjson/document.h" + +namespace paimon { + +namespace { + +constexpr const char kPathOption[] = "path"; + +// Maps the default branch "main" to "no branch": it is addressed as the bare table. The +// comparison ignores case, as the identifier the Java client sends is built the same way, +// so "MAIN" also resolves to the bare table and cannot address a branch of that name. +// `BranchManager::IsMainBranch`, which names the branch directory of a table, stays +// case-sensitive: this normalization only decides how a table is addressed on the server. +std::optional NormalizeBranch(std::optional branch) { + if (branch && StringUtils::ToLowerCase(branch.value()) == Identifier::kDefaultMainBranch) { + return std::nullopt; + } + return branch; +} + +// Builds the "$branch_" object name addressing `branch` of `table_name` +// on the rest server. +std::string BranchObjectName(std::string table_name, const std::string& branch) { + table_name.append(Identifier::kSystemTableSplitter); + table_name.append(Identifier::kSystemBranchPrefix); + table_name.append(branch); + return table_name; +} + +// Builds the identifier sent to the rest server: the system table suffix is stripped +// while the branch stays in the object name, so the server resolves the branch itself and +// returns the branch's own schema. The path the server reports is the data table root in +// either case; the branch subdirectory is derived downstream from the branch option and +// must not be applied twice (see `ToTableSchema`). +Result ToLoadIdentifier(const Identifier& identifier) { + PAIMON_ASSIGN_OR_RAISE(std::string data_table_name, identifier.GetDataTableName()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + std::string object_name = branch ? BranchObjectName(std::move(data_table_name), branch.value()) + : std::move(data_table_name); + return Identifier(identifier.GetDatabaseName(), object_name); +} + +} // namespace + +RestCatalog::RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse) + : api_(std::move(api)), + fs_(fs), + warehouse_(warehouse), + table_default_options_(RestUtil::ExtractPrefixMap( + api_->GetMergedOptions(), CatalogOptions::TABLE_DEFAULT_OPTION_PREFIX)), + logger_(Logger::GetLogger("RestCatalog")) {} + +Result> RestCatalog::Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, const RestHttpClient::Config& http_config) { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr api, + RestApi::Create(options, warehouse, /*config_required=*/true, http_config)); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, + CoreOptions::FromMap(api->GetMergedOptions(), file_system)); + return std::unique_ptr( + new RestCatalog(std::move(api), core_options.GetFileSystem(), warehouse)); +} + +const std::map& RestCatalog::GetOptions() const { + return api_->GetMergedOptions(); +} + +Status RestCatalog::CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "createDatabase")); + Status status = api_->CreateDatabase(name, options); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ListDatabases() const { + return api_->ListDatabases(); +} + +Result RestCatalog::DatabaseExists(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return true; + } + Result response = api_->GetDatabase(db_name); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Status RestCatalog::DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemDatabase(name, "dropDatabase")); + if (!cascade) { + Result> tables = ListTables(name); + if (!tables.ok()) { + if (tables.status().IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return tables.status(); + } + if (!tables.value().empty()) { + return Status::Invalid( + fmt::format("Cannot drop non-empty database {}. Use cascade=true to force.", name)); + } + } + Status status = api_->DropDatabase(name); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +std::string RestCatalog::GetDatabaseLocation(const std::string& db_name) const { + // The virtual "sys" database has no location and is unknown to the server. + if (CatalogUtils::IsSystemDatabase(db_name)) { + return ""; + } + Result response = api_->GetDatabase(db_name); + if (!response.ok()) { + PAIMON_LOG_WARN(logger_, "failed to get location of database %s: %s", db_name.c_str(), + response.status().ToString().c_str()); + return ""; + } + return response.value().GetLocation(); +} + +Result> RestCatalog::ListTables(const std::string& db_name) const { + if (CatalogUtils::IsSystemDatabase(db_name)) { + return GlobalSystemTableLoader::GetSupportedTableNames(api_->GetMergedOptions()); + } + return api_->ListTables(db_name); +} + +Status RestCatalog::CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "createTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "createTable")); + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(c_schema)); + std::map effective_options = options; + for (const auto& [key, value] : table_default_options_) { + effective_options.emplace(key, value); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_schema, + TableSchema::Create(TableSchema::FIRST_SCHEMA_ID, schema, partition_keys, + primary_keys, effective_options)); + std::string schema_json; + try { + rapidjson::Document doc; + doc.SetObject(); + rapidjson::Document::AllocatorType& allocator = doc.GetAllocator(); + doc.AddMember(rapidjson::StringRef("fields"), + RapidJsonUtil::SerializeValue(table_schema->Fields(), &allocator).Move(), + allocator); + doc.AddMember(rapidjson::StringRef("partitionKeys"), + RapidJsonUtil::SerializeValue(partition_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("primaryKeys"), + RapidJsonUtil::SerializeValue(primary_keys, &allocator).Move(), allocator); + doc.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(effective_options, &allocator).Move(), + allocator); + schema_json = RestUtil::JsonToString(doc); + } catch (const std::exception& e) { + return Status::SerializationError("failed to serialize create table schema: ", e.what()); + } + Status status = api_->CreateTable(identifier, schema_json); + if (status.IsExist() && ignore_if_exists) { + return Status::OK(); + } + return status; +} + +Result RestCatalog::TableExists(const Identifier& identifier) const { + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + return GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions()); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return false; + } + } + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + Result response = api_->GetTable(load_identifier); + if (response.ok()) { + return true; + } + if (response.status().IsNotExist()) { + return false; + } + return response.status(); +} + +Result RestCatalog::GetTableLocation(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(load_identifier)); + return response.GetPath(); +} + +Status RestCatalog::DropTable(const Identifier& identifier, bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "dropTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "dropTable")); + Status status = api_->DropTable(identifier); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Status RestCatalog::RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(to_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(from_table, "renameTable")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(to_table, "renameTable")); + Status status = api_->RenameTable(from_table, to_table); + if (status.IsNotExist() && ignore_if_not_exists) { + return Status::OK(); + } + return status; +} + +Result> RestCatalog::ToTableSchema( + const GetTableResponse& response, const std::optional& branch) { + std::string table_schema_json; + try { + rapidjson::Document src; + src.Parse(response.GetSchemaJson().c_str()); + if (src.HasParseError() || !src.IsObject() || !src.HasMember("fields") || + !src["fields"].IsArray()) { + return Status::Invalid("invalid table schema json from the rest server"); + } + rapidjson::Document out; + out.SetObject(); + rapidjson::Document::AllocatorType& allocator = out.GetAllocator(); + out.AddMember(rapidjson::StringRef("version"), TableSchema::CURRENT_VERSION, allocator); + out.AddMember(rapidjson::StringRef("id"), response.GetSchemaId(), allocator); + PAIMON_ASSIGN_OR_RAISE(int32_t highest_field_id, + TableSchema::ComputeHighestFieldId(src["fields"])); + rapidjson::Value fields(rapidjson::kArrayType); + fields.CopyFrom(src["fields"], allocator); + out.AddMember(rapidjson::StringRef("highestFieldId"), highest_field_id, allocator); + out.AddMember(rapidjson::StringRef("fields"), fields.Move(), allocator); + // A missing or wrong-typed member fails instead of defaulting to an empty value, + // which would silently change table semantics (an omitted "partitionKeys" would + // load a partitioned table as unpartitioned); Java rejects absent members too. + for (const char* key : {"partitionKeys", "primaryKeys"}) { + if (!src.HasMember(key)) { + return Status::Invalid(fmt::format( + "invalid table schema json from the rest server: missing '{}'", key)); + } + if (!src[key].IsArray()) { + return Status::Invalid(fmt::format( + "invalid table schema json from the rest server: '{}' is not an array", key)); + } + rapidjson::Value keys(rapidjson::kArrayType); + keys.CopyFrom(src[key], allocator); + out.AddMember(rapidjson::StringRef(key), keys.Move(), allocator); + } + if (!src.HasMember("options")) { + return Status::Invalid( + "invalid table schema json from the rest server: missing 'options'"); + } + if (!src["options"].IsObject()) { + return Status::Invalid( + "invalid table schema json from the rest server: 'options' is not an object"); + } + auto options_map = + RapidJsonUtil::DeserializeValue>(src["options"]); + options_map[kPathOption] = response.GetPath(); + response.GetAuditFields().PutAuditOptionsTo(&options_map); + if (branch) { + options_map[Options::BRANCH] = branch.value(); + } + out.AddMember(rapidjson::StringRef("options"), + RapidJsonUtil::SerializeValue(options_map, &allocator).Move(), allocator); + if (src.HasMember("comment") && src["comment"].IsString()) { + rapidjson::Value comment; + comment.CopyFrom(src["comment"], allocator); + out.AddMember(rapidjson::StringRef("comment"), comment.Move(), allocator); + } + // The server's audit time keeps the conversion deterministic; 0 (the epoch) + // when the server did not report an "updatedAt". + out.AddMember(rapidjson::StringRef("timeMillis"), + response.GetAuditFields().updated_at.value_or(0), allocator); + table_schema_json = RestUtil::JsonToString(out); + } catch (const std::exception& e) { + return Status::Invalid("failed to convert rest table schema: ", e.what()); + } + return TableSchema::CreateFromJson(table_schema_json); +} + +Result> RestCatalog::LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const { + PAIMON_ASSIGN_OR_RAISE(GetTableResponse response, api_->GetTable(data_identifier)); + if (table_path != nullptr) { + *table_path = response.GetPath(); + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr schema, ToTableSchema(response, branch)); + return std::shared_ptr(std::move(schema)); +} + +Result> RestCatalog::LoadTableSchema(const Identifier& identifier) const { + // Serve the global system tables of the "sys" database locally, like + // FileSystemCatalog. + if (CatalogUtils::IsSystemDatabase(identifier.GetDatabaseName())) { + PAIMON_ASSIGN_OR_RAISE(bool supported, + GlobalSystemTableLoader::IsSupported(identifier.GetTableName(), + api_->GetMergedOptions())); + if (!supported) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + GlobalSystemTableContext context; + context.catalog = this; + context.fs = fs_; + context.warehouse = warehouse_; + context.catalog_options = api_->GetMergedOptions(); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + GlobalSystemTableLoader::Load(identifier.GetTableName(), context)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(bool is_system_table, identifier.IsSystemTable()); + PAIMON_ASSIGN_OR_RAISE(std::optional branch, identifier.GetBranchName()); + branch = NormalizeBranch(std::move(branch)); + PAIMON_ASSIGN_OR_RAISE(Identifier load_identifier, ToLoadIdentifier(identifier)); + if (is_system_table) { + PAIMON_ASSIGN_OR_RAISE(std::optional system_table_name, + identifier.GetSystemTableName()); + if (!system_table_name || !SystemTableLoader::IsSupported(system_table_name.value())) { + return Status::NotExist(fmt::format("{} not exist", identifier.ToString())); + } + std::string table_path; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr latest_schema, + LoadDataTableSchema(load_identifier, branch, &table_path)); + std::map dynamic_options; + if (branch) { + dynamic_options[Options::BRANCH] = branch.value(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr system_table, + SystemTableLoader::Load(system_table_name.value(), fs_, table_path, + latest_schema, dynamic_options)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr arrow_schema, + system_table->ArrowSchema()); + return std::make_shared(std::move(arrow_schema)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, + LoadDataTableSchema(load_identifier, branch, nullptr)); + return std::static_pointer_cast(schema); +} + +Result> RestCatalog::GetTable(const Identifier& identifier) const { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr schema, LoadTableSchema(identifier)); + return std::make_shared
(schema, identifier.GetDatabaseName(), identifier.GetTableName()); +} + +std::string RestCatalog::GetRootPath() const { + return warehouse_; +} + +std::shared_ptr RestCatalog::GetFileSystem() const { + return fs_; +} + +Result> RestCatalog::ListSnapshots(const Identifier& identifier, + const std::string& branch) const { + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotBranch(identifier, "listSnapshots")); + PAIMON_RETURN_NOT_OK(CatalogUtils::CheckNotSystemTable(identifier, "listSnapshots")); + std::optional normalized_branch = + NormalizeBranch(branch.empty() ? std::nullopt : std::make_optional(branch)); + std::string object_name = + normalized_branch ? BranchObjectName(identifier.GetTableName(), normalized_branch.value()) + : identifier.GetTableName(); + Identifier load_identifier(identifier.GetDatabaseName(), object_name); + // The Catalog interface has no pagination, so all pages are fetched; the server + // does not order snapshots across pages while the contract requires ascending ids. + PAIMON_ASSIGN_OR_RAISE(std::vector snapshots, api_->ListSnapshots(load_identifier)); + std::sort(snapshots.begin(), snapshots.end(), + [](const Snapshot& a, const Snapshot& b) { return a.Id() < b.Id(); }); + std::vector result; + result.reserve(snapshots.size()); + for (const auto& snapshot : snapshots) { + result.push_back(snapshot.ToSnapshotInfo()); + } + return result; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog.h b/src/paimon/rest/rest_catalog.h new file mode 100644 index 00000000..984aad70 --- /dev/null +++ b/src/paimon/rest/rest_catalog.h @@ -0,0 +1,101 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/catalog/catalog.h" +#include "paimon/logging.h" +#include "paimon/rest/rest_api.h" +#include "paimon/result.h" +#include "paimon/status.h" + +struct ArrowSchema; + +namespace paimon { +class FileSystem; +class TableSchema; + +/// A catalog backed by a REST catalog server. Metadata operations are delegated to +/// `RestApi`; table data is accessed through the file system configured by the +/// server-merged options. +class RestCatalog : public Catalog { + public: + /// Creates the catalog: fetches and merges "/v1/config" from the server configured + /// by `CatalogOptions::URI`, then builds the file system from the merged options. + /// + /// @param warehouse The warehouse identifier sent to the server; may be empty. + static Result> Create( + const std::string& warehouse, const std::map& options, + const std::shared_ptr& file_system, + const RestHttpClient::Config& http_config = RestHttpClient::Config()); + + Status CreateDatabase(const std::string& name, + const std::map& options, + bool ignore_if_exists) override; + Status CreateTable(const Identifier& identifier, ArrowSchema* c_schema, + const std::vector& partition_keys, + const std::vector& primary_keys, + const std::map& options, + bool ignore_if_exists) override; + Status DropDatabase(const std::string& name, bool ignore_if_not_exists, bool cascade) override; + Status DropTable(const Identifier& identifier, bool ignore_if_not_exists) override; + Status RenameTable(const Identifier& from_table, const Identifier& to_table, + bool ignore_if_not_exists) override; + Result> ListDatabases() const override; + Result> ListTables(const std::string& db_name) const override; + Result DatabaseExists(const std::string& db_name) const override; + Result TableExists(const Identifier& identifier) const override; + std::string GetDatabaseLocation(const std::string& db_name) const override; + Result GetTableLocation(const Identifier& identifier) const override; + Result> LoadTableSchema(const Identifier& identifier) const override; + std::string GetRootPath() const override; + std::shared_ptr GetFileSystem() const override; + Result> GetTable(const Identifier& identifier) const override; + Result> ListSnapshots(const Identifier& identifier, + const std::string& branch) const override; + + /// Options merged with the server side config. + const std::map& GetOptions() const override; + + private: + RestCatalog(std::unique_ptr api, const std::shared_ptr& fs, + const std::string& warehouse); + + /// Loads the table from the server and converts the response to a `TableSchema` + /// (options are enriched with the table path, audit info and branch). + Result> LoadDataTableSchema( + const Identifier& data_identifier, const std::optional& branch, + std::string* table_path) const; + + static Result> ToTableSchema( + const GetTableResponse& response, const std::optional& branch); + + std::unique_ptr api_; + std::shared_ptr fs_; + std::string warehouse_; + /// The "table-default." options of the merged config, applied to `CreateTable` + /// options when absent. + std::map table_default_options_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_catalog_test.cpp b/src/paimon/rest/rest_catalog_test.cpp new file mode 100644 index 00000000..242104fb --- /dev/null +++ b/src/paimon/rest/rest_catalog_test.cpp @@ -0,0 +1,1047 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_catalog.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/table.h" +#include "paimon/catalog_options.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/defs.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/rest/rest_api.h" +#include "paimon/schema/schema.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { + +constexpr const char kToken[] = "test-token"; +constexpr const char kPrefix[] = "paimon"; +constexpr const char kWarehouse[] = "wh1"; + +// The in-memory catalog state behind the mock rest server. +struct MockCatalogState { + struct TableData { + std::string schema_json; + int64_t schema_id = 0; + std::string path; + }; + std::map> databases; + // headers of the last request, with lower-cased names + std::map last_headers; + // when set, every request except "/v1/config" fails with this http code + std::optional force_error_code; + // guards all fields above: the handler runs on the server's accept thread while + // tests seed and inspect the state + std::mutex mutex; +}; + +MockRestServer::Response JsonResponse(int32_t code, const std::string& body) { + MockRestServer::Response response; + response.code = code; + response.body = body; + return response; +} + +MockRestServer::Response MockError(int32_t code, const std::string& resource_type, + const std::string& resource_name, const std::string& message) { + ErrorResponse error(resource_type, resource_name, message, code); + return JsonResponse(code, error.ToJsonString().value()); +} + +std::string SnapshotJson(int64_t id) { + return fmt::format( + R"({{"version":3,"id":{},"schemaId":0,"baseManifestList":"bml","deltaManifestList":"dml",)" + R"("commitUser":"user1","commitIdentifier":1,"commitKind":"APPEND","timeMillis":100,)" + R"("totalRecordCount":10,"deltaRecordCount":1}})", + id); +} + +std::string TableResponseJson(const std::string& name, const MockCatalogState::TableData& table) { + return fmt::format( + R"({{"id":"1","name":"{}","path":"{}","isExternal":false,"schemaId":{},"schema":{},)" + R"("owner":"owner1","updatedAt":123}})", + name, table.path, table.schema_id, table.schema_json); +} + +// Serves `names` one item per page to exercise the pagination loop of the client. +std::pair, std::optional> PageOf( + const std::vector& names, const MockRestServer::Request& request) { + size_t index = 0; + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter != request.query_params.end()) { + // A malformed token must not throw: an exception on the accept thread would + // terminate the test binary. + index = static_cast( + StringUtils::StringToValue(token_iter->second).value_or(0)); + } + std::vector page; + std::optional next_page_token; + if (index < names.size()) { + page.push_back(names[index]); + if (index + 1 < names.size()) { + next_page_token = std::to_string(index + 1); + } + } + return {page, next_page_token}; +} + +// Implements the subset of the rest catalog protocol used by `RestCatalog` on top of +// `MockCatalogState`. +MockRestServer::Response HandleCatalogRequest(MockCatalogState* state, + const MockRestServer::Request& request) { + std::lock_guard lock(state->mutex); + state->last_headers = request.headers; + auto auth_iter = request.headers.find("authorization"); + if (auth_iter == request.headers.end() || + auth_iter->second != std::string("Bearer ") + kToken) { + return MockError(401, "", "", "invalid token"); + } + if (request.path == "/v1/config") { + auto warehouse_iter = request.query_params.find("warehouse"); + if (warehouse_iter == request.query_params.end() || warehouse_iter->second != kWarehouse) { + return MockError(400, "", "", "unexpected warehouse"); + } + ConfigResponse config( + {{RestApi::kOptionUrlPrefix, kPrefix}, + {"header.x-server-header", "from-config"}, + {"table-default.write-only", "true"}, + {"table-default.bucket", "8"}}, + {{"server-override", "from-server"}, {"header.x-shared-header", "from-config"}}); + return JsonResponse(200, config.ToJsonString().value()); + } + if (state->force_error_code) { + return MockError(state->force_error_code.value(), "", "", "injected failure"); + } + const std::string base = std::string("/v1/") + kPrefix; + if (request.path.rfind(base, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string rest = request.path.substr(base.size()); + + if (rest == "/databases") { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, tables] : state->databases) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListDatabasesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateDatabaseRequest create_request("", {}); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create database request"); + } + if (state->databases.count(create_request.GetName()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeDatabase, + create_request.GetName(), "database already exists"); + } + state->databases[create_request.GetName()] = {}; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + if (rest == "/tables/rename" && request.method == "POST") { + RenameTableRequest rename_request("", "", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &rename_request).ok()) { + return MockError(400, "", "", "bad rename table request"); + } + auto db_iter = state->databases.find(rename_request.GetSourceDatabase()); + if (db_iter == state->databases.end() || + db_iter->second.count(rename_request.GetSourceTable()) == 0) { + return MockError(404, ErrorResponse::kResourceTypeTable, + rename_request.GetSourceTable(), "table not found"); + } + auto& dest_tables = state->databases[rename_request.GetDestinationDatabase()]; + if (dest_tables.count(rename_request.GetDestinationTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, + rename_request.GetDestinationTable(), "table already exists"); + } + dest_tables[rename_request.GetDestinationTable()] = + db_iter->second[rename_request.GetSourceTable()]; + db_iter->second.erase(rename_request.GetSourceTable()); + return JsonResponse(200, ""); + } + + const std::string databases_prefix = "/databases/"; + if (rest.rfind(databases_prefix, 0) != 0) { + return MockError(404, "", "", "unknown path " + request.path); + } + std::string remainder = rest.substr(databases_prefix.size()); + size_t tables_pos = remainder.find("/tables"); + + if (tables_pos == std::string::npos) { + const std::string& db_name = remainder; + auto db_iter = state->databases.find(db_name); + if (request.method == "GET") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + std::string body = fmt::format( + R"({{"id":"1","name":"{}","location":"{}/{}.db","options":{{"dbk":"dbv"}}}})", + db_name, kWarehouse, db_name); + return JsonResponse(200, body); + } + if (request.method == "DELETE") { + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, + "database not found"); + } + state->databases.erase(db_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + std::string db_name = remainder.substr(0, tables_pos); + auto db_iter = state->databases.find(db_name); + if (db_iter == state->databases.end()) { + return MockError(404, ErrorResponse::kResourceTypeDatabase, db_name, "database not found"); + } + auto& tables = db_iter->second; + std::string table_part = remainder.substr(tables_pos + std::strlen("/tables")); + + if (table_part.empty()) { + if (request.method == "GET") { + std::vector names; + for (const auto& [name, table] : tables) { + names.push_back(name); + } + auto [page, next_page_token] = PageOf(names, request); + ListTablesResponse response(page, next_page_token); + return JsonResponse(200, response.ToJsonString().value()); + } + if (request.method == "POST") { + CreateTableRequest create_request("", "", ""); + if (!RapidJsonUtil::FromJsonString(request.body, &create_request).ok()) { + return MockError(400, "", "", "bad create table request"); + } + if (tables.count(create_request.GetTable()) > 0) { + return MockError(409, ErrorResponse::kResourceTypeTable, create_request.GetTable(), + "table already exists"); + } + MockCatalogState::TableData table; + table.schema_json = create_request.GetSchemaJson(); + table.schema_id = 0; + table.path = fmt::format("{}/{}.db/{}", kWarehouse, db_name, create_request.GetTable()); + tables[create_request.GetTable()] = table; + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); + } + + // "/{table}" or "/{table}/snapshots" + std::string table_name = table_part.substr(1); + bool list_snapshots = false; + const std::string snapshots_suffix = "/snapshots"; + if (table_name.size() > snapshots_suffix.size() && + table_name.compare(table_name.size() - snapshots_suffix.size(), snapshots_suffix.size(), + snapshots_suffix) == 0) { + table_name = table_name.substr(0, table_name.size() - snapshots_suffix.size()); + list_snapshots = true; + } + auto table_iter = tables.find(table_name); + if (table_iter == tables.end()) { + return MockError(404, ErrorResponse::kResourceTypeTable, table_name, "table not found"); + } + if (list_snapshots) { + // two pages, out of order to exercise pagination and sorting + auto token_iter = request.query_params.find(RestApi::kQueryParamPageToken); + if (token_iter == request.query_params.end()) { + return JsonResponse( + 200, fmt::format(R"({{"snapshots":[{}],"nextPageToken":"1"}})", SnapshotJson(2))); + } + return JsonResponse(200, fmt::format(R"({{"snapshots":[{}]}})", SnapshotJson(1))); + } + if (request.method == "GET") { + return JsonResponse(200, TableResponseJson(table_name, table_iter->second)); + } + if (request.method == "DELETE") { + tables.erase(table_iter); + return JsonResponse(200, ""); + } + return MockError(400, "", "", "unsupported method"); +} + +} // namespace + +class RestCatalogTest : public ::testing::Test { + protected: + void SetUp() override { + state_ = std::make_shared(); + ASSERT_OK_AND_ASSIGN( + server_, MockRestServer::Start([state = state_](const MockRestServer::Request& req) { + return HandleCatalogRequest(state.get(), req); + })); + options_ = { + {CatalogOptions::METASTORE, "rest"}, + {CatalogOptions::URI, server_->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + {Options::FILE_SYSTEM, "local"}, + // mock_format is linked statically into the test binary, so its factory is + // registered in the binary's own registry even when the real format plugin + // dylibs register into a different one (macOS two-level namespace) + {Options::FILE_FORMAT, "mock_format"}, + {Options::MANIFEST_FORMAT, "mock_format"}, + {"header.x-client-header", "from-client"}, + {"header.x-shared-header", "from-client"}, + }; + } + + void TearDown() override { + if (server_) { + server_->Stop(); + } + } + + Result> CreateRestCatalog() { + return RestCatalog::Create(kWarehouse, options_, nullptr); + } + + Status CreateSampleTable(Catalog* catalog, const Identifier& identifier, + bool ignore_if_exists = false) { + std::shared_ptr schema = + arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::utf8())}); + struct ArrowSchema c_schema; + if (!arrow::ExportSchema(*schema, &c_schema).ok()) { + return Status::Invalid("failed to export arrow schema"); + } + Status status = + catalog->CreateTable(identifier, &c_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"f0"}, {{"bucket", "2"}}, ignore_if_exists); + // CreateTable takes ownership of the exported schema only once it reaches + // arrow::ImportSchema, which an identifier rejected by its checks never does + if (c_schema.release != nullptr) { + c_schema.release(&c_schema); + } + return status; + } + + // Seeds `schema_json` as table `table_name` of "db1" behind the mock server and + // expects loading the table to fail with an Invalid status carrying + // `expected_message`. + void ExpectBrokenSchemaRejected(Catalog* catalog, const std::string& table_name, + const std::string& schema_json, + const std::string& expected_message) { + MockCatalogState::TableData table_data; + table_data.schema_json = schema_json; + table_data.path = "wh1/db1.db/" + table_name; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"][table_name] = table_data; + } + Status status = catalog->GetTable(Identifier("db1", table_name)).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, expected_message); + } + + std::shared_ptr state_; + std::unique_ptr server_; + std::map options_; +}; + +TEST_F(RestCatalogTest, CreateMergesServerConfig) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + const std::map& merged = catalog->GetOptions(); + ASSERT_EQ(kPrefix, merged.at(RestApi::kOptionUrlPrefix)); + ASSERT_EQ("from-server", merged.at("server-override")); + ASSERT_EQ(kWarehouse, catalog->GetRootPath()); + ASSERT_NE(nullptr, catalog->GetFileSystem()); +} + +TEST_F(RestCatalogTest, CatalogFactoryMetastoreDispatch) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(kWarehouse, options_)); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_TRUE(databases.empty()); + options_[CatalogOptions::METASTORE] = "something-else"; + Status status = Catalog::Create(kWarehouse, options_).status(); + ASSERT_TRUE(status.IsInvalid()) << status.ToString(); + ASSERT_NOK_WITH_MSG(status, "unsupported metastore"); +} + +TEST_F(RestCatalogTest, CreateWithWrongTokenFails) { + options_[CatalogOptions::TOKEN] = "wrong-token"; + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "not authorized"); +} + +TEST_F(RestCatalogTest, CreateRejectsInvalidOptions) { + // all rejected by client side validation, before any request reaches the server + const std::map valid_options = options_; + + options_.erase(CatalogOptions::URI); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'uri' must be configured"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN_PROVIDER); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "'token.provider' must be configured"); + + options_ = valid_options; + options_[CatalogOptions::TOKEN_PROVIDER] = "dlf"; + Status unsupported_provider = CreateRestCatalog().status(); + ASSERT_TRUE(unsupported_provider.IsNotImplemented()) << unsupported_provider.ToString(); + ASSERT_NOK_WITH_MSG(unsupported_provider, "unsupported token provider"); + + options_ = valid_options; + options_.erase(CatalogOptions::TOKEN); + ASSERT_NOK_WITH_MSG(CreateRestCatalog().status(), "bear token provider"); +} + +TEST_F(RestCatalogTest, DatabaseOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateDatabase("db2", {}, /*ignore_if_exists=*/false)); + Status duplicated = catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/true)); + + // the mock server returns one database per page + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + ASSERT_EQ((std::vector{"db1", "db2"}), databases); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("db1")); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db3")); + ASSERT_FALSE(exists); + + ASSERT_EQ("wh1/db1.db", catalog->GetDatabaseLocation("db1")); + ASSERT_EQ("", catalog->GetDatabaseLocation("db3")); + + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, /*cascade=*/false)); + ASSERT_OK(catalog->DropDatabase("db2", /*ignore_if_not_exists=*/true, /*cascade=*/false)); + Status missing = catalog->DropDatabase("db2", /*ignore_if_not_exists=*/false, + /*cascade=*/false); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_NOK_WITH_MSG( + catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/false), + "non-empty database"); + // cascade drop skips the emptiness check + ASSERT_OK(catalog->DropDatabase("db1", /*ignore_if_not_exists=*/false, /*cascade=*/true)); + ASSERT_OK_AND_ASSIGN(exists, catalog->DatabaseExists("db1")); + ASSERT_FALSE(exists); +} + +TEST_F(RestCatalogTest, TableOperations) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + + Status missing_db = CreateSampleTable(catalog.get(), Identifier("db_missing", "t1")); + ASSERT_TRUE(missing_db.IsNotExist()) << missing_db.ToString(); + + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + Status duplicated = CreateSampleTable(catalog.get(), identifier); + ASSERT_TRUE(duplicated.IsExist()) << duplicated.ToString(); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier, /*ignore_if_exists=*/true)); + + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1"}), tables); + Status list_missing = catalog->ListTables("db_missing").status(); + ASSERT_TRUE(list_missing.IsNotExist()) << list_missing.ToString(); + + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(identifier)); + ASSERT_EQ("t1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ((std::vector{"f0", "f1"}), schema->FieldNames()); + ASSERT_EQ((std::vector{"f0"}), schema->PrimaryKeys()); + ASSERT_EQ(0, schema->Id()); + // options are enriched with the table path and audit info from the server + ASSERT_EQ("wh1/db1.db/t1", schema->Options().at("path")); + ASSERT_EQ("owner1", schema->Options().at("owner")); + + // "table-default." options of the merged config apply only where the caller left the + // option unset: "write-only" is taken from the config, "bucket" keeps the value passed + // to CreateTable instead of the configured "table-default.bucket" of 8 + ASSERT_EQ("true", schema->Options().at("write-only")); + ASSERT_EQ("2", schema->Options().at("bucket")); + + // timeMillis is backed by the server's audit "updatedAt" instead of the current + // time, keeping the conversion deterministic + std::shared_ptr table_schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, table_schema); + ASSERT_EQ(123, table_schema->TimeMillis()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr loaded_schema, + catalog->LoadTableSchema(identifier)); + ASSERT_EQ((std::vector{"f0", "f1"}), loaded_schema->FieldNames()); + Status schema_missing = catalog->LoadTableSchema(Identifier("db1", "t2")).status(); + ASSERT_TRUE(schema_missing.IsNotExist()) << schema_missing.ToString(); + + // a schema comment of the server response is carried into the table schema + MockCatalogState::TableData commented; + commented.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {},)" + R"( "comment": "a table comment"})"; + commented.path = "wh1/db1.db/commented"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["commented"] = commented; + } + ASSERT_OK_AND_ASSIGN(std::shared_ptr
commented_table, + catalog->GetTable(Identifier("db1", "commented"))); + ASSERT_EQ("a table comment", commented_table->LatestSchema()->Comment().value_or("")); + + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false)); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t2"))); + ASSERT_TRUE(exists); + ASSERT_OK(catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/true)); + Status rename_missing = catalog->RenameTable(identifier, Identifier("db1", "t3"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(rename_missing.IsNotExist()) << rename_missing.ToString(); + + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/false)); + ASSERT_OK(catalog->DropTable(Identifier("db1", "t2"), /*ignore_if_not_exists=*/true)); + Status drop_missing = catalog->DropTable(Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false); + ASSERT_TRUE(drop_missing.IsNotExist()) << drop_missing.ToString(); +} + +TEST_F(RestCatalogTest, ClientAndServerHeadersAreSent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK_AND_ASSIGN(std::vector databases, catalog->ListDatabases()); + // "header." options from both the client and the merged server config are sent as + // http headers on every request + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("from-client", state_->last_headers.at("x-client-header")); + ASSERT_EQ("from-config", state_->last_headers.at("x-server-header")); + // when the client and the server config set the same "header." option, the + // merged config wins (overrides > client options > defaults) + ASSERT_EQ("from-config", state_->last_headers.at("x-shared-header")); + ASSERT_EQ(std::string("Bearer ") + kToken, state_->last_headers.at("authorization")); + } + // a request carrying a body declares the json content type (set before the auth + // headers are merged, so a signing auth provider covers it) + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + { + std::lock_guard lock(state_->mutex); + ASSERT_EQ("application/json", state_->last_headers.at("content-type")); + } +} + +TEST_F(RestCatalogTest, SystemTableSchema) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the "options" system table has a static schema and needs no file system access + Identifier system_identifier("db1", "t1$options"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(system_identifier)); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t9$options"))); + ASSERT_FALSE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(system_identifier)); + ASSERT_EQ((std::vector{"key", "value"}), schema->FieldNames()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(system_identifier)); + ASSERT_EQ("t1$options", table->Name()); + ASSERT_EQ((std::vector{"key", "value"}), table->LatestSchema()->FieldNames()); + + Status unsupported = catalog->LoadTableSchema(Identifier("db1", "t1$unsupported")).status(); + ASSERT_TRUE(unsupported.IsNotExist()) << unsupported.ToString(); +} + +TEST_F(RestCatalogTest, BranchTableLoadsBranchSchemaFromServer) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + + // the identifier sent to the server keeps the branch, so the server resolves it and + // returns the branch's own schema. The path it reports is the data table root, not + // the branch subdirectory: readers derive "/branch/branch-" from the + // branch option, so a branch path here would be applied twice + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "b0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.schema_id = 3; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + + Identifier branch_identifier("db1", "t1$branch_b1"); + ASSERT_OK_AND_ASSIGN(bool exists, catalog->TableExists(branch_identifier)); + ASSERT_TRUE(exists); + // a branch the server does not know is missing instead of silently falling back to + // the main table + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("db1", "t1$branch_missing"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::string location, catalog->GetTableLocation(branch_identifier)); + ASSERT_EQ("wh1/db1.db/t1", location); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(branch_identifier)); + ASSERT_EQ("t1$branch_b1", table->Name()); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + // the branch's own schema and schema id, not the main table's + ASSERT_EQ((std::vector{"b0"}), schema->FieldNames()); + ASSERT_EQ(3, schema->Id()); + ASSERT_EQ("b1", schema->Options().at(Options::BRANCH)); + + // the default branch is addressed as the bare table: "t1$branch_main" resolves to + // "t1" and carries no branch option + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_table, + catalog->GetTable(Identifier("db1", "t1$branch_main"))); + std::shared_ptr main_schema = + std::dynamic_pointer_cast(main_table->LatestSchema()); + ASSERT_NE(nullptr, main_schema); + ASSERT_EQ((std::vector{"f0", "f1"}), main_schema->FieldNames()); + ASSERT_EQ(0, main_schema->Options().count(Options::BRANCH)); + + // the default branch is matched ignoring case, as in the Java client, so + // "t1$branch_MAIN" addresses the bare table too + ASSERT_OK_AND_ASSIGN(std::shared_ptr
main_case_table, + catalog->GetTable(Identifier("db1", "t1$branch_MAIN"))); + std::shared_ptr main_case_schema = + std::dynamic_pointer_cast(main_case_table->LatestSchema()); + ASSERT_NE(nullptr, main_case_schema); + ASSERT_EQ((std::vector{"f0", "f1"}), main_case_schema->FieldNames()); + ASSERT_EQ(0, main_case_schema->Options().count(Options::BRANCH)); + + // a system table on a branch resolves against the branch's data table: the system + // suffix is stripped while the branch stays in the identifier sent to the server + ASSERT_OK_AND_ASSIGN(std::shared_ptr options_schema, + catalog->LoadTableSchema(Identifier("db1", "t1$branch_b1$options"))); + ASSERT_EQ((std::vector{"key", "value"}), options_schema->FieldNames()); + // a missing branch fails through the system table path too instead of silently + // resolving against the main table + Status missing_branch = + catalog->LoadTableSchema(Identifier("db1", "t1$branch_missing$options")).status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + ASSERT_NOK_WITH_MSG(catalog->DropTable(branch_identifier, /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(catalog->RenameTable(branch_identifier, Identifier("db1", "t2"), + /*ignore_if_not_exists=*/false), + "branch table"); + ASSERT_NOK_WITH_MSG(CreateSampleTable(catalog.get(), Identifier("db1", "t2$branch_b1")), + "branch table"); +} + +TEST_F(RestCatalogTest, NestedSchemaHighestFieldId) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + MockCatalogState::TableData table_data; + table_data.schema_json = R"({ + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "s", "type": {"type": "ROW", + "fields": [{"id": 3, "name": "inner", "type": "INT"}]}}, + {"id": 2, "name": "arr", "type": {"type": "ARRAY", + "element": {"type": "ROW", + "fields": [{"id": 7, "name": "deep", "type": "BIGINT"}]}}}, + {"id": 4, "name": "m", "type": {"type": "MAP", + "key": {"type": "ROW NOT NULL", + "fields": [{"id": 8, "name": "k", "type": "INT NOT NULL"}]}, + "value": {"type": "ROW", + "fields": [{"id": 9, "name": "v", "type": "INT"}]}}} + ], + "partitionKeys": [], + "primaryKeys": [], + "options": {} + })"; + table_data.schema_id = 5; + table_data.path = "wh1/db1.db/nested"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["nested"] = table_data; + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, + catalog->GetTable(Identifier("db1", "nested"))); + std::shared_ptr schema = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, schema); + ASSERT_EQ(5, schema->Id()); + // 9 lives inside the value row of the map: ROW, ARRAY element and MAP key/value + // must all be traversed + ASSERT_EQ(9, schema->HighestFieldId()); + ASSERT_EQ((std::vector{"f0", "s", "arr", "m"}), schema->FieldNames()); +} + +TEST_F(RestCatalogTest, ListTablesPaged) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t1"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t2"))); + ASSERT_OK(CreateSampleTable(catalog.get(), Identifier("db1", "t3"))); + ASSERT_OK_AND_ASSIGN(std::vector tables, catalog->ListTables("db1")); + ASSERT_EQ((std::vector{"t1", "t2", "t3"}), tables); +} + +TEST_F(RestCatalogTest, BrokenSchemaRejected) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + ExpectBrokenSchemaRejected(catalog.get(), "duplicate", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 0, "name": "f1", "type": "STRING"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // an id inside a nested row colliding with an outer field id is a duplicate too + ExpectBrokenSchemaRejected(catalog.get(), "duplicate_nested", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"},)" + R"( {"id": 1, "name": "s", "type": {"type": "ROW",)" + R"( "fields": [{"id": 0, "name": "inner", "type": "INT"}]}}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "duplicated"); + // a field without an integer id must fail the conversion instead of being + // silently skipped when computing highestFieldId + ExpectBrokenSchemaRejected(catalog.get(), "no_id", + R"({"fields": [{"name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + ExpectBrokenSchemaRejected(catalog.get(), "string_id", + R"({"fields": [{"id": "0", "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "integer id"); + // a field that is not an object fails too instead of being silently skipped + ExpectBrokenSchemaRejected(catalog.get(), "non_object", + R"({"fields": [1],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})", + "must be an object"); + // a missing or wrong-typed member is a visible failure instead of silently + // defaulting to an empty value (e.g. loading a partitioned table as + // unpartitioned) + ExpectBrokenSchemaRejected(catalog.get(), "missing_partition_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "primaryKeys": [], "options": {}})", + "missing 'partitionKeys'"); + ExpectBrokenSchemaRejected(catalog.get(), "missing_primary_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "options": {}})", + "missing 'primaryKeys'"); + ExpectBrokenSchemaRejected(catalog.get(), "missing_options", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": []})", + "missing 'options'"); + ExpectBrokenSchemaRejected(catalog.get(), "wrong_typed_partition_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": {}, "primaryKeys": [], "options": {}})", + "'partitionKeys' is not an array"); + ExpectBrokenSchemaRejected(catalog.get(), "wrong_typed_primary_keys", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": "f0", "options": {}})", + "'primaryKeys' is not an array"); + ExpectBrokenSchemaRejected(catalog.get(), "wrong_typed_options", + R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": []})", + "'options' is not an object"); +} + +TEST_F(RestCatalogTest, PartitionKeysRoundTrip) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + std::shared_ptr schema = + arrow::schema({arrow::field("f0", arrow::int32(), /*nullable=*/false), + arrow::field("f1", arrow::utf8(), /*nullable=*/false)}); + struct ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK(catalog->CreateTable(Identifier("db1", "pt"), &c_schema, + /*partition_keys=*/{"f1"}, /*primary_keys=*/{}, {}, + /*ignore_if_exists=*/false)); + // the partition keys survive both the create request and the load response conversion + ASSERT_OK_AND_ASSIGN(std::shared_ptr
table, catalog->GetTable(Identifier("db1", "pt"))); + std::shared_ptr loaded = + std::dynamic_pointer_cast(table->LatestSchema()); + ASSERT_NE(nullptr, loaded); + ASSERT_EQ((std::vector{"f1"}), loaded->PartitionKeys()); +} + +TEST_F(RestCatalogTest, ServerErrorIsPropagatedNotMappedToAbsent) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + { + std::lock_guard lock(state_->mutex); + state_->force_error_code = 500; + } + // a server failure surfaces as an error instead of "does not exist" + Status db_status = catalog->DatabaseExists("db1").status(); + ASSERT_NOK_WITH_MSG(db_status, "server error"); + Status table_status = catalog->TableExists(Identifier("db1", "t1")).status(); + ASSERT_NOK_WITH_MSG(table_status, "server error"); +} + +TEST_F(RestCatalogTest, SystemTableChecks) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_NOK_WITH_MSG(catalog->CreateDatabase("sys", {}, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropDatabase("sys", false, false), "system database"); + ASSERT_NOK_WITH_MSG(catalog->DropTable(Identifier("sys", "t"), false), "system table"); + ASSERT_NOK_WITH_MSG( + catalog->RenameTable(Identifier("db1", "t1$snapshots"), Identifier("db1", "t2"), false), + "system table"); +} + +TEST_F(RestCatalogTest, SystemDatabase) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + // the "sys" database and its global system tables are resolved locally without + // contacting the server, like in FileSystemCatalog + ASSERT_OK_AND_ASSIGN(bool exists, catalog->DatabaseExists("sys")); + ASSERT_TRUE(exists); + + ASSERT_OK_AND_ASSIGN(std::vector sys_tables, catalog->ListTables("sys")); + ASSERT_TRUE(std::find(sys_tables.begin(), sys_tables.end(), "tables") != sys_tables.end()); + + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "tables"))); + ASSERT_TRUE(exists); + ASSERT_OK_AND_ASSIGN(exists, catalog->TableExists(Identifier("sys", "unsupported"))); + ASSERT_FALSE(exists); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr schema, + catalog->LoadTableSchema(Identifier("sys", "tables"))); + ASSERT_FALSE(schema->FieldNames().empty()); + Status missing = catalog->LoadTableSchema(Identifier("sys", "unsupported")).status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); +} + +TEST_F(RestCatalogTest, ListSnapshots) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, CreateRestCatalog()); + ASSERT_OK(catalog->CreateDatabase("db1", {}, /*ignore_if_exists=*/false)); + Identifier identifier("db1", "t1"); + ASSERT_OK(CreateSampleTable(catalog.get(), identifier)); + + ASSERT_OK_AND_ASSIGN(std::vector snapshots, + catalog->ListSnapshots(identifier, "")); + ASSERT_EQ(2, snapshots.size()); + // fetched via two pages and sorted by snapshot id + ASSERT_EQ(1, snapshots[0].snapshot_id); + ASSERT_EQ(2, snapshots[1].snapshot_id); + ASSERT_EQ("user1", snapshots[0].commit_user); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, snapshots[0].commit_kind); + + // the default branch is addressed as the bare table, so passing it explicitly + // equals the branch-less call + ASSERT_OK_AND_ASSIGN(std::vector main_snapshots, + catalog->ListSnapshots(identifier, "main")); + ASSERT_EQ(2, main_snapshots.size()); + // the match ignores case, as in the Java client, so "MAIN" is the default branch too + ASSERT_OK_AND_ASSIGN(std::vector main_case_snapshots, + catalog->ListSnapshots(identifier, "MAIN")); + ASSERT_EQ(2, main_case_snapshots.size()); + + Status missing = catalog->ListSnapshots(Identifier("db1", "t9"), "").status(); + ASSERT_TRUE(missing.IsNotExist()) << missing.ToString(); + + // a non-main branch is sent under its branch object name, so the server resolves + // the branch and lists its own snapshots + MockCatalogState::TableData branch_data; + branch_data.schema_json = R"({"fields": [{"id": 0, "name": "f0", "type": "INT NOT NULL"}],)" + R"( "partitionKeys": [], "primaryKeys": [], "options": {}})"; + branch_data.path = "wh1/db1.db/t1"; + { + std::lock_guard lock(state_->mutex); + state_->databases["db1"]["t1$branch_b1"] = branch_data; + } + ASSERT_OK_AND_ASSIGN(std::vector branch_snapshots, + catalog->ListSnapshots(identifier, "b1")); + ASSERT_EQ(2, branch_snapshots.size()); + // a branch the server does not know is missing instead of silently falling back + // to the main table + Status missing_branch = catalog->ListSnapshots(identifier, "b_missing").status(); + ASSERT_TRUE(missing_branch.IsNotExist()) << missing_branch.ToString(); + + // a branch must be passed as the branch argument, not encoded in the identifier + ASSERT_NOK_WITH_MSG(catalog->ListSnapshots(Identifier("db1", "t1$branch_b1"), "").status(), + "branch table"); +} + +TEST(RestApiErrorTest, ErrorToStatus) { + RestHttpClient::Response response; + response.code = 404; + response.body = R"({"message": "no table", "resourceType": "TABLE", "resourceName": "t1"})"; + response.headers["x-request-id"] = "req-123"; + Status status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(status.IsNotExist()); + ASSERT_TRUE(status.ToString().find("requestId:req-123") != std::string::npos) + << status.ToString(); + + response.code = 409; + Status exist_status = RestApi::ErrorToStatus(response); + ASSERT_TRUE(exist_status.IsExist()); + + response.code = 501; + response.body = ""; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotImplemented()); + + // a body that is not an error object at all is reported as such, since the body + // itself is never echoed + response.code = 500; + response.body = "not-a-json"; + Status unparsable = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(unparsable, "server error"); + ASSERT_NOK_WITH_MSG(unparsable, "unparsable error response body (http status 500)"); + ASSERT_EQ(std::string::npos, unparsable.ToString().find("not-a-json")) << unparsable.ToString(); + + // the code of the error body wins over the http status when they disagree (e.g. a + // gateway rewriting the status) + response.code = 500; + response.body = R"({"message": "gone", "code": 404})"; + ASSERT_TRUE(RestApi::ErrorToStatus(response).IsNotExist()); + + // an error object without a message is told apart from an unparsable body, and the + // resource info is kept + response.code = 404; + response.body = R"({"resourceType": "TABLE", "resourceName": "t1"})"; + Status empty_message = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(empty_message, "empty error message (http status 404)"); + ASSERT_TRUE(empty_message.ToString().find("resource name: t1") != std::string::npos) + << empty_message.ToString(); + + // server messages that may embed secrets are redacted as a whole + response.code = 400; + response.body = R"({"message": "bad option password=abc123", "code": 400})"; + Status redacted = RestApi::ErrorToStatus(response); + ASSERT_TRUE(redacted.IsInvalid()) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("abc123") == std::string::npos) << redacted.ToString(); + ASSERT_TRUE(redacted.ToString().find("******") != std::string::npos) << redacted.ToString(); + + // any header carrying a request id is used when x-request-id is absent + response.code = 404; + response.body = ""; + response.headers.clear(); + response.headers["x-amz-request-id"] = "amz-1"; + Status fallback = RestApi::ErrorToStatus(response); + ASSERT_TRUE(fallback.ToString().find("requestId:amz-1") != std::string::npos) + << fallback.ToString(); + + // the "unknown" placeholder is not a real request id + response.headers.clear(); + response.headers["x-request-id"] = "unknown"; + Status unknown_id = RestApi::ErrorToStatus(response); + ASSERT_TRUE(unknown_id.ToString().find("requestId") == std::string::npos) + << unknown_id.ToString(); + + // 401/403 map to IOError; the mapped code is carried as a status detail so + // callers can distinguish them + response.headers.clear(); + response.code = 401; + Status not_authorized = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(not_authorized, "not authorized"); + ASSERT_NE(nullptr, not_authorized.detail()); + ASSERT_EQ(std::string(RestErrorDetail::kTypeId), not_authorized.detail()->type_id()); + ASSERT_EQ(401, std::static_pointer_cast(not_authorized.detail())->GetCode()); + response.code = 403; + Status forbidden = RestApi::ErrorToStatus(response); + ASSERT_NOK_WITH_MSG(forbidden, "forbidden"); + ASSERT_EQ(403, std::static_pointer_cast(forbidden.detail())->GetCode()); + + // 503 and the codes without an own mapping (e.g. 429) become IOError with a + // message naming the code + response.code = 503; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "service unavailable"); + response.code = 429; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 429"); + response.code = 418; + ASSERT_NOK_WITH_MSG(RestApi::ErrorToStatus(response), "rest request failed with code 418"); +} + +TEST(RestApiErrorTest, MalformedSuccessBodyFails) { + // a 200 response whose body is not the expected json must fail, not crash or + // return partial data + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = "not-a-json"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + Status list_status = api->ListDatabases().status(); + ASSERT_NOK(list_status); + // the body is not echoed into the error: a successful response may contain + // credentials + ASSERT_EQ(std::string::npos, list_status.ToString().find("not-a-json")) + << list_status.ToString(); + ASSERT_NOK(api->GetTable(Identifier("db1", "t1")).status()); + + Status config_status = RestApi::Create(options, "", /*config_required=*/true).status(); + ASSERT_NOK(config_status); + ASSERT_EQ(std::string::npos, config_status.ToString().find("not-a-json")) + << config_status.ToString(); +} + +TEST(RestApiErrorTest, PagedListingStopsOnEmptyPageWithToken) { + // a server that keeps returning a page token with no data must not loop forever + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"databases":[],"nextPageToken":"more"})"; + return response; + })); + std::map options = { + {CatalogOptions::URI, server->GetBaseUri()}, + {CatalogOptions::TOKEN_PROVIDER, "bear"}, + {CatalogOptions::TOKEN, kToken}, + }; + ASSERT_OK_AND_ASSIGN(std::unique_ptr api, + RestApi::Create(options, "", /*config_required=*/false)); + ASSERT_OK_AND_ASSIGN(std::vector databases, api->ListDatabases()); + ASSERT_TRUE(databases.empty()); + ASSERT_EQ(1, request_count.load()); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_http_client.cpp b/src/paimon/rest/rest_http_client.cpp new file mode 100644 index 00000000..b17b0a28 --- /dev/null +++ b/src/paimon/rest/rest_http_client.cpp @@ -0,0 +1,462 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "curl/curl.h" +#include "fmt/format.h" +#include "paimon/common/utils/http_client.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/common/utils/url_utils.h" +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { + +size_t WriteBodyCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* body = static_cast(user_data); + body->append(data, size * nmemb); + return size * nmemb; +} + +size_t WriteHeaderCallback(char* data, size_t size, size_t nmemb, void* user_data) { + auto* headers = static_cast(user_data); + size_t total = size * nmemb; + // A status line starts the header block of the next response on the connection + // (e.g. after a followed redirect); dropping the previous headers leaves only the + // final response's. + if (total >= 5 && std::strncmp(data, "HTTP/", 5) == 0) { + headers->clear(); + } + ParseHttpHeaderLine(data, total, headers); + return total; +} + +// Only the methods `Execute` supports are classified; POST is the sole non-idempotent +// one among them. +bool IsIdempotent(const std::string& method) { + return method == "GET" || method == "DELETE"; +} + +bool IsRetriableCode(int64_t code) { + return code == HttpStatus::kTooManyRequests || code == HttpStatus::kServiceUnavailable; +} + +// Parses the RFC 1123 form of an http date, e.g. "Wed, 21 Oct 2015 07:28:00 GMT", into +// unix epoch seconds. Month names are matched against a fixed English table so the +// result does not depend on the process locale. +std::optional ParseHttpDateSeconds(const std::string& value) { + std::vector parts = StringUtils::Split(value, " ", /*ignore_empty=*/true); + if (parts.size() != 6 || parts[5] != "GMT" || parts[0].size() != 4 || parts[0][3] != ',') { + return std::nullopt; + } + static constexpr const char* kMonths[] = {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + int64_t month = 0; + for (size_t i = 0; i < 12; i++) { + if (parts[2] == kMonths[i]) { + month = static_cast(i) + 1; + break; + } + } + std::optional day = StringUtils::StringToValue(parts[1]); + std::optional year = StringUtils::StringToValue(parts[3]); + std::vector time_parts = StringUtils::Split(parts[4], ":", + /*ignore_empty=*/false); + if (month == 0 || !day || !year || time_parts.size() != 3) { + return std::nullopt; + } + std::optional hour = StringUtils::StringToValue(time_parts[0]); + std::optional minute = StringUtils::StringToValue(time_parts[1]); + std::optional second = StringUtils::StringToValue(time_parts[2]); + if (!hour || !minute || !second || day.value() < 1 || day.value() > 31 || year.value() < 1970 || + year.value() > 9999 || hour.value() > 23 || minute.value() > 59 || second.value() > 60) { + return std::nullopt; + } + // Days between 1970-01-01 and the given civil date ("days from civil" algorithm). + int64_t y = year.value() - (month <= 2 ? 1 : 0); + int64_t era = y / 400; + int64_t yoe = y - era * 400; + int64_t doy = (153 * (month + (month > 2 ? -3 : 9)) + 2) / 5 + day.value() - 1; + int64_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + int64_t days = era * 146097 + doe - 719468; + return days * 86400 + hour.value() * 3600 + minute.value() * 60 + second.value(); +} + +// Allowlist of the transient transport failures: an established connection breaking +// mid-request (send/receive failures, HTTP/2 stream errors) or a response truncated +// mid-body. The request reached a live server, so retrying an idempotent method is +// likely to succeed. Every other CURLcode (resolution/connect failures, timeouts, TLS +// failures, malformed urls, redirect loops, ...) is permanent and costs one attempt. +bool IsRetriableTransportError(CURLcode code) { + switch (code) { + case CURLE_SEND_ERROR: + case CURLE_RECV_ERROR: + case CURLE_PARTIAL_FILE: + case CURLE_HTTP2: + case CURLE_HTTP2_STREAM: + return true; + default: + return false; + } +} + +// libcurl sends no User-Agent of its own, so requests would reach the server unnamed. A +// "header.User-Agent" option still wins: an explicit request header overrides +// CURLOPT_USERAGENT. +constexpr const char kDefaultUserAgent[] = "paimon-cpp"; + +// A valid HTTP header field name per RFC 7230: one or more tchar. +bool IsValidHeaderName(const std::string& name) { + if (name.empty()) { + return false; + } + static constexpr char kTcharSymbols[] = "!#$%&'*+-.^_`|~"; + for (char c : name) { + if (c != '\0' && (std::isalnum(static_cast(c)) || + std::strchr(kTcharSymbols, c) != nullptr)) { + continue; + } + return false; + } + return true; +} + +// libcurl sends header strings verbatim, so a value containing CR, LF or NUL could +// inject additional header lines into the request. +bool IsValidHeaderValue(const std::string& value) { + return value.find_first_of("\r\n") == std::string::npos && + value.find('\0') == std::string::npos; +} + +} // namespace + +// A libcurl easy handle owns the connection cache of the connections it opened, so +// returning it to the pool after a request keeps the connection alive for the next one: +// re-creating a handle per request would pay a TCP and TLS handshake every time. A handle +// must not be used by two threads at once, hence handing it out exclusively. +class RestHttpClient::HandlePool { + public: + HandlePool() : curl_guard_(EnsureCurlGlobalInit()) {} + + ~HandlePool() { + for (CURL* handle : handles_) { + curl_easy_cleanup(handle); + } + } + + CURL* Acquire() const { + std::scoped_lock lock(mutex_); + if (handles_.empty()) { + return curl_easy_init(); + } + CURL* handle = handles_.back(); + handles_.pop_back(); + return handle; + } + + void Release(CURL* handle) const { + // Resetting drops the options of the finished request while keeping the + // connection cache, so the next request starts from a clean handle. + curl_easy_reset(handle); + std::scoped_lock lock(mutex_); + handles_.push_back(handle); + } + + private: + std::shared_ptr curl_guard_; + mutable std::mutex mutex_; + mutable std::vector handles_; +}; + +RestHttpClient::RestHttpClient(const std::string& base_uri, const Config& config) + : handle_pool_(std::make_unique()), + base_uri_(base_uri), + config_(config), + logger_(Logger::GetLogger("RestHttpClient")) {} + +RestHttpClient::~RestHttpClient() = default; + +Result> RestHttpClient::Create(const std::string& base_uri) { + return Create(base_uri, Config()); +} + +Result> RestHttpClient::Create(const std::string& base_uri, + const Config& config) { + if (base_uri.empty()) { + return Status::Invalid("uri of the http client is empty"); + } + return std::unique_ptr(new RestHttpClient(NormalizeUri(base_uri), config)); +} + +std::string RestHttpClient::NormalizeUri(const std::string& uri) { + std::string normalized = uri; + StringUtils::Trim(&normalized); + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (normalized.rfind("http://", 0) != 0 && normalized.rfind("https://", 0) != 0) { + normalized = "http://" + normalized; + } + return normalized; +} + +std::string RestHttpClient::BuildQueryString( + const std::map& query_params) { + std::string query; + for (const auto& [key, value] : query_params) { + if (!query.empty()) { + query.push_back('&'); + } + query.append(UrlUtils::EncodeString(key)); + query.push_back('='); + query.append(UrlUtils::EncodeString(value)); + } + return query; +} + +Result RestHttpClient::ExecuteOnce( + const std::string& method, const std::string& url, + const std::map& headers, const std::string& body, + bool* transport_retriable) const { + CURL* curl = handle_pool_->Acquire(); + if (curl == nullptr) { + return Status::IOError("failed to create curl handle"); + } + ScopeGuard release_curl([this, curl] { handle_pool_->Release(curl); }); + Response response; + curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, kDefaultUserAgent); + // TLS 1.0/1.1 are not accepted, as in the Java client. The curl constants of the + // options below are `int`, while curl reads a `long` from the variadic argument. + curl_easy_setopt(curl, CURLOPT_SSLVERSION, + static_cast(CURL_SSLVERSION_TLSv1_2)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, + static_cast(config_.connect_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, + static_cast(config_.request_timeout_ms)); // NOLINT(runtime/int) + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, WriteBodyCallback); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &response.body); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, WriteHeaderCallback); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &response.headers); + // Follow redirects transparently, restricted to http(s) targets. Without + // CURLOPT_POSTREDIR a 301/302 would replay a body-carrying request as a bodyless + // GET. A 303 is left to become a GET, which is what it is defined to mean. + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L); + curl_easy_setopt(curl, CURLOPT_POSTREDIR, + static_cast(CURL_REDIR_POST_301 | // NOLINT(runtime/int) + CURL_REDIR_POST_302)); +#if CURL_AT_LEAST_VERSION(7, 85, 0) + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS_STR, "http,https"); +#else + curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, + static_cast(CURLPROTO_HTTP | CURLPROTO_HTTPS)); // NOLINT(runtime/int) +#endif + + if (method == "POST") { + curl_easy_setopt(curl, CURLOPT_POST, 1L); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } else if (method == "DELETE") { + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + if (!body.empty()) { + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, body.c_str()); + curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, + static_cast(body.size())); // NOLINT(runtime/int) + } + } + + struct curl_slist* header_list = nullptr; + ScopeGuard free_headers([&header_list] { curl_slist_free_all(header_list); }); + auto append_header = [&header_list](const std::string& line) -> bool { + struct curl_slist* updated_list = curl_slist_append(header_list, line.c_str()); + if (updated_list == nullptr) { + return false; + } + header_list = updated_list; + return true; + }; + // An empty "Expect:" header disables libcurl's automatic "Expect: 100-continue". + bool headers_ok = append_header("Expect:"); + for (const auto& [name, value] : headers) { + headers_ok = headers_ok && append_header(name + ": " + value); + } + if (!headers_ok) { + return Status::IOError("failed to create http headers"); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, header_list); + + CURLcode curl_code = curl_easy_perform(curl); + if (curl_code != CURLE_OK) { + *transport_retriable = IsRetriableTransportError(curl_code); + // Neither the url nor the response body is echoed in errors: both may carry + // credentials. + return Status::IOError( + fmt::format("http {} request failed: {}", method, curl_easy_strerror(curl_code))); + } + // curl writes a `long` through the pointer, so the type cannot be narrowed here. + // NOLINTNEXTLINE(google-runtime-int) + long http_code = 0; // NOLINT(runtime/int) + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code); + response.code = http_code; + return response; +} + +std::optional RestHttpClient::ComputeRetryDelayMs(const Config& config, + int32_t execution_count, + const Response* response, + int64_t now_epoch_seconds, + int64_t remaining_budget_ms) { + if (remaining_budget_ms <= 0) { + return std::nullopt; + } + if (response != nullptr) { + auto iter = response->headers.find("retry-after"); + if (iter != response->headers.end()) { + std::optional retry_after_ms; + std::optional retry_after_seconds = + StringUtils::StringToValue(iter->second); + if (retry_after_seconds) { + // Clamped so that a misbehaving server cannot overflow the conversion. + constexpr int64_t kMaxSeconds = std::numeric_limits::max() / 1000; + retry_after_ms = + std::clamp(retry_after_seconds.value(), -kMaxSeconds, kMaxSeconds) * 1000; + } else { + std::optional date_seconds = ParseHttpDateSeconds(iter->second); + if (date_seconds) { + retry_after_ms = (date_seconds.value() - now_epoch_seconds) * 1000; + } + } + // Non-positive values fall through to the exponential backoff. A delay + // beyond the remaining budget stops retrying entirely: sleeping less than + // requested would just hit the throttle again. `retry_max_delay_ms` is + // deliberately not applied here, as in the Java client, so a longer + // `Retry-After` is honored whenever it fits the budget. + if (retry_after_ms && retry_after_ms.value() > 0) { + if (retry_after_ms.value() > remaining_budget_ms) { + return std::nullopt; + } + return retry_after_ms.value(); + } + } + } + int64_t multiplier = static_cast(1) << std::clamp(execution_count - 1, 0, 6); + int64_t delay_ms = config.retry_base_delay_ms * multiplier; + if (delay_ms > 0) { + static thread_local std::mt19937 generator( + std::random_device{}()); // NOLINT(whitespace/braces) + std::uniform_int_distribution jitter(0, delay_ms / 10); + delay_ms += jitter(generator); + } + delay_ms = std::min(delay_ms, config.retry_max_delay_ms); + if (delay_ms > remaining_budget_ms) { + return std::nullopt; + } + return delay_ms; +} + +std::optional RestHttpClient::GetRetryDelayMs(int32_t execution_count, + const Response* response, + int64_t remaining_budget_ms) const { + return ComputeRetryDelayMs(config_, execution_count, response, + static_cast(std::time(nullptr)), remaining_budget_ms); +} + +Result RestHttpClient::Execute( + const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, const std::string& body) const { + if (method != "GET" && method != "POST" && method != "DELETE") { + return Status::Invalid(fmt::format("unsupported http method: {}", method)); + } + for (const auto& [name, value] : headers) { + // Neither the name nor the value is echoed in errors: the name may contain + // the very control bytes this check rejects, the value may carry credentials. + if (!IsValidHeaderName(name)) { + return Status::Invalid("invalid http header name"); + } + if (!IsValidHeaderValue(value)) { + return Status::Invalid(fmt::format("invalid http header value for '{}'", name)); + } + } + std::string url = base_uri_ + path; + if (!query_params.empty()) { + url += "?" + BuildQueryString(query_params); + } + bool idempotent = IsIdempotent(method); + std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now(); + int32_t execution_count = 0; + while (true) { + execution_count++; + bool transport_retriable = false; + Result result = ExecuteOnce(method, url, headers, body, &transport_retriable); + bool retriable; + if (result.ok()) { + retriable = IsRetriableCode(result.value().code); + } else { + retriable = idempotent && transport_retriable; + } + int64_t elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + std::optional delay_ms; + if (retriable && execution_count <= config_.max_retries) { + delay_ms = GetRetryDelayMs(execution_count, result.ok() ? &result.value() : nullptr, + config_.retry_timeout_ms - elapsed_ms); + } + if (!delay_ms) { + if (result.ok()) { + const std::string request_id = RestUtil::ExtractRequestId(result.value().headers); + PAIMON_LOG_DEBUG( + logger_, "[rest] requestId:%s method:%s path:%s status:%lld duration:%lldms", + request_id.c_str(), method.c_str(), path.c_str(), + static_cast(result.value().code), // NOLINT(runtime/int) + static_cast(elapsed_ms)); // NOLINT(runtime/int) + } + return result; + } + std::string reason = result.ok() ? fmt::format("http status {}", result.value().code) + : result.status().ToString(); + PAIMON_LOG_WARN(logger_, "[rest] retrying method:%s path:%s in %lld ms (retry %d/%d): %s", + method.c_str(), path.c_str(), + static_cast(delay_ms.value()), // NOLINT(runtime/int) + execution_count, config_.max_retries, reason.c_str()); + if (delay_ms.value() > 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms.value())); + } + } +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client.h b/src/paimon/rest/rest_http_client.h new file mode 100644 index 00000000..07712795 --- /dev/null +++ b/src/paimon/rest/rest_http_client.h @@ -0,0 +1,149 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/logging.h" +#include "paimon/result.h" +#include "paimon/status.h" + +namespace paimon { + +/// HTTP status codes used by the REST catalog. +struct HttpStatus { + static constexpr int64_t kOk = 200; + static constexpr int64_t kAccepted = 202; + static constexpr int64_t kNoContent = 204; + static constexpr int64_t kBadRequest = 400; + static constexpr int64_t kUnauthorized = 401; + static constexpr int64_t kForbidden = 403; + static constexpr int64_t kNotFound = 404; + static constexpr int64_t kConflict = 409; + static constexpr int64_t kTooManyRequests = 429; + static constexpr int64_t kInternalServerError = 500; + static constexpr int64_t kNotImplemented = 501; + static constexpr int64_t kServiceUnavailable = 503; +}; + +/// A blocking HTTP client for the REST catalog based on libcurl. HTTP 429/503 +/// responses are retried for all methods, transient transport errors only for +/// idempotent ones, with exponential backoff honoring a positive `Retry-After` +/// response header (delta-seconds or HTTP-date form). A backoff sleep is bounded by +/// `retry_max_delay_ms` and the whole request by `retry_timeout_ms`; a `Retry-After` +/// beyond the remaining budget stops retrying rather than shortening the sleep. +/// Redirects to http(s) targets are followed transparently, keeping the method and +/// body of POST/DELETE requests. +class RestHttpClient { + public: + struct Config { + int64_t connect_timeout_ms = 180 * 1000; + int64_t request_timeout_ms = 180 * 1000; + int32_t max_retries = 5; + int64_t retry_base_delay_ms = 1000; + /// Upper bound for a single backoff sleep; a longer `Retry-After` is still + /// honored, bounded only by `retry_timeout_ms`. + int64_t retry_max_delay_ms = 64 * 1000; + /// Overall budget for one `Execute` call, covering all attempts and retry + /// sleeps; a retry whose delay does not fit the remaining budget is not + /// attempted. + int64_t retry_timeout_ms = 5 * 60 * 1000; + }; + + struct Response { + int64_t code = 0; + std::string body; + /// Response headers with lower-cased names. + std::map headers; + + bool IsSuccessful() const { + return code == HttpStatus::kOk || code == HttpStatus::kAccepted || + code == HttpStatus::kNoContent; + } + }; + + /// Creates a client against `base_uri`. The uri is normalized: surrounding + /// whitespace and all trailing '/' are stripped and "http://" is prepended when no + /// scheme is present. + static Result> Create(const std::string& base_uri); + static Result> Create(const std::string& base_uri, + const Config& config); + + ~RestHttpClient(); + + /// Executes `method` ("GET", "POST" or "DELETE") on `path` (already url-encoded, + /// starting with '/'); query parameter keys and values are url-encoded internally. + /// Header names must be valid HTTP tokens and values must not contain CR, LF or + /// NUL; a violating header fails the request before anything is sent. Returns the + /// final response, which may carry a non-2xx code, or an error status when the + /// request could not be transported at all. Only transient transport errors (an + /// established connection breaking mid-request or a truncated response body) are + /// retried; every other transport failure fails immediately. + Result Execute(const std::string& method, const std::string& path, + const std::map& query_params, + const std::map& headers, + const std::string& body) const; + + const std::string& GetBaseUri() const { + return base_uri_; + } + + static std::string NormalizeUri(const std::string& uri); + + /// Builds "k1=v1&k2=v2" with url-encoded keys and values. + static std::string BuildQueryString(const std::map& query_params); + + /// Computes the delay before the retry following the `execution_count`-th attempt + /// (counted from 1), in ms, or `nullopt` when retrying should stop. A positive + /// `Retry-After` header of `response` (delta-seconds, or HTTP-date resolved against + /// `now_epoch_seconds`) wins and is never shortened: beyond `remaining_budget_ms` + /// it yields `nullopt`. Otherwise exponential backoff with up to 10% jitter + /// applies, clamped to `retry_max_delay_ms`, yielding `nullopt` when even the + /// clamped delay exceeds `remaining_budget_ms`. `response` is null when the attempt + /// failed with a transport error. + static std::optional ComputeRetryDelayMs(const Config& config, int32_t execution_count, + const Response* response, + int64_t now_epoch_seconds, + int64_t remaining_budget_ms); + + private: + RestHttpClient(const std::string& base_uri, const Config& config); + + /// Pool of the libcurl easy handles used by `ExecuteOnce`, also keeping libcurl's + /// global state alive for the lifetime of the client. + class HandlePool; + + /// On a transport failure, `transport_retriable` reports whether the failure kind + /// may be retried; see `Execute` for which kinds are not. + Result ExecuteOnce(const std::string& method, const std::string& url, + const std::map& headers, + const std::string& body, bool* transport_retriable) const; + + std::optional GetRetryDelayMs(int32_t execution_count, const Response* response, + int64_t remaining_budget_ms) const; + + std::unique_ptr handle_pool_; + std::string base_uri_; + Config config_; + std::shared_ptr logger_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_http_client_test.cpp b/src/paimon/rest/rest_http_client_test.cpp new file mode 100644 index 00000000..34d07fe6 --- /dev/null +++ b/src/paimon/rest/rest_http_client_test.cpp @@ -0,0 +1,565 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_http_client.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/rest/mock_rest_server.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { + +namespace { +RestHttpClient::Config FastRetryConfig(int32_t max_retries) { + RestHttpClient::Config config; + config.max_retries = max_retries; + config.retry_base_delay_ms = 1; + return config; +} +} // namespace + +TEST(RestHttpClientTest, NormalizeUri) { + ASSERT_EQ("http://localhost:80", RestHttpClient::NormalizeUri("localhost:80/")); + ASSERT_EQ("http://localhost", RestHttpClient::NormalizeUri("http://localhost//")); + ASSERT_EQ("https://foo.bar", RestHttpClient::NormalizeUri(" https://foo.bar ")); +} + +TEST(RestHttpClientTest, BuildQueryString) { + ASSERT_EQ("a=1&b=x+y%2Fz", RestHttpClient::BuildQueryString({{"a", "1"}, {"b", "x y/z"}})); +} + +TEST(RestHttpClientTest, GetWithHeadersAndQuery) { + // the handler runs on the mock server's accept thread, so every state it shares with + // the test body is mutex guarded, here and in the tests below + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/config", {{"warehouse", "wh 1"}}, + {{"Authorization", "Bearer token1"}}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ("application/json", response.headers.at("content-type")); + std::lock_guard lock(mutex); + ASSERT_EQ("GET", last_request.method); + ASSERT_EQ("/v1/config", last_request.path); + ASSERT_EQ("wh 1", last_request.query_params.at("warehouse")); + ASSERT_EQ("Bearer token1", last_request.headers.at("authorization")); +} + +TEST(RestHttpClientTest, PostBody) { + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {{"Content-Type", "application/json"}}, + R"({"name": "db1"})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("POST", last_request.method); + ASSERT_EQ(R"({"name": "db1"})", last_request.body); + ASSERT_EQ("application/json", last_request.headers.at("content-type")); +} + +TEST(RestHttpClientTest, NotFoundIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 404; + response.body = R"({"message": "not found", "code": 404})"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(404, response.code); + ASSERT_FALSE(response.IsSuccessful()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ServiceUnavailableIsRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ < 2) { + response.code = 503; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + // 429/503 responses are retried even for non-idempotent POST. + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("POST", "/v1/databases", {}, {}, "{}")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, RetryAfterHeaderIsHonored) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 429; + response.headers["Retry-After"] = "1"; + } + return response; + })); + // the backoff base is large so that the elapsed time below proves the Retry-After + // header took precedence: without it this test would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_GE(elapsed, std::chrono::seconds(1)); + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, ComputeRetryDelayMs) { + RestHttpClient::Config config; + config.retry_base_delay_ms = 1000; + constexpr int64_t kBudgetMs = 1000 * 1000; + // without a usable Retry-After header: exponential backoff with up to 10% jitter, + // the multiplier capped at 2^6 + std::optional delay = + RestHttpClient::ComputeRetryDelayMs(config, 1, nullptr, 0, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 1000); + ASSERT_LE(delay.value(), 1100); + delay = RestHttpClient::ComputeRetryDelayMs(config, 3, nullptr, 0, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 4000); + ASSERT_LE(delay.value(), 4400); + // the capped multiplier lands exactly on the default per-retry bound, so the + // jitter is clamped away + delay = RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0, kBudgetMs); + ASSERT_EQ(64000, delay); + // an out-of-contract execution count degrades to the first-attempt delay instead + // of shifting by a negative amount + delay = RestHttpClient::ComputeRetryDelayMs(config, 0, nullptr, 0, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 1000); + ASSERT_LE(delay.value(), 1100); + + // the delta-seconds form takes precedence over the backoff, without jitter + RestHttpClient::Response response; + response.headers["retry-after"] = "7"; + ASSERT_EQ(7000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, kBudgetMs)); + + // the HTTP-date form is resolved against the given "now" + constexpr int64_t kDateEpoch = 1445412480; // Wed, 21 Oct 2015 07:28:00 GMT + response.headers["retry-after"] = "Wed, 21 Oct 2015 07:28:00 GMT"; + ASSERT_EQ(5000, + RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch - 5, kBudgetMs)); + + // a date in the past falls back to the backoff + delay = RestHttpClient::ComputeRetryDelayMs(config, 1, &response, kDateEpoch + 1, kBudgetMs); + ASSERT_TRUE(delay); + ASSERT_GE(delay.value(), 1000); + ASSERT_LE(delay.value(), 1100); +} + +TEST(RestHttpClientTest, RetryDelayBounds) { + RestHttpClient::Config config; + config.retry_base_delay_ms = 1000; + config.retry_max_delay_ms = 10 * 1000; + RestHttpClient::Response response; + + // the per-retry bound applies to the backoff only: a Retry-After beyond it is + // still honored as long as it fits the remaining budget + response.headers["retry-after"] = "11"; + ASSERT_EQ(11000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 1000 * 1000)); + // a Retry-After beyond the remaining budget stops retrying instead of sleeping + // less than the server requested + response.headers["retry-after"] = "9"; + ASSERT_FALSE(RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 8000)); + ASSERT_EQ(9000, RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 9000)); + + // the backoff is clamped to the per-retry bound... + ASSERT_EQ(10000, RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0, 1000 * 1000)); + // ...and stops retrying when even the clamped delay does not fit the budget + ASSERT_FALSE(RestHttpClient::ComputeRetryDelayMs(config, 100, nullptr, 0, 9999)); + // an exhausted budget stops retrying before any header is consulted + ASSERT_FALSE(RestHttpClient::ComputeRetryDelayMs(config, 1, &response, 0, 0)); +} + +TEST(RestHttpClientTest, ExcessiveRetryAfterStopsRetrying) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 503; + // an hour is far beyond the default overall retry budget + response.headers["Retry-After"] = "3600"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + // the 503 is returned as-is after a single attempt instead of five shortened + // sleeps the server did not ask for + ASSERT_EQ(503, response.code); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, InvalidRetryAfterFallsBackToBackoff) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + int32_t count = request_count++; + if (count == 0) { + response.code = 429; + // non-positive values are ignored + response.headers["Retry-After"] = "0"; + } else if (count == 2) { + response.code = 429; + // neither delta-seconds nor a valid http date + response.headers["Retry-After"] = "Sat, 32 Foo 2015 99:99:99 GMT"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); + ASSERT_OK_AND_ASSIGN(response, client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(4, request_count.load()); +} + +TEST(RestHttpClientTest, RetriesExhausted) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 503; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(2))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(503, response.code); + // initial attempt + 2 retries + ASSERT_EQ(3, request_count.load()); +} + +TEST(RestHttpClientTest, EmptyReplyIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + // closing without a response maps to CURLE_GOT_NOTHING, a + // non-retriable transport error that, unlike timeouts or + // TLS failures, is deterministic in a unit test + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, ConnectionRefusedIsNotRetried) { + // a closed port yields CURLE_COULDNT_CONNECT, another of the non-retriable + // transport errors; the port of a stopped mock server is known to be closed + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + return MockRestServer::Response(); + })); + std::string base_uri = server->GetBaseUri(); + server->Stop(); + // the backoff base is large so that the elapsed time below proves the failure was + // not retried: a single retry would sleep for a minute + RestHttpClient::Config config; + config.max_retries = 5; + config.retry_base_delay_ms = 60 * 1000; + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(base_uri, config)); + auto start = std::chrono::steady_clock::now(); + ASSERT_NOK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + auto elapsed = std::chrono::steady_clock::now() - start; + ASSERT_LT(elapsed, std::chrono::seconds(30)); +} + +TEST(RestHttpClientTest, DeleteWithBodyIsSent) { + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + last_request = request; + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + // a body on DELETE keeps the DELETE method line instead of degrading to POST + ASSERT_OK_AND_ASSIGN( + RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {{"Content-Type", "application/json"}}, + R"({"purge": true})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("DELETE", last_request.method); + ASSERT_EQ(R"({"purge": true})", last_request.body); +} + +TEST(RestHttpClientTest, ReusedHandleDoesNotCarryRequestState) { + // the client pools its curl handles to keep connections alive, so a request must not + // inherit the method or the body of the request that used the handle before it + std::mutex mutex; + std::vector requests; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + std::lock_guard lock(mutex); + requests.push_back(request); + return MockRestServer::Response(); + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK(client->Execute("POST", "/v1/databases", {}, {}, R"({"name": "db1"})").status()); + ASSERT_OK( + client->Execute("DELETE", "/v1/databases/db1", {}, {}, R"({"purge": true})").status()); + ASSERT_OK(client->Execute("GET", "/v1/databases", {}, {}, "").status()); + std::lock_guard lock(mutex); + ASSERT_EQ(3, requests.size()); + ASSERT_EQ("GET", requests[2].method); + ASSERT_TRUE(requests[2].body.empty()) << requests[2].body; +} + +TEST(RestHttpClientTest, TransportErrorOmitsUrlAndQuery) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.close_without_response = true; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + Status status = + client->Execute("GET", "/v1/secret-path", {{"sig", "topsecret1"}}, {}, "").status(); + ASSERT_NOK(status); + // the error must not echo the path or query values, which may carry credentials + // (e.g. a presigned url) + ASSERT_EQ(std::string::npos, status.ToString().find("secret-path")) << status.ToString(); + ASSERT_EQ(std::string::npos, status.ToString().find("topsecret1")) << status.ToString(); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForGet) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + response.body = R"({"ok": true})"; + if (request_count++ == 0) { + // cutting the response off mid-body is a retriable + // transport error, and GET is idempotent + response.missing_body_bytes = 5; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/v1/databases", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsRetriedForDelete) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + // DELETE is the other idempotent method (see the GET + // test above) + response.missing_body_bytes = 5; + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("DELETE", "/v1/databases/db1", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(2, request_count.load()); +} + +TEST(RestHttpClientTest, TruncatedResponseIsNotRetriedForPost) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.body = R"({"ok": true})"; + // the same retriable transport error kind as in the GET + // test above, but POST is not idempotent + response.missing_body_bytes = 5; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("POST", "/v1/databases", {}, {}, "{}").status()); + ASSERT_EQ(1, request_count.load()); +} + +TEST(RestHttpClientTest, RedirectIsFollowed) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request_count++ == 0) { + response.code = 302; + response.headers["Location"] = "/v1/config"; + } else { + response.body = R"({"ok": true})"; + } + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("GET", "/old", {}, {}, "")); + ASSERT_EQ(200, response.code); + ASSERT_EQ(R"({"ok": true})", response.body); + ASSERT_EQ(2, request_count.load()); + // only the final response's headers are reported, not the redirect's + ASSERT_EQ(0, response.headers.count("location")); +} + +TEST(RestHttpClientTest, PostRedirectKeepsMethodAndBody) { + std::mutex mutex; + MockRestServer::Request last_request; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + MockRestServer::Response response; + if (request.path == "/old") { + // 302 is one of the codes that would degrade a POST + // to a bodyless GET without CURLOPT_POSTREDIR + response.code = 302; + response.headers["Location"] = "/v1/databases"; + return response; + } + std::lock_guard lock(mutex); + last_request = request; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri())); + ASSERT_OK_AND_ASSIGN(RestHttpClient::Response response, + client->Execute("POST", "/old", {}, {}, R"({"name": "db1"})")); + ASSERT_EQ(200, response.code); + std::lock_guard lock(mutex); + ASSERT_EQ("/v1/databases", last_request.path); + ASSERT_EQ("POST", last_request.method); + ASSERT_EQ(R"({"name": "db1"})", last_request.body); +} + +TEST(RestHttpClientTest, RedirectLoopIsNotRetried) { + std::atomic request_count{0}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr server, + MockRestServer::Start([&](const MockRestServer::Request& request) { + request_count++; + MockRestServer::Response response; + response.code = 302; + response.headers["Location"] = "/loop"; + return response; + })); + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create(server->GetBaseUri(), FastRetryConfig(5))); + ASSERT_NOK(client->Execute("GET", "/loop", {}, {}, "").status()); + // one attempt follows at most CURLOPT_MAXREDIRS (50) redirects and the resulting + // CURLE_TOO_MANY_REDIRECTS is a permanent transport error: retrying it would + // multiply the request count by the retry schedule + ASSERT_LE(request_count.load(), 51); +} + +TEST(RestHttpClientTest, InvalidHeadersAreRejected) { + // headers are validated before anything is sent, so no server is needed + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create("http://127.0.0.1:1")); + ASSERT_NOK_WITH_MSG(client->Execute("GET", "/", {}, {{"Bad Name", "v"}}, "").status(), + "invalid http header name"); + ASSERT_NOK_WITH_MSG( + client->Execute("GET", "/", {}, {{"Evil\r\nInjected: x", "v"}}, "").status(), + "invalid http header name"); + ASSERT_NOK_WITH_MSG(client->Execute("GET", "/", {}, {{"", "v"}}, "").status(), + "invalid http header name"); + ASSERT_NOK_WITH_MSG( + client->Execute("GET", "/", {}, {{"X-Ok", "a\r\nInjected: b"}}, "").status(), + "invalid http header value for 'X-Ok'"); + ASSERT_NOK_WITH_MSG( + client->Execute("GET", "/", {}, {{"X-Ok", std::string("a\0b", 3)}}, "").status(), + "invalid http header value for 'X-Ok'"); +} + +TEST(RestHttpClientTest, UnsupportedMethod) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr client, + RestHttpClient::Create("http://127.0.0.1:1")); + ASSERT_NOK_WITH_MSG(client->Execute("PATCH", "/", {}, {}, "").status(), + "unsupported http method"); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_messages.cpp b/src/paimon/rest/rest_messages.cpp new file mode 100644 index 00000000..324a8408 --- /dev/null +++ b/src/paimon/rest/rest_messages.cpp @@ -0,0 +1,386 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include + +#include "paimon/rest/rest_util.h" + +namespace paimon { + +namespace { + +constexpr const char kFieldMessage[] = "message"; +constexpr const char kFieldResourceType[] = "resourceType"; +constexpr const char kFieldResourceName[] = "resourceName"; +constexpr const char kFieldCode[] = "code"; +constexpr const char kFieldDefaults[] = "defaults"; +constexpr const char kFieldOverrides[] = "overrides"; +constexpr const char kFieldOwner[] = "owner"; +constexpr const char kFieldCreatedAt[] = "createdAt"; +constexpr const char kFieldCreatedBy[] = "createdBy"; +constexpr const char kFieldUpdatedAt[] = "updatedAt"; +constexpr const char kFieldUpdatedBy[] = "updatedBy"; +constexpr const char kFieldName[] = "name"; +constexpr const char kFieldOptions[] = "options"; +constexpr const char kFieldId[] = "id"; +constexpr const char kFieldLocation[] = "location"; +constexpr const char kFieldDatabases[] = "databases"; +constexpr const char kFieldTables[] = "tables"; +constexpr const char kFieldSnapshots[] = "snapshots"; +constexpr const char kFieldNextPageToken[] = "nextPageToken"; +constexpr const char kFieldPath[] = "path"; +constexpr const char kFieldIsExternal[] = "isExternal"; +constexpr const char kFieldSchemaId[] = "schemaId"; +constexpr const char kFieldSchema[] = "schema"; +constexpr const char kFieldIdentifier[] = "identifier"; +constexpr const char kFieldDatabase[] = "database"; +constexpr const char kFieldObject[] = "object"; +constexpr const char kFieldSource[] = "source"; +constexpr const char kFieldDestination[] = "destination"; + +void AddOptionalStringMember(rapidjson::Value* obj, const char* key, + const std::optional& value, + rapidjson::Document::AllocatorType* allocator) { + if (value) { + obj->AddMember(rapidjson::StringRef(key), + RapidJsonUtil::SerializeValue(value.value(), allocator).Move(), *allocator); + } +} + +rapidjson::Value SerializeIdentifier(const std::string& database, const std::string& table, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabase), + RapidJsonUtil::SerializeValue(database, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldObject), + RapidJsonUtil::SerializeValue(table, allocator).Move(), *allocator); + return obj; +} + +void DeserializeIdentifier(const rapidjson::Value& obj, const char* key, std::string* database, + std::string* table) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + const rapidjson::Value& identifier = obj[key]; + *database = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldDatabase); + *table = RapidJsonUtil::DeserializeKeyValue(identifier, kFieldObject); +} + +std::string DeserializeRawJsonMember(const rapidjson::Value& obj, const char* key) { + if (!obj.IsObject() || !obj.HasMember(key) || !obj[key].IsObject()) { + throw std::invalid_argument(std::string("member '") + key + + "' must exist and be an object"); + } + return RestUtil::JsonToString(obj[key]); +} + +} // namespace + +rapidjson::Value ErrorResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldMessage), + RapidJsonUtil::SerializeValue(message_, allocator).Move(), *allocator); + if (!resource_type_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceType), + RapidJsonUtil::SerializeValue(resource_type_, allocator).Move(), *allocator); + } + if (!resource_name_.empty()) { + obj.AddMember(rapidjson::StringRef(kFieldResourceName), + RapidJsonUtil::SerializeValue(resource_name_, allocator).Move(), *allocator); + } + obj.AddMember(rapidjson::StringRef(kFieldCode), + RapidJsonUtil::SerializeValue(code_, allocator).Move(), *allocator); + return obj; +} + +void ErrorResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + message_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldMessage, std::string()); + resource_type_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceType, std::string()); + resource_name_ = + RapidJsonUtil::DeserializeKeyValue(obj, kFieldResourceName, std::string()); + code_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldCode, 0); +} + +rapidjson::Value ConfigResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDefaults), + RapidJsonUtil::SerializeValue(defaults_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOverrides), + RapidJsonUtil::SerializeValue(overrides_, allocator).Move(), *allocator); + return obj; +} + +namespace { +// The server may send null values in `defaults`/`overrides`; a null carries no value to +// merge, so it is collected in `null_keys` (when given) instead of failing the parse. +std::map DeserializeStringMap(const rapidjson::Value& obj, + const char* key, + std::set* null_keys) { + std::map result; + if (!obj.IsObject() || !obj.HasMember(key) || obj[key].IsNull()) { + return result; + } + const rapidjson::Value& map_value = obj[key]; + if (!map_value.IsObject()) { + throw std::invalid_argument(std::string("member '") + key + "' must be an object"); + } + for (auto iter = map_value.MemberBegin(); iter != map_value.MemberEnd(); ++iter) { + if (iter->value.IsNull()) { + if (null_keys != nullptr) { + null_keys->insert(iter->name.GetString()); + } + continue; + } + if (!iter->value.IsString()) { + throw std::invalid_argument(std::string("member '") + key + + "' must only contain string values"); + } + result[iter->name.GetString()] = iter->value.GetString(); + } + return result; +} +} // namespace + +void ConfigResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + removed_keys_.clear(); + // A null default carries no value and never unsets anything; only a null override + // removes a key. + defaults_ = DeserializeStringMap(obj, kFieldDefaults, nullptr); + overrides_ = DeserializeStringMap(obj, kFieldOverrides, &removed_keys_); +} + +std::map ConfigResponse::Merge( + const std::map& client_options) const { + std::map merged = defaults_; + for (const auto& [key, value] : client_options) { + merged[key] = value; + } + for (const auto& [key, value] : overrides_) { + merged[key] = value; + } + for (const std::string& key : removed_keys_) { + merged.erase(key); + } + return merged; +} + +void RestAuditFields::ParseFrom(const rapidjson::Value& obj) { + owner = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldOwner, + std::nullopt); + created_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldCreatedAt, + std::nullopt); + created_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldCreatedBy, std::nullopt); + updated_at = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldUpdatedAt, + std::nullopt); + updated_by = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldUpdatedBy, std::nullopt); +} + +void RestAuditFields::AddTo(rapidjson::Value* obj, + rapidjson::Document::AllocatorType* allocator) const { + AddOptionalStringMember(obj, kFieldOwner, owner, allocator); + if (created_at) { + obj->AddMember(rapidjson::StringRef(kFieldCreatedAt), + RapidJsonUtil::SerializeValue(created_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldCreatedBy, created_by, allocator); + if (updated_at) { + obj->AddMember(rapidjson::StringRef(kFieldUpdatedAt), + RapidJsonUtil::SerializeValue(updated_at.value(), allocator).Move(), + *allocator); + } + AddOptionalStringMember(obj, kFieldUpdatedBy, updated_by, allocator); +} + +void RestAuditFields::PutAuditOptionsTo(std::map* options) const { + if (owner) { + (*options)[kFieldOwner] = owner.value(); + } + if (created_at) { + (*options)[kFieldCreatedAt] = std::to_string(created_at.value()); + } + if (created_by) { + (*options)[kFieldCreatedBy] = created_by.value(); + } + if (updated_at) { + (*options)[kFieldUpdatedAt] = std::to_string(updated_at.value()); + } + if (updated_by) { + (*options)[kFieldUpdatedBy] = updated_by.value(); + } +} + +rapidjson::Value CreateDatabaseRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + return obj; +} + +void CreateDatabaseRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); +} + +rapidjson::Value GetDatabaseResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldLocation), + RapidJsonUtil::SerializeValue(location_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldOptions), + RapidJsonUtil::SerializeValue(options_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetDatabaseResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName); + location_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldLocation, std::string()); + options_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldOptions, {}); + audit_.ParseFrom(obj); +} + +rapidjson::Value ListDatabasesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldDatabases), + RapidJsonUtil::SerializeValue(databases_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListDatabasesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + databases_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldDatabases, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListTablesResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldTables), + RapidJsonUtil::SerializeValue(tables_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListTablesResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + tables_ = RapidJsonUtil::DeserializeKeyValue>(obj, kFieldTables, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value ListSnapshotsResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSnapshots), + RapidJsonUtil::SerializeValue(snapshots_, allocator).Move(), *allocator); + AddOptionalStringMember(&obj, kFieldNextPageToken, next_page_token_, allocator); + return obj; +} + +void ListSnapshotsResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + snapshots_ = + RapidJsonUtil::DeserializeKeyValue>(obj, kFieldSnapshots, {}); + next_page_token_ = RapidJsonUtil::DeserializeKeyValue>( + obj, kFieldNextPageToken, std::nullopt); +} + +rapidjson::Value GetTableResponse::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldId), + RapidJsonUtil::SerializeValue(id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldDatabase), + RapidJsonUtil::SerializeValue(database_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldName), + RapidJsonUtil::SerializeValue(name_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldPath), + RapidJsonUtil::SerializeValue(path_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldIsExternal), + RapidJsonUtil::SerializeValue(is_external_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchemaId), + RapidJsonUtil::SerializeValue(schema_id_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + audit_.AddTo(&obj, allocator); + return obj; +} + +void GetTableResponse::FromJson(const rapidjson::Value& obj) noexcept(false) { + id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldId, std::string()); + database_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldDatabase, std::string()); + name_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldName, std::string()); + path_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldPath); + is_external_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldIsExternal, false); + schema_id_ = RapidJsonUtil::DeserializeKeyValue(obj, kFieldSchemaId); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); + audit_.ParseFrom(obj); +} + +rapidjson::Value CreateTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldIdentifier), + SerializeIdentifier(database_, table_, allocator).Move(), *allocator); + obj.AddMember(rapidjson::StringRef(kFieldSchema), + RestUtil::ParseToValue(schema_json_, allocator).Move(), *allocator); + return obj; +} + +void CreateTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldIdentifier, &database_, &table_); + schema_json_ = DeserializeRawJsonMember(obj, kFieldSchema); +} + +rapidjson::Value RenameTableRequest::ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) { + rapidjson::Value obj(rapidjson::kObjectType); + obj.AddMember(rapidjson::StringRef(kFieldSource), + SerializeIdentifier(source_database_, source_table_, allocator).Move(), + *allocator); + obj.AddMember(rapidjson::StringRef(kFieldDestination), + SerializeIdentifier(destination_database_, destination_table_, allocator).Move(), + *allocator); + return obj; +} + +void RenameTableRequest::FromJson(const rapidjson::Value& obj) noexcept(false) { + DeserializeIdentifier(obj, kFieldSource, &source_database_, &source_table_); + DeserializeIdentifier(obj, kFieldDestination, &destination_database_, &destination_table_); +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages.h b/src/paimon/rest/rest_messages.h new file mode 100644 index 00000000..b943e6e1 --- /dev/null +++ b/src/paimon/rest/rest_messages.h @@ -0,0 +1,389 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "paimon/common/utils/jsonizable.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/core/snapshot.h" +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +// Request and response objects of the REST catalog protocol. The JSON field names must +// stay aligned with the REST catalog open api. + +class ErrorResponse : public Jsonizable { + public: + static constexpr const char* kResourceTypeDatabase = "DATABASE"; + static constexpr const char* kResourceTypeTable = "TABLE"; + + ErrorResponse(const std::string& resource_type, const std::string& resource_name, + const std::string& message, int32_t code) + : resource_type_(resource_type), + resource_name_(resource_name), + message_(message), + code_(code) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetResourceType() const { + return resource_type_; + } + const std::string& GetResourceName() const { + return resource_name_; + } + const std::string& GetMessage() const { + return message_; + } + int32_t GetCode() const { + return code_; + } + + ErrorResponse() = default; + + private: + std::string resource_type_; + std::string resource_name_; + std::string message_; + int32_t code_ = 0; +}; + +/// The response of "/v1/config". +class ConfigResponse : public Jsonizable { + public: + ConfigResponse(const std::map& defaults, + const std::map& overrides) + : defaults_(defaults), overrides_(overrides) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + /// Merges with the client options; the precedence is + /// `overrides` > `client_options` > `defaults`. A key sent as null in `overrides` is + /// removed from the result, which is how the server unsets a client option; a null in + /// `defaults` is ignored. + std::map Merge( + const std::map& client_options) const; + + const std::map& GetDefaults() const { + return defaults_; + } + const std::map& GetOverrides() const { + return overrides_; + } + + ConfigResponse() = default; + + private: + std::map defaults_; + std::map overrides_; + /// Keys the server sent as null in `overrides`; `Merge` removes them from the result + /// and `ToJson` does not re-emit them. + std::set removed_keys_; +}; + +/// The audit fields shared by database/table responses. +struct RestAuditFields { + std::optional owner; + std::optional created_at; + std::optional created_by; + std::optional updated_at; + std::optional updated_by; + + void ParseFrom(const rapidjson::Value& obj); + void AddTo(rapidjson::Value* obj, rapidjson::Document::AllocatorType* allocator) const; + /// Adds the present fields to `options`, keyed by their JSON field names. + void PutAuditOptionsTo(std::map* options) const; +}; + +class CreateDatabaseRequest : public Jsonizable { + public: + CreateDatabaseRequest(const std::string& name, + const std::map& options) + : name_(name), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetName() const { + return name_; + } + const std::map& GetOptions() const { + return options_; + } + + CreateDatabaseRequest() = default; + + private: + std::string name_; + std::map options_; +}; + +class GetDatabaseResponse : public Jsonizable { + public: + GetDatabaseResponse(const std::string& id, const std::string& name, const std::string& location, + const std::map& options) + : id_(id), name_(name), location_(location), options_(options) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetLocation() const { + return location_; + } + const std::map& GetOptions() const { + return options_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetDatabaseResponse() = default; + + private: + std::string id_; + std::string name_; + std::string location_; + std::map options_; + RestAuditFields audit_; +}; + +class ListDatabasesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListDatabasesResponse(const std::vector& databases, + const std::optional& next_page_token) + : databases_(databases), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return databases_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListDatabasesResponse() = default; + + private: + std::vector databases_; + std::optional next_page_token_; +}; + +class ListTablesResponse : public Jsonizable { + public: + using ItemType = std::string; + + ListTablesResponse(const std::vector& tables, + const std::optional& next_page_token) + : tables_(tables), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return tables_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListTablesResponse() = default; + + private: + std::vector tables_; + std::optional next_page_token_; +}; + +/// Each element is a snapshot in the same JSON layout as the snapshot files. +class ListSnapshotsResponse : public Jsonizable { + public: + using ItemType = Snapshot; + + ListSnapshotsResponse(const std::vector& snapshots, + const std::optional& next_page_token) + : snapshots_(snapshots), next_page_token_(next_page_token) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::vector& Data() const { + return snapshots_; + } + const std::optional& NextPageToken() const { + return next_page_token_; + } + + ListSnapshotsResponse() = default; + + private: + std::vector snapshots_; + std::optional next_page_token_; +}; + +/// The nested `schema` object (fields/partitionKeys/primaryKeys/options/comment) is +/// kept as a raw JSON string and converted to a `TableSchema` by the catalog. +class GetTableResponse : public Jsonizable { + public: + GetTableResponse(const std::string& id, const std::string& database, const std::string& name, + const std::string& path, bool is_external, int64_t schema_id, + const std::string& schema_json) + : id_(id), + database_(database), + name_(name), + path_(path), + is_external_(is_external), + schema_id_(schema_id), + schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetId() const { + return id_; + } + /// The database of the table. The server reports it so that a table addressed by its + /// id alone can be resolved; it is empty on the endpoints that address a table by + /// database and name, which are the ones `RestCatalog` uses. + const std::string& GetDatabase() const { + return database_; + } + const std::string& GetName() const { + return name_; + } + const std::string& GetPath() const { + return path_; + } + bool IsExternal() const { + return is_external_; + } + int64_t GetSchemaId() const { + return schema_id_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + const RestAuditFields& GetAuditFields() const { + return audit_; + } + + GetTableResponse() = default; + + private: + std::string id_; + std::string database_; + std::string name_; + std::string path_; + bool is_external_ = false; + int64_t schema_id_ = 0; + std::string schema_json_; + RestAuditFields audit_; +}; + +/// `schema_json` uses the same schema JSON layout as `GetTableResponse`. +class CreateTableRequest : public Jsonizable { + public: + CreateTableRequest(const std::string& database, const std::string& table, + const std::string& schema_json) + : database_(database), table_(table), schema_json_(schema_json) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetDatabase() const { + return database_; + } + const std::string& GetTable() const { + return table_; + } + const std::string& GetSchemaJson() const { + return schema_json_; + } + + CreateTableRequest() = default; + + private: + std::string database_; + std::string table_; + std::string schema_json_; +}; + +class RenameTableRequest : public Jsonizable { + public: + RenameTableRequest(const std::string& source_database, const std::string& source_table, + const std::string& destination_database, + const std::string& destination_table) + : source_database_(source_database), + source_table_(source_table), + destination_database_(destination_database), + destination_table_(destination_table) {} + + rapidjson::Value ToJson(rapidjson::Document::AllocatorType* allocator) const + noexcept(false) override; + void FromJson(const rapidjson::Value& obj) noexcept(false) override; + + const std::string& GetSourceDatabase() const { + return source_database_; + } + const std::string& GetSourceTable() const { + return source_table_; + } + const std::string& GetDestinationDatabase() const { + return destination_database_; + } + const std::string& GetDestinationTable() const { + return destination_table_; + } + + RenameTableRequest() = default; + + private: + std::string source_database_; + std::string source_table_; + std::string destination_database_; + std::string destination_table_; +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_messages_test.cpp b/src/paimon/rest/rest_messages_test.cpp new file mode 100644 index 00000000..7a29b77c --- /dev/null +++ b/src/paimon/rest/rest_messages_test.cpp @@ -0,0 +1,267 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_messages.h" + +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/utils/rapidjson_util.h" +#include "paimon/testing/utils/testharness.h" +#include "rapidjson/document.h" + +namespace paimon::test { + +TEST(RestMessagesTest, ErrorResponseRoundTrip) { + ErrorResponse response(ErrorResponse::kResourceTypeDatabase, "db1", "database db1 not found", + 404); + ASSERT_OK_AND_ASSIGN(std::string json, response.ToJsonString()); + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString(json)); + ASSERT_EQ("DATABASE", parsed.GetResourceType()); + ASSERT_EQ("db1", parsed.GetResourceName()); + ASSERT_EQ("database db1 not found", parsed.GetMessage()); + ASSERT_EQ(404, parsed.GetCode()); +} + +TEST(RestMessagesTest, ErrorResponseLenientParse) { + ASSERT_OK_AND_ASSIGN(ErrorResponse parsed, ErrorResponse::FromJsonString("{}")); + ASSERT_EQ("", parsed.GetMessage()); + ASSERT_EQ(0, parsed.GetCode()); +} + +TEST(RestMessagesTest, ConfigResponseMerge) { + // null values may be sent by the server: a null default carries nothing to merge, + // a null override unsets the key + std::string json = R"({ + "defaults": {"prefix": "server-prefix", "a": "default-a", "b": "default-b", + "nullable": null, "unset-default": "default-value"}, + "overrides": {"c": "override-c", "a": "override-a", + "unset-client": null, "unset-default": null, "unset-absent": null} + })"; + ASSERT_OK_AND_ASSIGN(ConfigResponse config, ConfigResponse::FromJsonString(json)); + std::map client = { + {"a", "client-a"}, {"b", "client-b"}, {"d", "client-d"}, {"unset-client", "client-value"}}; + std::map merged = config.Merge(client); + // overrides > client options > defaults + ASSERT_EQ("override-a", merged["a"]); + ASSERT_EQ("client-b", merged["b"]); + ASSERT_EQ("override-c", merged["c"]); + ASSERT_EQ("client-d", merged["d"]); + ASSERT_EQ("server-prefix", merged["prefix"]); + ASSERT_EQ(0, merged.count("nullable")); + // a null override beats the client option and the default; on an unknown key it is + // a no-op + ASSERT_EQ(0, merged.count("unset-client")); + ASSERT_EQ(0, merged.count("unset-default")); + ASSERT_EQ(0, merged.count("unset-absent")); +} + +TEST(RestMessagesTest, ListResponsesParse) { + ASSERT_OK_AND_ASSIGN(ListDatabasesResponse databases, + ListDatabasesResponse::FromJsonString( + R"({"databases": ["db1", "db2"], "nextPageToken": "token1"})")); + ASSERT_EQ((std::vector{"db1", "db2"}), databases.Data()); + ASSERT_EQ("token1", databases.NextPageToken().value()); + + ASSERT_OK_AND_ASSIGN(ListTablesResponse tables, + ListTablesResponse::FromJsonString(R"({"tables": ["t1"]})")); + ASSERT_EQ((std::vector{"t1"}), tables.Data()); + ASSERT_FALSE(tables.NextPageToken().has_value()); + + // nextPageToken may be serialized as explicit null by the server + ASSERT_OK_AND_ASSIGN( + ListTablesResponse null_token, + ListTablesResponse::FromJsonString(R"({"tables": [], "nextPageToken": null})")); + ASSERT_FALSE(null_token.NextPageToken().has_value()); +} + +TEST(RestMessagesTest, ListSnapshotsResponseParse) { + std::string json = R"({ + "snapshots": [{ + "version": 3, + "id": 7, + "schemaId": 2, + "baseManifestList": "manifest-list-1", + "deltaManifestList": "manifest-list-2", + "commitUser": "user1", + "commitIdentifier": 9, + "commitKind": "APPEND", + "timeMillis": 1234567, + "totalRecordCount": 100, + "deltaRecordCount": 10, + "watermark": 42 + }, { + "version": 3, + "id": 8, + "schemaId": 2, + "baseManifestList": "manifest-list-3", + "deltaManifestList": "manifest-list-4", + "commitUser": "user1", + "commitIdentifier": 10, + "commitKind": "COMPACT", + "timeMillis": 1234568, + "totalRecordCount": 100, + "deltaRecordCount": 0 + }], + "nextPageToken": null + })"; + ASSERT_OK_AND_ASSIGN(ListSnapshotsResponse response, + ListSnapshotsResponse::FromJsonString(json)); + ASSERT_EQ(2, response.Data().size()); + const Snapshot& snapshot = response.Data()[0]; + ASSERT_EQ(7, snapshot.Id()); + ASSERT_EQ(2, snapshot.SchemaId()); + ASSERT_EQ("user1", snapshot.CommitUser()); + ASSERT_EQ(1234567, snapshot.TimeMillis()); + SnapshotInfo info = snapshot.ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::APPEND, info.commit_kind); + ASSERT_EQ(100, info.total_record_count.value()); + ASSERT_EQ(42, info.watermark.value()); + SnapshotInfo compact_info = response.Data()[1].ToSnapshotInfo(); + ASSERT_EQ(SnapshotInfo::CommitKind::COMPACT, compact_info.commit_kind); + ASSERT_FALSE(compact_info.watermark.has_value()); +} + +TEST(RestMessagesTest, GetDatabaseResponseParse) { + std::string json = R"({ + "id": "10", + "name": "db1", + "location": "/warehouse/db1.db", + "options": {"k1": "v1"}, + "owner": "owner1", + "createdAt": 100, + "createdBy": "creator", + "updatedAt": 200, + "updatedBy": "updater" + })"; + ASSERT_OK_AND_ASSIGN(GetDatabaseResponse response, GetDatabaseResponse::FromJsonString(json)); + ASSERT_EQ("db1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db", response.GetLocation()); + ASSERT_EQ("v1", response.GetOptions().at("k1")); + std::map options; + response.GetAuditFields().PutAuditOptionsTo(&options); + ASSERT_EQ("owner1", options.at("owner")); + ASSERT_EQ("100", options.at("createdAt")); + ASSERT_EQ("updater", options.at("updatedBy")); +} + +TEST(RestMessagesTest, GetTableResponseParse) { + std::string json = R"({ + "id": "42", + "database": "db1", + "name": "t1", + "path": "/warehouse/db1.db/t1", + "isExternal": false, + "schemaId": 3, + "schema": { + "fields": [ + {"id": 0, "name": "f0", "type": "INT NOT NULL"}, + {"id": 1, "name": "f1", "type": "STRING"} + ], + "partitionKeys": [], + "primaryKeys": ["f0"], + "options": {"bucket": "2"}, + "comment": "a table" + }, + "updatedAt": 300 + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("42", response.GetId()); + ASSERT_EQ("db1", response.GetDatabase()); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ("/warehouse/db1.db/t1", response.GetPath()); + ASSERT_FALSE(response.IsExternal()); + ASSERT_EQ(3, response.GetSchemaId()); + ASSERT_EQ(300, response.GetAuditFields().updated_at.value()); + rapidjson::Document schema; + schema.Parse(response.GetSchemaJson().c_str()); + ASSERT_FALSE(schema.HasParseError()); + ASSERT_EQ(2u, schema["fields"].Size()); + ASSERT_STREQ("f0", schema["fields"][0]["name"].GetString()); +} + +TEST(RestMessagesTest, UnknownFieldsAreIgnored) { + // forward compatibility: fields a newer server adds must be ignored + std::string json = R"({ + "id": "42", "name": "t1", "path": "p", "schemaId": 3, + "schema": {"fields": []}, + "futureField": {"nested": true}, "anotherUnknown": [1, 2] + })"; + ASSERT_OK_AND_ASSIGN(GetTableResponse response, GetTableResponse::FromJsonString(json)); + ASSERT_EQ("t1", response.GetName()); + ASSERT_EQ(3, response.GetSchemaId()); + // the endpoints addressing a table by database and name report no "database" + ASSERT_EQ("", response.GetDatabase()); +} + +TEST(RestMessagesTest, MissingRequiredFieldsFail) { + // "path", "schemaId" and "schema" are required + ASSERT_NOK(GetTableResponse::FromJsonString(R"({"id": "42", "name": "t1"})").status()); + // a non-object identifier is rejected + CreateTableRequest bad_identifier("", "", ""); + ASSERT_NOK(RapidJsonUtil::FromJsonString(R"({"identifier": "not-an-object", "schema": {}})", + &bad_identifier)); +} + +TEST(RestMessagesTest, CreateTableRequestSerialize) { + CreateTableRequest request( + "db1", "t1", R"({"fields": [], "partitionKeys": [], "primaryKeys": [], "options": {}})"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("db1", doc["identifier"]["database"].GetString()); + ASSERT_STREQ("t1", doc["identifier"]["object"].GetString()); + ASSERT_TRUE(doc["schema"].IsObject()); + ASSERT_TRUE(doc["schema"]["fields"].IsArray()); + + ASSERT_OK_AND_ASSIGN(CreateTableRequest parsed, CreateTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetDatabase()); + ASSERT_EQ("t1", parsed.GetTable()); +} + +TEST(RestMessagesTest, InvalidSchemaJsonErrorOmitsPayload) { + CreateTableRequest request("db1", "t1", R"({"fields": [], "options": {"token": "top-secret")"); + Status status = request.ToJsonString().status(); + ASSERT_NOK_WITH_MSG(status, "invalid json"); + // the payload may carry credentials, so it must not be echoed back in the error + ASSERT_EQ(std::string::npos, status.ToString().find("top-secret")) << status.ToString(); +} + +TEST(RestMessagesTest, RenameTableRequestSerialize) { + RenameTableRequest request("db1", "t1", "db1", "t2"); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + rapidjson::Document doc; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + ASSERT_STREQ("t1", doc["source"]["object"].GetString()); + ASSERT_STREQ("t2", doc["destination"]["object"].GetString()); + + ASSERT_OK_AND_ASSIGN(RenameTableRequest parsed, RenameTableRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetSourceDatabase()); + ASSERT_EQ("t2", parsed.GetDestinationTable()); +} + +TEST(RestMessagesTest, CreateDatabaseRequestRoundTrip) { + CreateDatabaseRequest request("db1", {{"k1", "v1"}}); + ASSERT_OK_AND_ASSIGN(std::string json, request.ToJsonString()); + ASSERT_OK_AND_ASSIGN(CreateDatabaseRequest parsed, CreateDatabaseRequest::FromJsonString(json)); + ASSERT_EQ("db1", parsed.GetName()); + ASSERT_EQ("v1", parsed.GetOptions().at("k1")); +} + +} // namespace paimon::test diff --git a/src/paimon/rest/rest_util.cpp b/src/paimon/rest/rest_util.cpp new file mode 100644 index 00000000..b887d256 --- /dev/null +++ b/src/paimon/rest/rest_util.cpp @@ -0,0 +1,74 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_util.h" + +#include + +#include "fmt/format.h" +#include "rapidjson/error/en.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/writer.h" + +namespace paimon { + +std::map RestUtil::ExtractPrefixMap( + const std::map& options, const std::string& prefix) { + std::map result; + for (const auto& [key, value] : options) { + if (key.size() > prefix.size() && key.compare(0, prefix.size(), prefix) == 0) { + result[key.substr(prefix.size())] = value; + } + } + return result; +} + +std::string RestUtil::ExtractRequestId(const std::map& headers) { + auto iter = headers.find(kRequestIdHeader); + if (iter != headers.end() && !iter->second.empty()) { + return iter->second; + } + for (const auto& [name, value] : headers) { + if (!value.empty() && name.find("request-id") != std::string::npos) { + return value; + } + } + return kUnknownRequestId; +} + +std::string RestUtil::JsonToString(const rapidjson::Value& value) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value.Accept(writer); + return buffer.GetString(); +} + +rapidjson::Value RestUtil::ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator) { + rapidjson::Document doc; + doc.Parse(json.c_str()); + if (doc.HasParseError()) { + // The payload may carry credentials and be arbitrarily large; report only the error. + throw std::invalid_argument(fmt::format("invalid json: {} (at offset {})", + rapidjson::GetParseError_En(doc.GetParseError()), + doc.GetErrorOffset())); + } + rapidjson::Value value; + value.CopyFrom(doc, *allocator); + return value; +} + +} // namespace paimon diff --git a/src/paimon/rest/rest_util.h b/src/paimon/rest/rest_util.h new file mode 100644 index 00000000..407217c6 --- /dev/null +++ b/src/paimon/rest/rest_util.h @@ -0,0 +1,61 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "rapidjson/allocators.h" +#include "rapidjson/document.h" +#include "rapidjson/rapidjson.h" + +namespace paimon { + +/// Utilities for the REST catalog; URL encoding and decoding live in `UrlUtils`, +/// credential redaction in `SensitiveConfigUtils`. +class RestUtil { + public: + RestUtil() = delete; + ~RestUtil() = delete; + + /// Header under which the rest server reports the id of a request. + static constexpr const char* kRequestIdHeader = "x-request-id"; + /// Placeholder `ExtractRequestId` returns when no header carries a request id. + static constexpr const char* kUnknownRequestId = "unknown"; + + /// Extract all options whose key starts with `prefix`, with the prefix stripped from + /// the resulting keys. A key exactly equal to the prefix is dropped instead of + /// yielding an empty key. + static std::map ExtractPrefixMap( + const std::map& options, const std::string& prefix); + + /// Returns the request id tying a log line or an error to the server side trace: the + /// `kRequestIdHeader` value, falling back to any other header whose name contains + /// "request-id" (a gateway may report it under its own name, e.g. "x-amz-request-id"), + /// then to `kUnknownRequestId`. `headers` must have lower-cased names. + static std::string ExtractRequestId(const std::map& headers); + + /// Serialize a rapidjson value to a compact JSON string. + static std::string JsonToString(const rapidjson::Value& value); + + /// Parse a JSON string into a rapidjson value owned by `allocator`. + /// Throws `std::invalid_argument` on parse error, consistent with `Jsonizable`. + static rapidjson::Value ParseToValue(const std::string& json, + rapidjson::Document::AllocatorType* allocator); +}; + +} // namespace paimon diff --git a/src/paimon/rest/rest_util_test.cpp b/src/paimon/rest/rest_util_test.cpp new file mode 100644 index 00000000..e76baf15 --- /dev/null +++ b/src/paimon/rest/rest_util_test.cpp @@ -0,0 +1,47 @@ +/* + * Copyright 2026-present Alibaba Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "paimon/rest/rest_util.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace paimon::test { + +TEST(RestUtilTest, ExtractPrefixMap) { + std::map options = { + {"header.k1", "v1"}, {"header.k2", "v2"}, {"other", "v3"}, {"header.", "v4"}}; + std::map expected = {{"k1", "v1"}, {"k2", "v2"}}; + ASSERT_EQ(expected, RestUtil::ExtractPrefixMap(options, "header.")); +} + +TEST(RestUtilTest, ExtractRequestId) { + ASSERT_EQ("req-1", RestUtil::ExtractRequestId({{"x-request-id", "req-1"}})); + // a gateway may report the request id under a name of its own + ASSERT_EQ("amz-1", RestUtil::ExtractRequestId({{"x-amz-request-id", "amz-1"}})); + // the dedicated header wins over a gateway one + ASSERT_EQ("req-1", RestUtil::ExtractRequestId( + {{"x-request-id", "req-1"}, {"x-amz-request-id", "amz-1"}})); + // an empty value carries no id and falls through like an absent header + ASSERT_EQ("amz-1", + RestUtil::ExtractRequestId({{"x-request-id", ""}, {"x-amz-request-id", "amz-1"}})); + ASSERT_EQ(RestUtil::kUnknownRequestId, RestUtil::ExtractRequestId({})); + ASSERT_EQ(RestUtil::kUnknownRequestId, RestUtil::ExtractRequestId({{"content-type", "json"}})); +} + +} // namespace paimon::test diff --git a/test/inte/read_inte_test.cpp b/test/inte/read_inte_test.cpp index 9ebf2d60..4ad82d0c 100644 --- a/test/inte/read_inte_test.cpp +++ b/test/inte/read_inte_test.cpp @@ -3915,7 +3915,14 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { {Options::FILE_SYSTEM, "local"}, {Options::FILE_FORMAT, "orc"}, {CatalogOptionsSystemTable::kEnabledOption, "true"}, - {"custom.catalog.option", "test-value"}}; + {"custom.catalog.option", "test-value"}, + {"token", "bearer-credential-1"}, + {"dlf.access-key-secret", "aksecret-1"}, + {"fs.s3a.access.key", "s3akey-1"}, + {"fs.azure.account-key.store1", "azkey-1"}, + {"client.credential", "cred-1"}, + {"fs.azure.sas.container", "sas-1"}, + {"dlf.access-key-id", "an-access-key-id-1"}}; auto dir = UniqueTestDirectory::Create(); ASSERT_TRUE(dir); std::string warehouse = PathUtil::JoinPath(dir->Str(), "warehouse"); @@ -3937,6 +3944,26 @@ TEST(SystemTableReadInteTest, TestReadGlobalCatalogOptions) { } ASSERT_EQ(result_map["file-system"], "local"); ASSERT_EQ(result_map["file.format"], "orc"); + // credential-carrying options stay listed but their values are masked; the key match + // ignores separators, so "fs.s3a.access.key" hits the "accesskey" marker + ASSERT_EQ(result_map["token"], "******"); + ASSERT_EQ(result_map["dlf.access-key-secret"], "******"); + ASSERT_EQ(result_map["fs.s3a.access.key"], "******"); + ASSERT_EQ(result_map["fs.azure.account-key.store1"], "******"); + ASSERT_EQ(result_map["client.credential"], "******"); + ASSERT_EQ(result_map["fs.azure.sas.container"], "******"); + // an identifier-like key keeps a four character tail, which names the credential in a + // support case without disclosing it + ASSERT_EQ(result_map["dlf.access-key-id"], "****id-1"); + for (const auto& [key, value] : result_map) { + ASSERT_EQ(std::string::npos, value.find("bearer-credential-1")) << key; + ASSERT_EQ(std::string::npos, value.find("aksecret-1")) << key; + ASSERT_EQ(std::string::npos, value.find("s3akey-1")) << key; + ASSERT_EQ(std::string::npos, value.find("azkey-1")) << key; + ASSERT_EQ(std::string::npos, value.find("cred-1")) << key; + ASSERT_EQ(std::string::npos, value.find("sas-1")) << key; + ASSERT_EQ(std::string::npos, value.find("an-access-key-id-1")) << key; + } } TEST(SystemTableReadInteTest, TestReadGlobalAllTableOptions) {