Antalya 26.5: Token Authentication and Authorization

ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt

@@ -398,6 +398,8 @@ Dresseler
 DuckDB
 Duployan
 Durre
+ECDSA
+EdDSA
 ECMA
 EDAC
 ETag
@@ -732,6 +734,8 @@ Jpan
 JumpConsistentHash
 Jupyter
 Jurc
+jwks
+JWKS
 KDevelop
 KafkaAssignedPartitions
 KafkaBackgroundReads
@@ -4099,6 +4103,7 @@ uuid
 uuids
 uuidv
 vCPU
+validators
 vLLM
 varPop
 varPopStable
@@ -4117,6 +4122,8 @@ vectorscan
 vendoring
 verificationDepth
 verificationMode
+verifier
+verifiers
 versionedcollapsingmergetree
 vhost
 virtualized

docs/en/operations/external-authenticators/index.md

@@ -19,4 +19,5 @@ The following external authenticators and directories are supported:
 - [LDAP](/operations/external-authenticators/ldap#ldap-external-authenticator) [Authenticator](./ldap.md#ldap-external-authenticator) and [Directory](./ldap.md#ldap-external-user-directory)
 - Kerberos [Authenticator](/operations/external-authenticators/kerberos#kerberos-as-an-external-authenticator-for-existing-users)
 - [SSL X.509 authentication](/operations/external-authenticators/ssl-x509)
-- HTTP [Authenticator](./http.md)
+- HTTP [Authenticator](./http.md)
+- Token-based [Authenticator](./tokens.md)

src/Access/AccessControl.cpp

@@ -5,6 +5,7 @@
 #include <Access/UsersConfigAccessStorage.h>
 #include <Access/DiskAccessStorage.h>
 #include <Access/LDAPAccessStorage.h>
+#include <Access/TokenAccessStorage.h>
 #include <Access/ContextAccess.h>
 #include <Access/EnabledSettings.h>
 #include <Access/EnabledRolesInfo.h>
@@ -43,6 +44,7 @@ namespace ErrorCodes
     extern const int REQUIRED_PASSWORD;
     extern const int CANNOT_COMPILE_REGEXP;
     extern const int BAD_ARGUMENTS;
+    extern const int INVALID_CONFIG_PARAMETER;
 }
 
 namespace
@@ -292,6 +294,8 @@ void AccessControl::setupFromMainConfig(const Poco::Util::AbstractConfiguration
     setDefaultPasswordTypeFromConfig(config_.getString("default_password_type", "sha256_password"));
     setPasswordComplexityRulesFromConfig(config_);
 
+    setTokenAuthEnabled(config_.getBool("enable_token_auth", true));
+
     setBcryptWorkfactor(config_.getInt("bcrypt_workfactor", 12));
 
     /// Optional improvements in access control system.
@@ -430,6 +434,12 @@ void AccessControl::addLDAPStorage(const String & storage_name_, const Poco::Uti
     LOG_DEBUG(getLogger(), "Added {} access storage '{}', LDAP server name: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getLDAPServerName());
 }
 
+void AccessControl::addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_)
+{
+    auto new_storage = std::make_shared<TokenAccessStorage>(storage_name_, *this, config_, prefix_);
+    addStorage(new_storage);
+    LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName());
+}
 
 void AccessControl::addStoragesFromUserDirectoriesConfig(
     const Poco::Util::AbstractConfiguration & config,
@@ -442,6 +452,8 @@ void AccessControl::addStoragesFromUserDirectoriesConfig(
     Strings keys_in_user_directories;
     config.keys(key, keys_in_user_directories);
 
+    bool has_token_storage = false;
+
     for (const String & key_in_user_directories : keys_in_user_directories)
     {
         String prefix = key + "." + key_in_user_directories;
@@ -455,6 +467,8 @@ void AccessControl::addStoragesFromUserDirectoriesConfig(
             type = DiskAccessStorage::STORAGE_TYPE;
         else if (type == "ldap")
             type = LDAPAccessStorage::STORAGE_TYPE;
+        else if (type == "token")
+            type = TokenAccessStorage::STORAGE_TYPE;
 
         String name = config.getString(prefix + ".name", type);
 
@@ -488,6 +502,20 @@ void AccessControl::addStoragesFromUserDirectoriesConfig(
             bool allow_backup = config.getBool(prefix + ".allow_backup", true);
             addReplicatedStorage(name, zookeeper_path, get_zookeeper_function, allow_backup);
         }
+        else if (type == TokenAccessStorage::STORAGE_TYPE)
+        {
+            if (!isTokenAuthEnabled())
+            {
+                LOG_INFO(getLogger(), "Token authentication is disabled, skipping token user directory '{}'", name);
+                continue;
+            }
+
+            if (has_token_storage)
+                throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Only one `token` section can be defined.");
+
+            addTokenStorage(name, config, prefix);
+            has_token_storage = true;
+        }
         else
             throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown storage type '{}' at {} in config", type, prefix);
     }
@@ -666,7 +694,7 @@ void AccessControl::restoreFromBackup(RestorerFromBackup & restorer, const Strin
 
 void AccessControl::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config)
 {
-    external_authenticators->setConfiguration(config, getLogger());
+    external_authenticators->setConfiguration(config, getLogger(), isTokenAuthEnabled());
 }
 
 
@@ -945,4 +973,14 @@ bool AccessControl::getAllowBetaTierSettings() const
 {
     return allow_beta_tier_settings;
 }
+
+void AccessControl::setTokenAuthEnabled(bool enable)
+{
+    enable_token_auth = enable;
+}
+
+bool AccessControl::isTokenAuthEnabled() const
+{
+    return enable_token_auth;
+}
 }

src/Access/AccessControl.h

@@ -93,6 +93,8 @@ class AccessControl : public MultipleAccessStorage
     /// Adds LDAPAccessStorage which allows querying remote LDAP server for user info.
     void addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_);
 
+    void addTokenStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_);
+
     void addReplicatedStorage(const String & storage_name,
                               const String & zookeeper_path,
                               const zkutil::GetZooKeeper & get_zookeeper_function,
@@ -271,6 +273,10 @@ class AccessControl : public MultipleAccessStorage
     bool getAllowExperimentalTierSettings() const;
     bool getAllowBetaTierSettings() const;
 
+    /// Controls whether token-based auth is enabled.
+    void setTokenAuthEnabled(bool enable);
+    bool isTokenAuthEnabled() const;
+
 private:
     class ContextAccessCache;
     class CustomSettingsPrefixes;
@@ -307,6 +313,7 @@ class AccessControl : public MultipleAccessStorage
     std::atomic_bool enable_user_name_access_type = true;
     std::atomic_bool enable_read_write_grants = false;
     std::atomic_bool allow_impersonate_user = false;
+    std::atomic_bool enable_token_auth = true;
 };
 
 }

src/Access/Authentication.cpp

@@ -329,7 +329,10 @@ Authentication::CredentialsCheckResult Authentication::areCredentialsValid(
     const ClientInfo & client_info,
     SettingsChanges & settings)
 {
-    if (!credentials.isReady())
+    /// It is OK for TokenCredentials to be not ready:
+    /// When auth request happens, we do not even know the username.
+    /// Token is resolved a bit later and the user information will be put in credentials
+    if (!typeid_cast<const TokenCredentials *>(&credentials) && !credentials.isReady())
         return CredentialsCheckResult::Fail;
 
     if (const auto * gss_acceptor_context = typeid_cast<const GSSAcceptorContext *>(&credentials))
@@ -377,6 +380,18 @@ Authentication::CredentialsCheckResult Authentication::areCredentialsValid(
     }
 #endif
 
+    if (const auto * token_credentials = typeid_cast<const TokenCredentials *>(&credentials))
+    {
+        if (authentication_method.getType() != AuthenticationType::JWT)
+            return CredentialsCheckResult::Fail;
+
+        return external_authenticators.checkTokenCredentials(
+                   *token_credentials,
+                   authentication_method.getTokenProcessorName(),
+                   authentication_method.getJWTClaims())
+                   ? CredentialsCheckResult::Success : CredentialsCheckResult::Fail;
+    }
+
     if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast<const AlwaysAllowCredentials *>(&credentials))
         return CredentialsCheckResult::Success;
 

src/Access/AuthenticationData.cpp

@@ -28,13 +28,16 @@
 #     include <bcrypt.h>
 #endif
 
+#if USE_JWT_CPP
+#include <picojson/picojson.h>
+#endif
+
 namespace CurrentMetrics
 {
     extern const Metric BcryptCacheBytes;
     extern const Metric BcryptCacheSize;
 }
 
-
 namespace DB
 {
 
@@ -411,7 +414,10 @@ boost::intrusive_ptr<ASTAuthenticationData> AuthenticationData::toAST() const
         }
         case AuthenticationType::JWT:
         {
-            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+            const auto & claims = getJWTClaims();
+            if (!claims.empty())
+                node->children.push_back(make_intrusive<ASTLiteral>(claims));
+            break;
         }
         case AuthenticationType::KERBEROS:
         {
@@ -689,6 +695,22 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
         auth_data.setHTTPAuthenticationServerName(server);
         auth_data.setHTTPAuthenticationScheme(scheme);
     }
+#if USE_JWT_CPP
+    else if (query.type == AuthenticationType::JWT)
+    {
+        if (!args.empty())
+        {
+            String value = checkAndGetLiteralArgument<String>(args[0], "claims");
+            picojson::value json_obj;
+            auto error = picojson::parse(json_obj, value);
+            if (!error.empty())
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: {}", error);
+            if (!json_obj.is<picojson::object>())
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad JWT claims: is not an object");
+            auth_data.setJWTClaims(value);
+        }
+    }
+#endif
     else
     {
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected ASTAuthenticationData structure");

src/Access/AuthenticationData.h

@@ -82,6 +82,12 @@ class AuthenticationData
     time_t getValidUntil() const { return valid_until; }
     void setValidUntil(time_t valid_until_) { valid_until = valid_until_; }
 
+    const String & getJWTClaims() const { return jwt_claims; }
+    void setJWTClaims(const String & jwt_claims_) { jwt_claims = jwt_claims_; }
+
+    const String & getTokenProcessorName() const { return token_processor_name; }
+    void setTokenProcessorName(const String & token_processor_name_) { token_processor_name = token_processor_name_; }
+
     friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs);
     friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); }
 
@@ -121,6 +127,8 @@ class AuthenticationData
     String http_auth_server_name;
     HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC;
     time_t valid_until = 0;
+    String jwt_claims;
+    String token_processor_name;
 };
 
 }

src/Access/Common/JWKSProvider.cpp

@@ -0,0 +1,113 @@
+#include <Access/Common/JWKSProvider.h>
+
+#if USE_JWT_CPP
+#include <Common/Exception.h>
+#include <mutex>
+#include <Poco/Net/HTTPRequest.h>
+#include <Poco/Net/HTTPResponse.h>
+#include <Poco/Net/HTTPSClientSession.h>
+#include <Poco/StreamCopier.h>
+#include <fstream>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int AUTHENTICATION_FAILED;
+    extern const int INVALID_CONFIG_PARAMETER;
+}
+
+JWKSType JWKSClient::getJWKS()
+{
+    {
+        std::shared_lock lock(mutex);
+        auto now = std::chrono::high_resolution_clock::now();
+        auto diff = std::chrono::duration<double>(now - last_request_send).count();
+        if (diff < static_cast<double>(refresh_timeout) && cached_jwks.has_value())
+            return cached_jwks.value();
+    }
+
+    std::unique_lock lock(mutex);
+    auto now = std::chrono::high_resolution_clock::now();
+    auto diff = std::chrono::duration<double>(now - last_request_send).count();
+    if (diff < static_cast<double>(refresh_timeout) && cached_jwks.has_value())
+        return cached_jwks.value();
+
+    Poco::Net::HTTPResponse response;
+    std::string response_string;
+
+    Poco::Net::HTTPRequest request{Poco::Net::HTTPRequest::HTTP_GET, jwks_uri.getPathAndQuery()};
+
+    if (jwks_uri.getScheme() == "https")
+    {
+        Poco::Net::HTTPSClientSession session = Poco::Net::HTTPSClientSession(jwks_uri.getHost(), jwks_uri.getPort());
+        session.sendRequest(request);
+        std::istream & response_stream = session.receiveResponse(response);
+        if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !response_stream)
+            throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}",
+                response.getStatus(), response.getReason());
+        Poco::StreamCopier::copyToString(response_stream, response_string);
+    }
+    else
+    {
+        Poco::Net::HTTPClientSession session = Poco::Net::HTTPClientSession(jwks_uri.getHost(), jwks_uri.getPort());
+        session.sendRequest(request);
+        std::istream & response_stream = session.receiveResponse(response);
+        if (response.getStatus() != Poco::Net::HTTPResponse::HTTP_OK || !response_stream)
+            throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to get user info by access token, code: {}, reason: {}", response.getStatus(), response.getReason());
+        Poco::StreamCopier::copyToString(response_stream, response_string);
+    }
+
+    last_request_send = std::chrono::high_resolution_clock::now();
+
+    JWKSType parsed_jwks;
+
+    try
+    {
+        parsed_jwks = jwt::parse_jwks(response_string);
+    }
+    catch (const std::exception & e)
+    {
+        throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse JWKS: {}", e.what());
+    }
+
+    cached_jwks = std::move(parsed_jwks);
+    return cached_jwks.value();
+}
+
+StaticJWKSParams::StaticJWKSParams(const std::string & static_jwks_, const std::string & static_jwks_file_)
+{
+    if (static_jwks_.empty() && static_jwks_file_.empty())
+        throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER,
+                        "JWT validator misconfigured: `static_jwks` or `static_jwks_file` keys must be present in static JWKS validator configuration");
+    if (!static_jwks_.empty() && !static_jwks_file_.empty())
+        throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER,
+                        "JWT validator misconfigured: `static_jwks` and `static_jwks_file` keys cannot both be present in static JWKS validator configuration");
+
+    static_jwks = static_jwks_;
+    static_jwks_file = static_jwks_file_;
+}
+
+StaticJWKS::StaticJWKS(const StaticJWKSParams & params)
+{
+    String content = String(params.static_jwks);
+    if (!params.static_jwks_file.empty())
+    {
+        std::ifstream ifs(params.static_jwks_file);
+        Poco::StreamCopier::copyToString(ifs, content);
+    }
+    try
+    {
+        auto keys = jwt::parse_jwks(content);
+        jwks = std::move(keys);
+    }
+    catch (const std::exception & e)
+    {
+        throw Exception(ErrorCodes::AUTHENTICATION_FAILED, "Failed to parse JWKS: {}", e.what());
+    }
+}
+
+}
+#endif