From f34fdcc5778db438d9d1122ca03152872e1f1830 Mon Sep 17 00:00:00 2001 From: Remi Gacogne Date: Tue, 22 Oct 2019 17:16:53 +0200 Subject: [PATCH] dnsdist: Add metrics about TLS handshake failures for DoH and DoT --- pdns/dnsdist-carbon.cc | 17 ++++++ pdns/dnsdist-console.cc | 1 + pdns/dnsdist-lua-inspection.cc | 31 +++++++++++ pdns/dnsdist-web.cc | 61 +++++++++++++++++---- pdns/dnsdist.hh | 5 ++ pdns/dnsdistdist/docs/reference/config.rst | 6 +++ pdns/dnsdistdist/doh.cc | 2 + pdns/dnsdistdist/libssl.cc | 63 ++++++++++++++++++++++ pdns/dnsdistdist/tcpiohandler.cc | 6 ++- pdns/doh.hh | 1 + pdns/libssl.hh | 15 ++++++ pdns/tcpiohandler.hh | 1 + 12 files changed, 197 insertions(+), 12 deletions(-) diff --git a/pdns/dnsdist-carbon.cc b/pdns/dnsdist-carbon.cc index ad8dfb7cb..72473ca00 100644 --- a/pdns/dnsdist-carbon.cc +++ b/pdns/dnsdist-carbon.cc @@ -139,6 +139,23 @@ try str<tlsResumptions.load() << " " << now << "\r\n"; str<tlsUnknownTicketKey.load() << " " << now << "\r\n"; str<tlsInactiveTicketKey.load() << " " << now << "\r\n"; + const TLSErrorCounters* errorCounters = nullptr; + if (front->tlsFrontend != nullptr) { + errorCounters = &front->tlsFrontend->d_tlsCounters; + } + else if (front->dohFrontend != nullptr) { + errorCounters = &front->dohFrontend->d_tlsCounters; + } + if (errorCounters != nullptr) { + str<d_dhKeyTooSmall << " " << now << "\r\n"; + str<d_inappropriateFallBack << " " << now << "\r\n"; + str<d_noSharedCipher << " " << now << "\r\n"; + str<d_unknownCipherType << " " << now << "\r\n"; + str<d_unknownKeyExchangeType << " " << now << "\r\n"; + str<d_unknownProtocol << " " << now << "\r\n"; + str<d_unsupportedEC << " " << now << "\r\n"; + str<d_unsupportedProtocol << " " << now << "\r\n"; + } } auto localPools = g_pools.getLocal(); diff --git a/pdns/dnsdist-console.cc b/pdns/dnsdist-console.cc index 83613b9de..75ef60d9d 100644 --- a/pdns/dnsdist-console.cc +++ b/pdns/dnsdist-console.cc @@ -559,6 +559,7 @@ const std::vector g_consoleKeywords{ { "showServers", true, "[{showUUIDs=false}]", "output all servers, optionally with their UUIDs" }, { "showTCPStats", true, "", "show some statistics regarding TCP" }, { "showTLSContexts", true, "", "list all the available TLS contexts" }, + { "showTLSErrorCounters", true, "", "show metrics about TLS handshake failures" }, { "showVersion", true, "", "show the current version" }, { "shutdown", true, "", "shut down `dnsdist`" }, { "SkipCacheAction", true, "", "Don’t lookup the cache for this query, don’t store the answer" }, diff --git a/pdns/dnsdist-lua-inspection.cc b/pdns/dnsdist-lua-inspection.cc index f1c453a5b..1c335f283 100644 --- a/pdns/dnsdist-lua-inspection.cc +++ b/pdns/dnsdist-lua-inspection.cc @@ -590,6 +590,37 @@ void setupLuaInspection() g_outputBuffer=ret.str(); }); + g_lua.writeFunction("showTLSErrorCounters", [] { + setLuaNoSideEffect(); + ostringstream ret; + boost::format fmt("%-3d %-20.20s %-23d %-23d %-23d %-23d %-23d %-23d %-23d %-23d"); + + ret << (fmt % "#" % "Address" % "DH key too small" % "Inappropriate fallback" % "No shared cipher" % "Unknown cipher type" % "Unknown exchange type" % "Unknown protocol" % "Unsupported EC" % "Unsupported protocol") << endl; + + size_t counter = 0; + for(const auto& f : g_frontends) { + if (!f->hasTLS()) { + continue; + } + const TLSErrorCounters* errorCounters = nullptr; + if (f->tlsFrontend != nullptr) { + errorCounters = &f->tlsFrontend->d_tlsCounters; + } + else if (f->dohFrontend != nullptr) { + errorCounters = &f->dohFrontend->d_tlsCounters; + } + if (errorCounters == nullptr) { + continue; + } + + ret << (fmt % counter % f->local.toStringWithPort() % errorCounters->d_dhKeyTooSmall % errorCounters->d_inappropriateFallBack % errorCounters->d_noSharedCipher % errorCounters->d_unknownCipherType % errorCounters->d_unknownKeyExchangeType % errorCounters->d_unknownProtocol % errorCounters->d_unsupportedEC % errorCounters->d_unsupportedProtocol) << endl; + ++counter; + } + ret << endl; + + g_outputBuffer=ret.str(); + }); + g_lua.writeFunction("dumpStats", [] { setLuaNoSideEffect(); vector leftcolumn, rightcolumn; diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc index 34437ba48..b8f2deadc 100644 --- a/pdns/dnsdist-web.cc +++ b/pdns/dnsdist-web.cc @@ -563,6 +563,9 @@ static void connectionThread(int sock, ComboAddress remote) output << "# HELP " << frontsbase << "tlsinactiveticketkeys " << "Amount of TLS sessions resumed from an inactive key" << "\n"; output << "# TYPE " << frontsbase << "tlsinactiveticketkeys " << "counter" << "\n"; + output << "# HELP " << frontsbase << "tlshandshakefailures " << "Amount of TLS handshake failures" << "\n"; + output << "# TYPE " << frontsbase << "tlshandshakefailures " << "counter" << "\n"; + std::map frontendDuplicates; for (const auto& front : g_frontends) { if (front->udpFD == -1 && front->tcpFD == -1) @@ -591,16 +594,37 @@ static void connectionThread(int sock, ComboAddress remote) output << frontsbase << "tcpcurrentconnections" << label << front->tcpCurrentConnections.load() << "\n"; output << frontsbase << "tcpavgqueriesperconnection" << label << front->tcpAvgQueriesPerConnection.load() << "\n"; output << frontsbase << "tcpavgconnectionduration" << label << front->tcpAvgConnectionDuration.load() << "\n"; - output << frontsbase << "tlsnewsessions" << label << front->tlsNewSessions.load() << "\n"; - output << frontsbase << "tlsresumptions" << label << front->tlsResumptions.load() << "\n"; - output << frontsbase << "tlsunknownticketkeys" << label << front->tlsUnknownTicketKey.load() << "\n"; - output << frontsbase << "tlsinactiveticketkeys" << label << front->tlsInactiveTicketKey.load() << "\n"; - - output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls10\"} " << front->tls10queries.load() << "\n"; - output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls11\"} " << front->tls11queries.load() << "\n"; - output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls12\"} " << front->tls12queries.load() << "\n"; - output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls13\"} " << front->tls13queries.load() << "\n"; - output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"unknown\"} " << front->tlsUnknownqueries.load() << "\n"; + if (front->hasTLS()) { + output << frontsbase << "tlsnewsessions" << label << front->tlsNewSessions.load() << "\n"; + output << frontsbase << "tlsresumptions" << label << front->tlsResumptions.load() << "\n"; + output << frontsbase << "tlsunknownticketkeys" << label << front->tlsUnknownTicketKey.load() << "\n"; + output << frontsbase << "tlsinactiveticketkeys" << label << front->tlsInactiveTicketKey.load() << "\n"; + + output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls10\"} " << front->tls10queries.load() << "\n"; + output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls11\"} " << front->tls11queries.load() << "\n"; + output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls12\"} " << front->tls12queries.load() << "\n"; + output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls13\"} " << front->tls13queries.load() << "\n"; + output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"unknown\"} " << front->tlsUnknownqueries.load() << "\n"; + + const TLSErrorCounters* errorCounters = nullptr; + if (front->tlsFrontend != nullptr) { + errorCounters = &front->tlsFrontend->d_tlsCounters; + } + else if (front->dohFrontend != nullptr) { + errorCounters = &front->dohFrontend->d_tlsCounters; + } + + if (errorCounters != nullptr) { + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"dhKeyTooSmall\"} " << errorCounters->d_dhKeyTooSmall << "\n"; + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"inappropriateFallBack\"} " << errorCounters->d_inappropriateFallBack << "\n"; + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"noSharedCipher\"} " << errorCounters->d_noSharedCipher << "\n"; + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"unknownCipherType\"} " << errorCounters->d_unknownCipherType << "\n"; + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"unknownKeyExchangeType\"} " << errorCounters->d_unknownKeyExchangeType << "\n"; + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"unknownProtocol\"} " << errorCounters->d_unknownProtocol << "\n"; + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"unsupportedEC\"} " << errorCounters->d_unsupportedEC << "\n"; + output << frontsbase << "tlshandshakefailures{frontend=\"" << frontName << "\",proto=\"" << proto << "\",error=\"unsupportedProtocol{\"} " << errorCounters->d_unsupportedProtocol << "\n"; + } + } } } @@ -805,6 +829,23 @@ static void connectionThread(int sock, ComboAddress remote) { "tls13Queries", (double) front->tls13queries }, { "tlsUnknownQueries", (double) front->tlsUnknownqueries }, }; + const TLSErrorCounters* errorCounters = nullptr; + if (front->tlsFrontend != nullptr) { + errorCounters = &front->tlsFrontend->d_tlsCounters; + } + else if (front->dohFrontend != nullptr) { + errorCounters = &front->dohFrontend->d_tlsCounters; + } + if (errorCounters != nullptr) { + frontend["tlsHandshakeFailuresDHKeyTooSmall"] = (double)errorCounters->d_dhKeyTooSmall; + frontend["tlsHandshakeFailuresInappropriateFallBack"] = (double)errorCounters->d_inappropriateFallBack; + frontend["tlsHandshakeFailuresNoSharedCipher"] = (double)errorCounters->d_noSharedCipher; + frontend["tlsHandshakeFailuresUnknownCipher"] = (double)errorCounters->d_unknownCipherType; + frontend["tlsHandshakeFailuresUnknownKeyExchangeType"] = (double)errorCounters->d_unknownKeyExchangeType; + frontend["tlsHandshakeFailuresUnknownProtocol"] = (double)errorCounters->d_unknownProtocol; + frontend["tlsHandshakeFailuresUnsupportedEC"] = (double)errorCounters->d_unsupportedEC; + frontend["tlsHandshakeFailuresUnsupportedProtocol"] = (double)errorCounters->d_unsupportedProtocol; + } frontends.push_back(frontend); } diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index 3efda3575..b83959c98 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -718,6 +718,11 @@ struct ClientState return udpFD == -1; } + bool hasTLS() const + { + return tlsFrontend != nullptr || dohFrontend != nullptr; + } + std::string getType() const { std::string result = udpFD != -1 ? "UDP" : "TCP"; diff --git a/pdns/dnsdistdist/docs/reference/config.rst b/pdns/dnsdistdist/docs/reference/config.rst index 91c820f42..813aa0809 100644 --- a/pdns/dnsdistdist/docs/reference/config.rst +++ b/pdns/dnsdistdist/docs/reference/config.rst @@ -839,6 +839,12 @@ Status, Statistics and More Print the list of all availables DNS over TLS contexts. +.. function:: showTLSErrorCounters() + + .. versionadded:: 1.4.0 + + Display metrics about TLS handshake failures. + .. function:: showVersion() Print the version of dnsdist diff --git a/pdns/dnsdistdist/doh.cc b/pdns/dnsdistdist/doh.cc index 32063f181..8a95da7e1 100644 --- a/pdns/dnsdistdist/doh.cc +++ b/pdns/dnsdistdist/doh.cc @@ -962,6 +962,8 @@ static std::unique_ptr getTLSContext(DOHFrontend& df SSL_CTX_set_tlsext_status_arg(ctx.get(), &ocspResponses); } + libssl_set_error_counters_callback(ctx, &df.d_tlsCounters); + h2o_ssl_register_alpn_protocols(ctx.get(), h2o_http2_alpn_protocols); if (df.d_tlsConfig.d_ticketKeyFile.empty()) { diff --git a/pdns/dnsdistdist/libssl.cc b/pdns/dnsdistdist/libssl.cc index 1c78e48c3..1cb9b3e2d 100644 --- a/pdns/dnsdistdist/libssl.cc +++ b/pdns/dnsdistdist/libssl.cc @@ -66,6 +66,7 @@ static void openssl_thread_cleanup() static std::atomic s_users; static int s_ticketsKeyIndex{-1}; +static int s_countersIndex{-1}; void registerOpenSSLUser() { @@ -75,6 +76,12 @@ void registerOpenSSLUser() if (s_ticketsKeyIndex == -1) { throw std::runtime_error("Error getting an index for tickets key"); } + + s_countersIndex = SSL_CTX_get_ex_new_index(0, nullptr, nullptr, nullptr, nullptr); + + if (s_countersIndex == -1) { + throw std::runtime_error("Error getting an index for counters"); + } #if (OPENSSL_VERSION_NUMBER < 0x1010000fL || defined LIBRESSL_VERSION_NUMBER) SSL_load_error_strings(); OpenSSL_add_ssl_algorithms(); @@ -147,6 +154,62 @@ int libssl_ticket_key_callback(SSL *s, OpenSSLTLSTicketKeysRing& keyring, unsign return 1; } +static void libssl_info_callback(const SSL *ssl, int where, int ret) +{ + SSL_CTX* sslCtx = SSL_get_SSL_CTX(ssl); + if (sslCtx == nullptr) { + return; + } + + TLSErrorCounters* counters = reinterpret_cast(SSL_CTX_get_ex_data(sslCtx, s_countersIndex)); + if (counters == nullptr) { + return; + } + + if (where & SSL_CB_ALERT) { + const long lastError = ERR_peek_last_error(); + switch (ERR_GET_REASON(lastError)) { +#ifdef SSL_R_DH_KEY_TOO_SMALL + case SSL_R_DH_KEY_TOO_SMALL: + ++counters->d_dhKeyTooSmall; + break; +#endif /* SSL_R_DH_KEY_TOO_SMALL */ + case SSL_R_NO_SHARED_CIPHER: + ++counters->d_noSharedCipher; + break; + case SSL_R_UNKNOWN_PROTOCOL: + ++counters->d_unknownProtocol; + break; + case SSL_R_UNSUPPORTED_PROTOCOL: +#ifdef SSL_R_VERSION_TOO_LOW + case SSL_R_VERSION_TOO_LOW: +#endif /* SSL_R_VERSION_TOO_LOW */ + ++counters->d_unsupportedProtocol; + break; + case SSL_R_INAPPROPRIATE_FALLBACK: + ++counters->d_inappropriateFallBack; + break; + case SSL_R_UNKNOWN_CIPHER_TYPE: + ++counters->d_unknownCipherType; + break; + case SSL_R_UNKNOWN_KEY_EXCHANGE_TYPE: + ++counters->d_unknownKeyExchangeType; + break; + case SSL_R_UNSUPPORTED_ELLIPTIC_CURVE: + ++counters->d_unsupportedEC; + break; + default: + break; + } + } +} + +void libssl_set_error_counters_callback(std::unique_ptr& ctx, TLSErrorCounters* counters) +{ + SSL_CTX_set_ex_data(ctx.get(), s_countersIndex, counters); + SSL_CTX_set_info_callback(ctx.get(), libssl_info_callback); +} + int libssl_ocsp_stapling_callback(SSL* ssl, const std::map& ocspMap) { auto pkey = SSL_get_privatekey(ssl); diff --git a/pdns/dnsdistdist/tcpiohandler.cc b/pdns/dnsdistdist/tcpiohandler.cc index 97931fa75..02b19454b 100644 --- a/pdns/dnsdistdist/tcpiohandler.cc +++ b/pdns/dnsdistdist/tcpiohandler.cc @@ -257,7 +257,7 @@ int OpenSSLTLSConnection::s_tlsConnIndex = -1; class OpenSSLTLSIOCtx: public TLSCtx { public: - OpenSSLTLSIOCtx(const TLSFrontend& fe): d_ticketKeys(fe.d_tlsConfig.d_numberOfTicketsKeys) + OpenSSLTLSIOCtx(TLSFrontend& fe): d_ticketKeys(fe.d_tlsConfig.d_numberOfTicketsKeys) { registerOpenSSLUser(); d_ticketsKeyRotationDelay = fe.d_tlsConfig.d_ticketsKeyRotationDelay; @@ -279,6 +279,8 @@ public: SSL_CTX_set_tlsext_status_arg(d_tlsCtx.get(), &d_ocspResponses); } + libssl_set_error_counters_callback(d_tlsCtx, &fe.d_tlsCounters); + try { if (fe.d_tlsConfig.d_ticketKeyFile.empty()) { handleTicketsKeyRotation(time(nullptr)); @@ -725,7 +727,7 @@ private: class GnuTLSIOCtx: public TLSCtx { public: - GnuTLSIOCtx(const TLSFrontend& fe): d_creds(std::unique_ptr(nullptr, gnutls_certificate_free_credentials)), d_enableTickets(fe.d_tlsConfig.d_enableTickets) + GnuTLSIOCtx(TLSFrontend& fe): d_creds(std::unique_ptr(nullptr, gnutls_certificate_free_credentials)), d_enableTickets(fe.d_tlsConfig.d_enableTickets) { int rc = 0; d_ticketsKeyRotationDelay = fe.d_tlsConfig.d_ticketsKeyRotationDelay; diff --git a/pdns/doh.hh b/pdns/doh.hh index 22d4747dc..a00ab85f2 100644 --- a/pdns/doh.hh +++ b/pdns/doh.hh @@ -48,6 +48,7 @@ struct DOHFrontend std::shared_ptr d_dsc{nullptr}; std::vector> d_responsesMap; TLSConfig d_tlsConfig; + TLSErrorCounters d_tlsCounters; std::string d_serverTokens{"h2o/dnsdist"}; #ifdef HAVE_DNS_OVER_HTTPS std::unique_ptr d_ticketKeys{nullptr}; diff --git a/pdns/libssl.hh b/pdns/libssl.hh index 9276b74e6..2a114b3f0 100644 --- a/pdns/libssl.hh +++ b/pdns/libssl.hh @@ -31,6 +31,19 @@ public: bool d_enableTickets{true}; }; +struct TLSErrorCounters +{ + std::atomic d_dhKeyTooSmall{0}; /* the other side sent a DH value that is not large enough */ + std::atomic d_inappropriateFallBack{0}; /* SCSV indicates that the client previously tried a higher version, + something bad is happening */ + std::atomic d_noSharedCipher{0}; /* we could not agree on a cipher to use */ + std::atomic d_unknownCipherType{0}; /* unknown cipher type */ + std::atomic d_unknownKeyExchangeType{0}; /* * unknown exchange type, weird */ + std::atomic d_unknownProtocol{0}; /* unknown protocol (SSLv2 or TLS 1.4, who knows? */ + std::atomic d_unsupportedEC{0}; /* unsupported elliptic curve */ + std::atomic d_unsupportedProtocol{0}; /* we don't accept this TLS version, sorry */ +}; + #ifdef HAVE_LIBSSL #include @@ -95,6 +108,8 @@ int libssl_get_last_key_type(std::unique_ptr& ctx); bool libssl_generate_ocsp_response(const std::string& certFile, const std::string& caCert, const std::string& caKey, const std::string& outFile, int ndays, int nmin); #endif +void libssl_set_error_counters_callback(std::unique_ptr& ctx, TLSErrorCounters* counters); + LibsslTLSVersion libssl_tls_version_from_string(const std::string& str); const std::string& libssl_tls_version_to_string(LibsslTLSVersion version); bool libssl_set_min_tls_version(std::unique_ptr& ctx, LibsslTLSVersion version); diff --git a/pdns/tcpiohandler.hh b/pdns/tcpiohandler.hh index b8efac21d..5a937368f 100644 --- a/pdns/tcpiohandler.hh +++ b/pdns/tcpiohandler.hh @@ -164,6 +164,7 @@ public: } TLSConfig d_tlsConfig; + TLSErrorCounters d_tlsCounters; ComboAddress d_addr; std::string d_provider; -- 2.40.0