From: Remi Gacogne Date: Thu, 10 Oct 2019 14:57:29 +0000 (+0200) Subject: dnsdist: Add metrics about unknown/inactive TLS ticket keys X-Git-Tag: dnsdist-1.4.0-rc4~30^2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=b608e6c6b2c67e8f7d16b8667a4459cc19bef6dd;p=pdns dnsdist: Add metrics about unknown/inactive TLS ticket keys --- diff --git a/pdns/dnsdist-carbon.cc b/pdns/dnsdist-carbon.cc index 755832815..ad8dfb7cb 100644 --- a/pdns/dnsdist-carbon.cc +++ b/pdns/dnsdist-carbon.cc @@ -130,13 +130,15 @@ try str<tcpCurrentConnections.load() << " " << now << "\r\n"; str<tcpAvgQueriesPerConnection.load() << " " << now << "\r\n"; str<tcpAvgConnectionDuration.load() << " " << now << "\r\n"; - str<tlsNewSessions.load() << " " << now << "\r\n"; - str<tlsResumptions.load() << " " << now << "\r\n"; str<tls10queries.load() << " " << now << "\r\n"; str<tls11queries.load() << " " << now << "\r\n"; str<tls12queries.load() << " " << now << "\r\n"; str<tls13queries.load() << " " << now << "\r\n"; str<tlsUnknownqueries.load() << " " << now << "\r\n"; + str<tlsNewSessions.load() << " " << now << "\r\n"; + str<tlsResumptions.load() << " " << now << "\r\n"; + str<tlsUnknownTicketKey.load() << " " << now << "\r\n"; + str<tlsInactiveTicketKey.load() << " " << now << "\r\n"; } auto localPools = g_pools.getLocal(); diff --git a/pdns/dnsdist-lua-inspection.cc b/pdns/dnsdist-lua-inspection.cc index ebad23400..e1597563d 100644 --- a/pdns/dnsdist-lua-inspection.cc +++ b/pdns/dnsdist-lua-inspection.cc @@ -566,12 +566,12 @@ void setupLuaInspection() ret << endl; ret << "Frontends:" << endl; - fmt = boost::format("%-3d %-20.20s %-20d %-20d %-25d %-20d %-20d %-20d %-20f %-20f %-15d %-15d %-15d %-15d %-15d %-15d %-15d"); - ret << (fmt % "#" % "Address" % "Connections" % "Died reading query" % "Died sending response" % "Gave up" % "Client timeouts" % "Downstream timeouts" % "Avg queries/conn" % "Avg duration" % "TLS new sessions" % "TLS Resumptions" % "TLS 1.0" % "TLS 1.1" % "TLS 1.2" % "TLS 1.3" % "TLS other") << endl; + fmt = boost::format("%-3d %-20.20s %-20d %-20d %-25d %-20d %-20d %-20d %-20f %-20f %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d"); + ret << (fmt % "#" % "Address" % "Connections" % "Died reading query" % "Died sending response" % "Gave up" % "Client timeouts" % "Downstream timeouts" % "Avg queries/conn" % "Avg duration" % "TLS new sessions" % "TLS Resumptions" % "TLS unknown ticket keys" % "TLS inactive ticket keys" % "TLS 1.0" % "TLS 1.1" % "TLS 1.2" % "TLS 1.3" % "TLS other") << endl; size_t counter = 0; for(const auto& f : g_frontends) { - ret << (fmt % counter % f->local.toStringWithPort() % f->tcpCurrentConnections % f->tcpDiedReadingQuery % f->tcpDiedSendingResponse % f->tcpGaveUp % f->tcpClientTimeouts % f->tcpDownstreamTimeouts % f->tcpAvgQueriesPerConnection % f->tcpAvgConnectionDuration % f->tlsNewSessions % f->tlsResumptions % f->tls10queries % f->tls11queries % f->tls12queries % f->tls13queries % f->tlsUnknownqueries) << endl; + ret << (fmt % counter % f->local.toStringWithPort() % f->tcpCurrentConnections % f->tcpDiedReadingQuery % f->tcpDiedSendingResponse % f->tcpGaveUp % f->tcpClientTimeouts % f->tcpDownstreamTimeouts % f->tcpAvgQueriesPerConnection % f->tcpAvgConnectionDuration % f->tlsNewSessions % f->tlsResumptions % f->tlsUnknownTicketKey % f->tlsInactiveTicketKey % f->tls10queries % f->tls11queries % f->tls12queries % f->tls13queries % f->tlsUnknownqueries) << endl; ++counter; } ret << endl; diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index ca1f6a31e..6e6419fb4 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -1807,11 +1807,11 @@ void setupLuaConfig(bool client) setLuaNoSideEffect(); try { ostringstream ret; - boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d"); - ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid") << endl; + boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d"); + ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid" % "# ticket keys" % "Rotation delay" % "Next rotation") << endl; size_t counter = 0; for (const auto& ctx : g_dohlocals) { - ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http1Stats.d_nbQueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses) << endl; + ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http1Stats.d_nbQueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses % ctx->getTicketsKeysCount() % ctx->d_ticketsKeyRotationDelay % ctx->getNextTicketsKeyRotation()) << endl; counter++; } g_outputBuffer = ret.str(); diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc index 24e3184c5..a2902ad66 100644 --- a/pdns/dnsdist-tcp.cc +++ b/pdns/dnsdist-tcp.cc @@ -1116,6 +1116,12 @@ static void handleIO(std::shared_ptr& state, struct else { ++state->d_ci.cs->tlsResumptions; } + if (state->d_handler.getResumedFromInactiveTicketKey()) { + ++state->d_ci.cs->tlsInactiveTicketKey; + } + if (state->d_handler.getUnknownTicketKey()) { + ++state->d_ci.cs->tlsUnknownTicketKey; + } } state->d_handshakeDoneTime = now; diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc index 6ce7e239f..1ea2e214c 100644 --- a/pdns/dnsdist-web.cc +++ b/pdns/dnsdist-web.cc @@ -552,12 +552,16 @@ static void connectionThread(int sock, ComboAddress remote) output << "# TYPE " << frontsbase << "tcpavgqueriesperconnection " << "gauge" << "\n"; output << "# HELP " << frontsbase << "tcpavgconnectionduration " << "The average duration of a TCP connection (ms)" << "\n"; output << "# TYPE " << frontsbase << "tcpavgconnectionduration " << "gauge" << "\n"; + output << "# HELP " << frontsbase << "tlsqueries " << "Number of queries received by dnsdist over TLS, by TLS version" << "\n"; + output << "# TYPE " << frontsbase << "tlsqueries " << "counter" << "\n"; output << "# HELP " << frontsbase << "tlsnewsessions " << "Amount of new TLS sessions negotiated" << "\n"; output << "# TYPE " << frontsbase << "tlsnewsessions " << "counter" << "\n"; output << "# HELP " << frontsbase << "tlsresumptions " << "Amount of TLS sessions resumed" << "\n"; output << "# TYPE " << frontsbase << "tlsresumptions " << "counter" << "\n"; - output << "# HELP " << frontsbase << "tlsqueries " << "Number of queries received by dnsdist over TLS, by TLS version" << "\n"; - output << "# TYPE " << frontsbase << "tlsqueries " << "counter" << "\n"; + output << "# HELP " << frontsbase << "tlsunknownticketkeys " << "Amount of attempts to resume TLS session from an unknown key (possibly expired)" << "\n"; + output << "# TYPE " << frontsbase << "tlsunknownticketkeys " << "counter" << "\n"; + output << "# HELP " << frontsbase << "tlsinactiveticketkeys " << "Amount of TLS sessions resumed from an inactive key" << "\n"; + output << "# TYPE " << frontsbase << "tlsinactiveticketkeys " << "counter" << "\n"; std::map frontendDuplicates; for (const auto& front : g_frontends) { @@ -589,6 +593,8 @@ static void connectionThread(int sock, ComboAddress remote) output << frontsbase << "tcpavgconnectionduration" << label << front->tcpAvgConnectionDuration.load() << "\n"; output << frontsbase << "tlsnewsessions" << label << front->tlsNewSessions.load() << "\n"; output << frontsbase << "tlsresumptions" << label << front->tlsResumptions.load() << "\n"; + output << frontsbase << "tlsunknownticketkeys" << label << front->tlsUnknownTicketKey.load() << "\n"; + output << frontsbase << "tlsinactiveticketkeys" << label << front->tlsInactiveTicketKey.load() << "\n"; output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls10\"} " << front->tls10queries.load() << "\n"; output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls11\"} " << front->tls11queries.load() << "\n"; @@ -772,6 +778,8 @@ static void connectionThread(int sock, ComboAddress remote) { "tcpAvgConnectionDuration", (double) front->tcpAvgConnectionDuration }, { "tlsNewSessions", (double) front->tlsNewSessions }, { "tlsResumptions", (double) front->tlsResumptions }, + { "tlsUnknownTicketKey", (double) front->tlsUnknownTicketKey }, + { "tlsInactiveTicketKey", (double) front->tlsInactiveTicketKey }, { "tls10Queries", (double) front->tls10queries }, { "tls11Queries", (double) front->tls11queries }, { "tls12Queries", (double) front->tls12queries }, diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index b28cf4440..e6094695c 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -685,12 +685,13 @@ struct ClientState std::atomic tcpCurrentConnections{0}; std::atomic tlsNewSessions{0}; // A new TLS session has been negotiated, no resumption std::atomic tlsResumptions{0}; // A TLS session has been resumed, either via session id or via a TLS ticket + std::atomic tlsUnknownTicketKey{0}; // A TLS ticket has been presented but we don't have the associated key (might have expired) + std::atomic tlsInactiveTicketKey{0}; // A TLS ticket has been successfully resumed but the key is no longer active, we should issue a new one std::atomic tls10queries{0}; // valid DNS queries received via TLSv1.0 std::atomic tls11queries{0}; // valid DNS queries received via TLSv1.1 std::atomic tls12queries{0}; // valid DNS queries received via TLSv1.2 std::atomic tls13queries{0}; // valid DNS queries received via TLSv1.3 std::atomic tlsUnknownqueries{0}; // valid DNS queries received via unknown TLS version - std::atomic tcpAvgQueriesPerConnection{0.0}; /* in ms */ std::atomic tcpAvgConnectionDuration{0.0}; diff --git a/pdns/dnsdistdist/doh.cc b/pdns/dnsdistdist/doh.cc index 8973bf4d1..f93abdd82 100644 --- a/pdns/dnsdistdist/doh.cc +++ b/pdns/dnsdistdist/doh.cc @@ -923,7 +923,17 @@ static int ticket_key_callback(SSL *s, unsigned char keyName[TLS_TICKETS_KEY_NAM df->handleTicketsKeyRotation(); - return libssl_ticket_key_callback(s, *df->d_ticketKeys, keyName, iv, ectx, hctx, enc); + auto ret = libssl_ticket_key_callback(s, *df->d_ticketKeys, keyName, iv, ectx, hctx, enc); + if (enc == 0) { + if (ret == 0) { + ++df->d_dsc->cs->tlsUnknownTicketKey; + } + else if (ret == 2) { + ++df->d_dsc->cs->tlsInactiveTicketKey; + } + } + + return ret; } static std::unique_ptr getTLSContext(DOHFrontend& df, diff --git a/pdns/dnsdistdist/tcpiohandler.cc b/pdns/dnsdistdist/tcpiohandler.cc index dd98fc217..dcf852099 100644 --- a/pdns/dnsdistdist/tcpiohandler.cc +++ b/pdns/dnsdistdist/tcpiohandler.cc @@ -25,6 +25,14 @@ public: { d_socket = socket; + if (!s_initTLSConnIndex.test_and_set()) { + /* not initialized yet */ + s_tlsConnIndex = SSL_get_ex_new_index(0, nullptr, nullptr, nullptr, nullptr); + if (s_tlsConnIndex == -1) { + throw std::runtime_error("Error getting an index for TLS connection data"); + } + } + if (!d_conn) { vinfolog("Error creating TLS object"); if (g_verbose) { @@ -36,6 +44,8 @@ public: if (!SSL_set_fd(d_conn.get(), d_socket)) { throw std::runtime_error("Error assigning socket"); } + + SSL_set_ex_data(d_conn.get(), s_tlsConnIndex, this); } IOState convertIORequestToIOState(int res) const @@ -232,12 +242,18 @@ public: return false; } + static int s_tlsConnIndex; private: + static std::atomic_flag s_initTLSConnIndex; + std::unique_ptr d_conn; unsigned int d_timeout; }; +std::atomic_flag OpenSSLTLSConnection::s_initTLSConnIndex = ATOMIC_FLAG_INIT; +int OpenSSLTLSConnection::s_tlsConnIndex = -1; + class OpenSSLTLSIOCtx: public TLSCtx { public: @@ -376,7 +392,22 @@ public: return -1; } - return libssl_ticket_key_callback(s, ctx->d_ticketKeys, keyName, iv, ectx, hctx, enc); + int ret = libssl_ticket_key_callback(s, ctx->d_ticketKeys, keyName, iv, ectx, hctx, enc); + if (enc == 0) { + if (ret == 0 || ret == 2) { + OpenSSLTLSConnection* conn = reinterpret_cast(SSL_get_ex_data(s, OpenSSLTLSConnection::s_tlsConnIndex)); + if (conn) { + if (ret == 0) { + conn->setUnknownTicketKey(); + } + else if (ret == 2) { + conn->setResumedFromInactiveTicketKey(); + } + } + } + } + + return ret; } static int ocspStaplingCb(SSL* ssl, void* arg) diff --git a/pdns/doh.hh b/pdns/doh.hh index 289b88361..6fc3d2b2c 100644 --- a/pdns/doh.hh +++ b/pdns/doh.hh @@ -123,6 +123,22 @@ struct DOHFrontend #endif /* HAVE_DNS_OVER_HTTPS */ + time_t getNextTicketsKeyRotation() const + { + return d_ticketsKeyNextRotation; + } + + size_t getTicketsKeysCount() const + { + size_t res = 0; +#ifdef HAVE_DNS_OVER_HTTPS + if (d_ticketKeys) { + res = d_ticketKeys->getKeysCount(); + } +#endif /* HAVE_DNS_OVER_HTTPS */ + return res; + } + private: time_t d_ticketsKeyNextRotation{0}; std::atomic_flag d_rotatingTicketsKey; diff --git a/pdns/tcpiohandler.hh b/pdns/tcpiohandler.hh index c5735729a..76ac5330a 100644 --- a/pdns/tcpiohandler.hh +++ b/pdns/tcpiohandler.hh @@ -22,8 +22,30 @@ public: virtual bool hasSessionBeenResumed() const = 0; virtual void close() = 0; + void setUnknownTicketKey() + { + d_unknownTicketKey = true; + } + + bool getUnknownTicketKey() const + { + return d_unknownTicketKey; + } + + void setResumedFromInactiveTicketKey() + { + d_resumedFromInactiveTicketKey = true; + } + + bool getResumedFromInactiveTicketKey() const + { + return d_resumedFromInactiveTicketKey; + } + protected: int d_socket{-1}; + bool d_unknownTicketKey{false}; + bool d_resumedFromInactiveTicketKey{false}; }; class TLSCtx @@ -308,6 +330,16 @@ public: return d_conn && d_conn->hasSessionBeenResumed(); } + bool getResumedFromInactiveTicketKey() const + { + return d_conn && d_conn->getResumedFromInactiveTicketKey(); + } + + bool getUnknownTicketKey() const + { + return d_conn && d_conn->getUnknownTicketKey(); + } + private: std::unique_ptr d_conn{nullptr}; int d_socket{-1};