From: Remi Gacogne Date: Mon, 23 Sep 2019 14:34:21 +0000 (+0200) Subject: dnsdist: Better use of labels in our DoH prometheus export X-Git-Tag: dnsdist-1.4.0-rc3^2~3 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=846b63bb255e0c2222cab32df1f4fb1ba8425254;p=pdns dnsdist: Better use of labels in our DoH prometheus export Also add stats about TLS session resumption. --- diff --git a/pdns/dnsdist-carbon.cc b/pdns/dnsdist-carbon.cc index a40e1dff6..8e9ed5b65 100644 --- a/pdns/dnsdist-carbon.cc +++ b/pdns/dnsdist-carbon.cc @@ -128,6 +128,8 @@ try str<tcpCurrentConnections.load() << " " << now << "\r\n"; str<tcpAvgQueriesPerConnection.load() << " " << now << "\r\n"; str<tcpAvgConnectionDuration.load() << " " << now << "\r\n"; + str<tlsNewSessions.load() << " " << now << "\r\n"; + str<tlsResumptions.load() << " " << now << "\r\n"; } auto localPools = g_pools.getLocal(); diff --git a/pdns/dnsdist-lua-inspection.cc b/pdns/dnsdist-lua-inspection.cc index bcae7276a..1a3cd227f 100644 --- a/pdns/dnsdist-lua-inspection.cc +++ b/pdns/dnsdist-lua-inspection.cc @@ -566,12 +566,12 @@ void setupLuaInspection() ret << endl; ret << "Frontends:" << endl; - fmt = boost::format("%-3d %-20.20s %-20d %-20d %-25d %-20d %-20d %-20d %-20f %-20f"); - ret << (fmt % "#" % "Address" % "Connections" % "Died reading query" % "Died sending response" % "Gave up" % "Client timeouts" % "Downstream timeouts" % "Avg queries/conn" % "Avg duration") << endl; + fmt = boost::format("%-3d %-20.20s %-20d %-20d %-25d %-20d %-20d %-20d %-20f %-20f %-15d %-15d"); + ret << (fmt % "#" % "Address" % "Connections" % "Died reading query" % "Died sending response" % "Gave up" % "Client timeouts" % "Downstream timeouts" % "Avg queries/conn" % "Avg duration" % "TLS new sessions" % "TLS Resumptions") << endl; size_t counter = 0; for(const auto& f : g_frontends) { - ret << (fmt % counter % f->local.toStringWithPort() % f->tcpCurrentConnections % f->tcpDiedReadingQuery % f->tcpDiedSendingResponse % f->tcpGaveUp % f->tcpClientTimeouts % f->tcpDownstreamTimeouts % f->tcpAvgQueriesPerConnection % f->tcpAvgConnectionDuration) << endl; + ret << (fmt % counter % f->local.toStringWithPort() % f->tcpCurrentConnections % f->tcpDiedReadingQuery % f->tcpDiedSendingResponse % f->tcpGaveUp % f->tcpClientTimeouts % f->tcpDownstreamTimeouts % f->tcpAvgQueriesPerConnection % f->tcpAvgConnectionDuration % f->tlsNewSessions % f->tlsResumptions) << endl; ++counter; } ret << endl; diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index c5de7973a..0d48ed0a2 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -1764,7 +1764,7 @@ void setupLuaConfig(bool client) ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "TLS 1.0" % "TLS 1.1" % "TLS 1.2" % "TLS 1.3" % "TLS other" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid") << endl; size_t counter = 0; for (const auto& ctx : g_dohlocals) { - ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http1Stats.d_nbQueries % ctx->d_tls10queries % ctx->d_tls11queries % ctx->d_tls12queries % ctx->d_tls13queries % ctx->d_tlsUnknownqueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses %ctx->d_validresponses) << endl; + ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http1Stats.d_nbQueries % ctx->d_tls10queries % ctx->d_tls11queries % ctx->d_tls12queries % ctx->d_tls13queries % ctx->d_tlsUnknownqueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses) << endl; counter++; } g_outputBuffer = ret.str(); diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc index bc1f54ec8..edd006b2a 100644 --- a/pdns/dnsdist-tcp.cc +++ b/pdns/dnsdist-tcp.cc @@ -1067,6 +1067,15 @@ static void handleIO(std::shared_ptr& state, struct if (state->d_state == IncomingTCPConnectionState::State::doingHandshake) { iostate = state->d_handler.tryHandshake(); if (iostate == IOState::Done) { + if (state->d_handler.isTLS()) { + if (!state->d_handler.hasTLSSessionBeenResumed()) { + ++state->d_ci.cs->tlsNewSessions; + } + else { + ++state->d_ci.cs->tlsResumptions; + } + } + state->d_handshakeDoneTime = now; state->d_state = IncomingTCPConnectionState::State::readingQuerySize; } diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc index 97f4f5d41..2ed1ab7f0 100644 --- a/pdns/dnsdist-web.cc +++ b/pdns/dnsdist-web.cc @@ -542,6 +542,10 @@ static void connectionThread(int sock, ComboAddress remote) output << "# TYPE " << frontsbase << "tcpavgqueriesperconnection " << "gauge" << "\n"; output << "# HELP " << frontsbase << "tcpavgconnectionduration " << "The average duration of a TCP connection (ms)" << "\n"; output << "# TYPE " << frontsbase << "tcpavgconnectionduration " << "gauge" << "\n"; + output << "# HELP " << frontsbase << "tlsnewsessions " << "Amount of new TLS sessions negotiated" << "\n"; + output << "# TYPE " << frontsbase << "tlsnewsessions " << "counter" << "\n"; + output << "# HELP " << frontsbase << "tlsresumptions " << "Amount of TLS sessions resumed" << "\n"; + output << "# TYPE " << frontsbase << "tlsresumptions " << "counter" << "\n"; std::map frontendDuplicates; for (const auto& front : g_frontends) { @@ -568,93 +572,50 @@ static void connectionThread(int sock, ComboAddress remote) output << frontsbase << "tcpcurrentconnections" << label << front->tcpCurrentConnections.load() << "\n"; output << frontsbase << "tcpavgqueriesperconnection" << label << front->tcpAvgQueriesPerConnection.load() << "\n"; output << frontsbase << "tcpavgconnectionduration" << label << front->tcpAvgConnectionDuration.load() << "\n"; + output << frontsbase << "tlsnewsessions" << label << front->tlsNewSessions.load() << "\n"; + output << frontsbase << "tlsresumptions" << label << front->tlsResumptions.load() << "\n"; } - const string dohfrontsbase = "dnsdist_doh_frontend_"; - output << "# HELP " << dohfrontsbase << "http_connects " << "Number of TCP connections establoshed to this frontend" << "\n"; - output << "# TYPE " << dohfrontsbase << "queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "tls10_queries " << "Number of valid DNS queries received via TLS 1.0" << "\n"; - output << "# TYPE " << dohfrontsbase << "tls10_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "tls11_queries " << "Number of valid DNS queries received via TLS 1.1" << "\n"; - output << "# TYPE " << dohfrontsbase << "tls11_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "tls12_queries " << "Number of valid DNS queries received via TLS 1.2" << "\n"; - output << "# TYPE " << dohfrontsbase << "tls12_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "tls13_queries " << "Number of valid DNS queries received via TLS 1.3" << "\n"; - output << "# TYPE " << dohfrontsbase << "tls13_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "tlsunknown_queries " << "Number of valid DNS queries received via an unknown TLS version" << "\n"; - output << "# TYPE " << dohfrontsbase << "tlsunknown_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "get_queries " << "Number of valid DNS queries received via GET" << "\n"; - output << "# TYPE " << dohfrontsbase << "get_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "post_queries " << "Number of valid DNS queries received via POST" << "\n"; - output << "# TYPE " << dohfrontsbase << "post_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "bad_requests " << "Number of requests that could not be converted to a DNS query" << "\n"; - output << "# TYPE " << dohfrontsbase << "bad_requests " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "error_responses " << "Number of responses sent by dnsdist indicating an error" << "\n"; - output << "# TYPE " << dohfrontsbase << "error_responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "redirect_responses " << "Number of responses sent by dnsdist indicating a redirect" << "\n"; - output << "# TYPE " << dohfrontsbase << "redirect_responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "valid_responses " << "Number of valid responses sent by dnsdist" << "\n"; - output << "# TYPE " << dohfrontsbase << "valid_responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http1_queries " << "Number of queries received over HTTP/1.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http1_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http1_nb200responses " << "Number of responses with a 200 status code sent over HTTP/1.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http1_nb200responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http1_nb400responses " << "Number of responses with a 400 status code sent over HTTP/1.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http1_nb400responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http1_nb403responses " << "Number of responses with a 403 status code sent over HTTP/1.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http1_nb403responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http1_nb500responses " << "Number of responses with a 500 status code sent over HTTP/1.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http1_nb500responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http1_nb502responses " << "Number of responses with a 502 status code sent over HTTP/1.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http1_nb502responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http1_nbotherresponses " << "Number of responses with another status code sent over HTTP/1.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http1_nbotherresponses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http2_queries " << "Number of queries received over HTTP/2.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http2_queries " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http2_nb200responses " << "Number of responses with a 200 status code sent over HTTP/2.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http2_nb200responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http2_nb400responses " << "Number of responses with a 400 status code sent over HTTP/2.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http2_nb400responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http2_nb403responses " << "Number of responses with a 403 status code sent over HTTP/2.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http2_nb403responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http2_nb500responses " << "Number of responses with a 500 status code sent over HTTP/2.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http2_nb500responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http2_nb502responses " << "Number of responses with a 502 status code sent over HTTP/2.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http2_nb502responses " << "counter" << "\n"; - output << "# HELP " << dohfrontsbase << "http2_nbotherresponses " << "Number of responses with another status code sent over HTTP/2.x" << "\n"; - output << "# TYPE " << dohfrontsbase << "http2_nbotherresponses " << "counter" << "\n"; + output << "# HELP " << frontsbase << "http_connects " << "Number of DoH TCP connections established to this frontend" << "\n"; + output << "# TYPE " << frontsbase << "http_connects " << "counter" << "\n"; + output << "# HELP " << frontsbase << "doh_responses " << "Number of DoH responses sent by dnsdist" << "\n"; + output << "# TYPE " << frontsbase << "doh_responses " << "counter" << "\n"; #ifdef HAVE_DNS_OVER_HTTPS for(const auto& doh : g_dohlocals) { - const std::string label = boost::str(boost::format("{address=\"%1%\"} ") % doh->d_local.toStringWithPort()); - - output << dohfrontsbase << "http_connects" << label << doh->d_httpconnects << "\n"; - output << dohfrontsbase << "tls10_queries" << label << doh->d_tls10queries << "\n"; - output << dohfrontsbase << "tls11_queries" << label << doh->d_tls11queries << "\n"; - output << dohfrontsbase << "tls12_queries" << label << doh->d_tls12queries << "\n"; - output << dohfrontsbase << "tls13_queries" << label << doh->d_tls13queries << "\n"; - output << dohfrontsbase << "tlsunknown_queries" << label << doh->d_tlsUnknownqueries << "\n"; - output << dohfrontsbase << "get_queries" << label << doh->d_getqueries << "\n"; - output << dohfrontsbase << "post_queries" << label << doh->d_postqueries << "\n"; - output << dohfrontsbase << "bad_requests" << label << doh->d_badrequests << "\n"; - output << dohfrontsbase << "error_responses" << label << doh->d_errorresponses << "\n"; - output << dohfrontsbase << "redirect_responses" << label << doh->d_redirectresponses << "\n"; - output << dohfrontsbase << "valid_responses" << label << doh->d_validresponses << "\n"; - - output << dohfrontsbase << "http1_queries" << label << doh->d_http1Stats.d_nbQueries << "\n"; - output << dohfrontsbase << "http1_nb200responses" << label << doh->d_http1Stats.d_nb200Responses << "\n"; - output << dohfrontsbase << "http1_nb400responses" << label << doh->d_http1Stats.d_nb400Responses << "\n"; - output << dohfrontsbase << "http1_nb403responses" << label << doh->d_http1Stats.d_nb403Responses << "\n"; - output << dohfrontsbase << "http1_nb500responses" << label << doh->d_http1Stats.d_nb500Responses << "\n"; - output << dohfrontsbase << "http1_nb502responses" << label << doh->d_http1Stats.d_nb502Responses << "\n"; - output << dohfrontsbase << "http1_nbotherresponses" << label << doh->d_http1Stats.d_nbOtherResponses << "\n"; - output << dohfrontsbase << "http2_queries" << label << doh->d_http2Stats.d_nbQueries << "\n"; - output << dohfrontsbase << "http2_nb200responses" << label << doh->d_http2Stats.d_nb200Responses << "\n"; - output << dohfrontsbase << "http2_nb400responses" << label << doh->d_http2Stats.d_nb400Responses << "\n"; - output << dohfrontsbase << "http2_nb403responses" << label << doh->d_http2Stats.d_nb403Responses << "\n"; - output << dohfrontsbase << "http2_nb500responses" << label << doh->d_http2Stats.d_nb500Responses << "\n"; - output << dohfrontsbase << "http2_nb502responses" << label << doh->d_http2Stats.d_nb502Responses << "\n"; - output << dohfrontsbase << "http2_nbotherresponses" << label << doh->d_http2Stats.d_nbOtherResponses << "\n"; + const std::string addrlabel = boost::str(boost::format("address=\"%1%\" ") % doh->d_local.toStringWithPort()); + const std::string label = "{" + addrlabel + "} "; + + output << frontsbase << "http_connects" << label << doh->d_httpconnects << "\n"; + output << frontsbase << "queries{tls=\"tls10\"," << addrlabel << "} " << doh->d_tls10queries << "\n"; + output << frontsbase << "queries{tls=\"tls11\"," << addrlabel << "} " << doh->d_tls11queries << "\n"; + output << frontsbase << "queries{tls=\"tls12\"," << addrlabel << "} " << doh->d_tls12queries << "\n"; + output << frontsbase << "queries{tls=\"tls13\"," << addrlabel << "} " << doh->d_tls13queries << "\n"; + output << frontsbase << "queries{tls=\"unknown\"," << addrlabel << "} " << doh->d_tlsUnknownqueries << "\n"; + output << frontsbase << "queries{httpmethod=\"get\"," << addrlabel << "} " << doh->d_getqueries << "\n"; + output << frontsbase << "queries{httpmethod=\"post\"," << addrlabel << "} " << doh->d_postqueries << "\n"; + output << frontsbase << "queries{httpversion=\"1\"," << addrlabel << "} " << doh->d_http1Stats.d_nbQueries << "\n"; + output << frontsbase << "queries{httpversion=\"2\"," << addrlabel << "} " << doh->d_http2Stats.d_nbQueries << "\n"; + + output << frontsbase << "queries{type=\"bad\"," << addrlabel << "} " << doh->d_badrequests << "\n"; + + output << frontsbase << "doh_responses{type=\"error\"," << addrlabel << "} " << doh->d_errorresponses << "\n"; + output << frontsbase << "doh_responses{type=\"redirect\"," << addrlabel << "} " << doh->d_redirectresponses << "\n"; + output << frontsbase << "doh_responses{type=\"valid\"," << addrlabel << "} " << doh->d_validresponses << "\n"; + + output << frontsbase << "doh_responses{httpversion=\"1\",status=\"200\"," << addrlabel << "} " << doh->d_http1Stats.d_nb200Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"1\",status=\"400\"," << addrlabel << "} " << doh->d_http1Stats.d_nb400Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"1\",status=\"403\"," << addrlabel << "} " << doh->d_http1Stats.d_nb403Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"1\",status=\"500\"," << addrlabel << "} " << doh->d_http1Stats.d_nb500Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"1\",status=\"502\"," << addrlabel << "} " << doh->d_http1Stats.d_nb502Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"1\",status=\"other\"," << addrlabel << "} " << doh->d_http1Stats.d_nbOtherResponses << "\n"; + + output << frontsbase << "doh_responses{httpversion=\"2\",status=\"200\"," << addrlabel << "} " << doh->d_http2Stats.d_nb200Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"2\",status=\"400\"," << addrlabel << "} " << doh->d_http2Stats.d_nb400Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"2\",status=\"403\"," << addrlabel << "} " << doh->d_http2Stats.d_nb403Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"2\",status=\"500\"," << addrlabel << "} " << doh->d_http2Stats.d_nb500Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"2\",status=\"502\"," << addrlabel << "} " << doh->d_http2Stats.d_nb502Responses << "\n"; + output << frontsbase << "doh_responses{httpversion=\"2\",status=\"other\"," << addrlabel << "} " << doh->d_http2Stats.d_nbOtherResponses << "\n"; } #endif /* HAVE_DNS_OVER_HTTPS */ @@ -765,6 +726,8 @@ static void connectionThread(int sock, ComboAddress remote) { "tcpCurrentConnections", (double) front->tcpCurrentConnections }, { "tcpAvgQueriesPerConnection", (double) front->tcpAvgQueriesPerConnection }, { "tcpAvgConnectionDuration", (double) front->tcpAvgConnectionDuration }, + { "tlsNewSessions", (double) front->tlsNewSessions }, + { "tlsResumptions", (double) front->tlsResumptions }, }; frontends.push_back(frontend); } diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index 620cf1d55..f9489349d 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -685,6 +685,9 @@ struct ClientState std::atomic tcpClientTimeouts{0}; std::atomic tcpDownstreamTimeouts{0}; std::atomic tcpCurrentConnections{0}; + std::atomic tlsNewSessions{0}; // A new TLS session has been negotiated, no resumption + std::atomic tlsResumptions{0}; // A TLS session has been resumed, either via session id or via a TLS ticket + std::atomic tcpAvgQueriesPerConnection{0.0}; /* in ms */ std::atomic tcpAvgConnectionDuration{0.0}; diff --git a/pdns/dnsdistdist/doh.cc b/pdns/dnsdistdist/doh.cc index 24ded5167..8b29580ff 100644 --- a/pdns/dnsdistdist/doh.cc +++ b/pdns/dnsdistdist/doh.cc @@ -561,6 +561,13 @@ try return 0; } + if (h2o_socket_get_ssl_session_reused(sock) == 0) { + ++dsc->cs->tlsNewSessions; + } + else { + ++dsc->cs->tlsResumptions; + } + if(auto tlsversion = h2o_socket_get_ssl_protocol_version(sock)) { if(!strcmp(tlsversion, "TLSv1.0")) ++dsc->df->d_tls10queries; diff --git a/pdns/dnsdistdist/tcpiohandler.cc b/pdns/dnsdistdist/tcpiohandler.cc index 835b3b395..6156da984 100644 --- a/pdns/dnsdistdist/tcpiohandler.cc +++ b/pdns/dnsdistdist/tcpiohandler.cc @@ -364,6 +364,15 @@ public: return std::string(); } + bool hasSessionBeenResumed() const override + { + if (d_conn) { + return SSL_session_reused(d_conn.get()) != 0; + } + return false; + } + + private: std::unique_ptr d_conn; unsigned int d_timeout; @@ -920,6 +929,14 @@ public: return std::string(); } + bool hasSessionBeenResumed() const override + { + if (d_conn) { + return gnutls_session_is_resumed(d_conn.get()) != 0; + } + return false; + } + void close() override { if (d_conn) { diff --git a/pdns/tcpiohandler.hh b/pdns/tcpiohandler.hh index 5356e5905..a1dad5340 100644 --- a/pdns/tcpiohandler.hh +++ b/pdns/tcpiohandler.hh @@ -18,6 +18,7 @@ public: virtual IOState tryWrite(std::vector& buffer, size_t& pos, size_t toWrite) = 0; virtual IOState tryRead(std::vector& buffer, size_t& pos, size_t toRead) = 0; virtual std::string getServerNameIndication() = 0; + virtual bool hasSessionBeenResumed() const = 0; virtual void close() = 0; protected: @@ -287,6 +288,16 @@ public: return std::string(); } + bool isTLS() const + { + return d_conn != nullptr; + } + + bool hasTLSSessionBeenResumed() const + { + return d_conn && d_conn->hasSessionBeenResumed(); + } + private: std::unique_ptr d_conn{nullptr}; int d_socket{-1};