dnsdist: Add metrics about unknown/inactive TLS ticket keys

diff --git a/pdns/dnsdist-carbon.cc b/pdns/dnsdist-carbon.cc
index 755832815b929a4dac3f1bf416b1993f59f73dfe..ad8dfb7cb14373c4541e93bbc23cb7ae23033c03 100644 (file)
--- a/pdns/dnsdist-carbon.cc
+++ b/pdns/dnsdist-carbon.cc
@@ -130,13 +130,15 @@ try
           str<<base<<"tcpcurrentconnections" << ' '<< front->tcpCurrentConnections.load() << " " << now << "\r\n";
           str<<base<<"tcpavgqueriesperconnection" << ' '<< front->tcpAvgQueriesPerConnection.load() << " " << now << "\r\n";
           str<<base<<"tcpavgconnectionduration" << ' '<< front->tcpAvgConnectionDuration.load() << " " << now << "\r\n";
-          str<<base<<"tlsnewsessions" << ' ' << front->tlsNewSessions.load() << " " << now << "\r\n";
-          str<<base<<"tlsresumptions" << ' ' << front->tlsResumptions.load() << " " << now << "\r\n";
           str<<base<<"tls10-queries" << ' ' << front->tls10queries.load() << " " << now << "\r\n";
           str<<base<<"tls11-queries" << ' ' << front->tls11queries.load() << " " << now << "\r\n";
           str<<base<<"tls12-queries" << ' ' << front->tls12queries.load() << " " << now << "\r\n";
           str<<base<<"tls13-queries" << ' ' << front->tls13queries.load() << " " << now << "\r\n";
           str<<base<<"tls-unknown-queries" << ' ' << front->tlsUnknownqueries.load() << " " << now << "\r\n";
+          str<<base<<"tlsnewsessions" << ' ' << front->tlsNewSessions.load() << " " << now << "\r\n";
+          str<<base<<"tlsresumptions" << ' ' << front->tlsResumptions.load() << " " << now << "\r\n";
+          str<<base<<"tlsunknownticketkeys" << ' ' << front->tlsUnknownTicketKey.load() << " " << now << "\r\n";
+          str<<base<<"tlsinactiveticketkeys" << ' ' << front->tlsInactiveTicketKey.load() << " " << now << "\r\n";
         }
 
         auto localPools = g_pools.getLocal();

diff --git a/pdns/dnsdist-lua-inspection.cc b/pdns/dnsdist-lua-inspection.cc
index ebad23400bce8fd1569dcb74e82e53b44b651bc0..e1597563d364c8a50f4d77fc225bbf4d8562dba1 100644 (file)
--- a/pdns/dnsdist-lua-inspection.cc
+++ b/pdns/dnsdist-lua-inspection.cc
@@ -566,12 +566,12 @@ void setupLuaInspection()
       ret << endl;
 
       ret << "Frontends:" << endl;
-      fmt = boost::format("%-3d %-20.20s %-20d %-20d %-25d %-20d %-20d %-20d %-20f %-20f %-15d %-15d %-15d %-15d %-15d %-15d %-15d");
-      ret << (fmt % "#" % "Address" % "Connections" % "Died reading query" % "Died sending response" % "Gave up" % "Client timeouts" % "Downstream timeouts" % "Avg queries/conn" % "Avg duration" % "TLS new sessions" % "TLS Resumptions" % "TLS 1.0" % "TLS 1.1" % "TLS 1.2" % "TLS 1.3" % "TLS other") << endl;
+      fmt = boost::format("%-3d %-20.20s %-20d %-20d %-25d %-20d %-20d %-20d %-20f %-20f %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d");
+      ret << (fmt % "#" % "Address" % "Connections" % "Died reading query" % "Died sending response" % "Gave up" % "Client timeouts" % "Downstream timeouts" % "Avg queries/conn" % "Avg duration" % "TLS new sessions" % "TLS Resumptions" % "TLS unknown ticket keys" % "TLS inactive ticket keys" % "TLS 1.0" % "TLS 1.1" % "TLS 1.2" % "TLS 1.3" % "TLS other") << endl;
 
       size_t counter = 0;
       for(const auto& f : g_frontends) {
-        ret << (fmt % counter % f->local.toStringWithPort() % f->tcpCurrentConnections % f->tcpDiedReadingQuery % f->tcpDiedSendingResponse % f->tcpGaveUp % f->tcpClientTimeouts % f->tcpDownstreamTimeouts % f->tcpAvgQueriesPerConnection % f->tcpAvgConnectionDuration % f->tlsNewSessions % f->tlsResumptions % f->tls10queries % f->tls11queries % f->tls12queries % f->tls13queries % f->tlsUnknownqueries) << endl;
+        ret << (fmt % counter % f->local.toStringWithPort() % f->tcpCurrentConnections % f->tcpDiedReadingQuery % f->tcpDiedSendingResponse % f->tcpGaveUp % f->tcpClientTimeouts % f->tcpDownstreamTimeouts % f->tcpAvgQueriesPerConnection % f->tcpAvgConnectionDuration % f->tlsNewSessions % f->tlsResumptions % f->tlsUnknownTicketKey % f->tlsInactiveTicketKey % f->tls10queries % f->tls11queries % f->tls12queries % f->tls13queries % f->tlsUnknownqueries) << endl;
         ++counter;
       }
       ret << endl;

diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc
index ca1f6a31e8603f571b8fe5d0d12ce5d27bc59323..6e6419fb4cee0d57f3e684456be3548b510d5592 100644 (file)
--- a/pdns/dnsdist-lua.cc
+++ b/pdns/dnsdist-lua.cc
@@ -1807,11 +1807,11 @@ void setupLuaConfig(bool client)
         setLuaNoSideEffect();
         try {
           ostringstream ret;
-          boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d");
-          ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid") << endl;
+          boost::format fmt("%-3d %-20.20s %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d %-15d");
+          ret << (fmt % "#" % "Address" % "HTTP" % "HTTP/1" % "HTTP/2" % "GET" % "POST" % "Bad" % "Errors" % "Redirects" % "Valid" % "# ticket keys" % "Rotation delay" % "Next rotation") << endl;
           size_t counter = 0;
           for (const auto& ctx : g_dohlocals) {
-            ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http1Stats.d_nbQueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses) << endl;
+            ret << (fmt % counter % ctx->d_local.toStringWithPort() % ctx->d_httpconnects % ctx->d_http1Stats.d_nbQueries % ctx->d_http1Stats.d_nbQueries % ctx->d_getqueries % ctx->d_postqueries % ctx->d_badrequests % ctx->d_errorresponses % ctx->d_redirectresponses % ctx->d_validresponses % ctx->getTicketsKeysCount() % ctx->d_ticketsKeyRotationDelay % ctx->getNextTicketsKeyRotation()) << endl;
             counter++;
           }
           g_outputBuffer = ret.str();

diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 24e3184c5718e4dc748e8befbd14196f404e7505..a2902ad6694a1b66c1bcf05d11369547c854a361 100644 (file)
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -1116,6 +1116,12 @@ static void handleIO(std::shared_ptr<IncomingTCPConnectionState>& state, struct
           else {
             ++state->d_ci.cs->tlsResumptions;
           }
+          if (state->d_handler.getResumedFromInactiveTicketKey()) {
+            ++state->d_ci.cs->tlsInactiveTicketKey;
+          }
+          if (state->d_handler.getUnknownTicketKey()) {
+            ++state->d_ci.cs->tlsUnknownTicketKey;
+          }
         }
 
         state->d_handshakeDoneTime = now;

diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc
index 6ce7e239f4beebd8f407631c4a85420963ba30bc..1ea2e214cb0ca9e21f920b1a36318fc6386c8e80 100644 (file)
--- a/pdns/dnsdist-web.cc
+++ b/pdns/dnsdist-web.cc
@@ -552,12 +552,16 @@ static void connectionThread(int sock, ComboAddress remote)
         output << "# TYPE " << frontsbase << "tcpavgqueriesperconnection " << "gauge" << "\n";
         output << "# HELP " << frontsbase << "tcpavgconnectionduration " << "The average duration of a TCP connection (ms)" << "\n";
         output << "# TYPE " << frontsbase << "tcpavgconnectionduration " << "gauge" << "\n";
+        output << "# HELP " << frontsbase << "tlsqueries " << "Number of queries received by dnsdist over TLS, by TLS version" << "\n";
+        output << "# TYPE " << frontsbase << "tlsqueries " << "counter" << "\n";
         output << "# HELP " << frontsbase << "tlsnewsessions " << "Amount of new TLS sessions negotiated" << "\n";
         output << "# TYPE " << frontsbase << "tlsnewsessions " << "counter" << "\n";
         output << "# HELP " << frontsbase << "tlsresumptions " << "Amount of TLS sessions resumed" << "\n";
         output << "# TYPE " << frontsbase << "tlsresumptions " << "counter" << "\n";
-        output << "# HELP " << frontsbase << "tlsqueries " << "Number of queries received by dnsdist over TLS, by TLS version" << "\n";
-        output << "# TYPE " << frontsbase << "tlsqueries " << "counter" << "\n";
+        output << "# HELP " << frontsbase << "tlsunknownticketkeys " << "Amount of attempts to resume TLS session from an unknown key (possibly expired)" << "\n";
+        output << "# TYPE " << frontsbase << "tlsunknownticketkeys " << "counter" << "\n";
+        output << "# HELP " << frontsbase << "tlsinactiveticketkeys " << "Amount of TLS sessions resumed from an inactive key" << "\n";
+        output << "# TYPE " << frontsbase << "tlsinactiveticketkeys " << "counter" << "\n";
 
         std::map<std::string,uint64_t> frontendDuplicates;
         for (const auto& front : g_frontends) {
@@ -589,6 +593,8 @@ static void connectionThread(int sock, ComboAddress remote)
             output << frontsbase << "tcpavgconnectionduration" << label << front->tcpAvgConnectionDuration.load() << "\n";
             output << frontsbase << "tlsnewsessions" << label << front->tlsNewSessions.load() << "\n";
             output << frontsbase << "tlsresumptions" << label << front->tlsResumptions.load() << "\n";
+            output << frontsbase << "tlsunknownticketkeys" << label << front->tlsUnknownTicketKey.load() << "\n";
+            output << frontsbase << "tlsinactiveticketkeys" << label << front->tlsInactiveTicketKey.load() << "\n";
 
             output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls10\"} " << front->tls10queries.load() << "\n";
             output << frontsbase << "tlsqueries{frontend=\"" << frontName << "\",proto=\"" << proto << "\",tls=\"tls11\"} " << front->tls11queries.load() << "\n";
@@ -772,6 +778,8 @@ static void connectionThread(int sock, ComboAddress remote)
           { "tcpAvgConnectionDuration", (double) front->tcpAvgConnectionDuration },
           { "tlsNewSessions", (double) front->tlsNewSessions },
           { "tlsResumptions", (double) front->tlsResumptions },
+          { "tlsUnknownTicketKey", (double) front->tlsUnknownTicketKey },
+          { "tlsInactiveTicketKey", (double) front->tlsInactiveTicketKey },
           { "tls10Queries", (double) front->tls10queries },
           { "tls11Queries", (double) front->tls11queries },
           { "tls12Queries", (double) front->tls12queries },

diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index b28cf44403f608aff3ee551267392d6107fc00d7..e6094695c954fc53e2d8393f6aaec13fe0fb1d16 100644 (file)
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -685,12 +685,13 @@ struct ClientState
   std::atomic<uint64_t> tcpCurrentConnections{0};
   std::atomic<uint64_t> tlsNewSessions{0}; // A new TLS session has been negotiated, no resumption
   std::atomic<uint64_t> tlsResumptions{0}; // A TLS session has been resumed, either via session id or via a TLS ticket
+  std::atomic<uint64_t> tlsUnknownTicketKey{0}; // A TLS ticket has been presented but we don't have the associated key (might have expired)
+  std::atomic<uint64_t> tlsInactiveTicketKey{0}; // A TLS ticket has been successfully resumed but the key is no longer active, we should issue a new one
   std::atomic<uint64_t> tls10queries{0};   // valid DNS queries received via TLSv1.0
   std::atomic<uint64_t> tls11queries{0};   // valid DNS queries received via TLSv1.1
   std::atomic<uint64_t> tls12queries{0};   // valid DNS queries received via TLSv1.2
   std::atomic<uint64_t> tls13queries{0};   // valid DNS queries received via TLSv1.3
   std::atomic<uint64_t> tlsUnknownqueries{0};   // valid DNS queries received via unknown TLS version
-
   std::atomic<double> tcpAvgQueriesPerConnection{0.0};
   /* in ms */
   std::atomic<double> tcpAvgConnectionDuration{0.0};

diff --git a/pdns/dnsdistdist/doh.cc b/pdns/dnsdistdist/doh.cc
index 8973bf4d18344314feba68231a44ae92809590a8..f93abdd82e46cfcca699b9851f4b1d192fe9fcd1 100644 (file)
--- a/pdns/dnsdistdist/doh.cc
+++ b/pdns/dnsdistdist/doh.cc
@@ -923,7 +923,17 @@ static int ticket_key_callback(SSL *s, unsigned char keyName[TLS_TICKETS_KEY_NAM
 
   df->handleTicketsKeyRotation();
 
-  return libssl_ticket_key_callback(s, *df->d_ticketKeys, keyName, iv, ectx, hctx, enc);
+  auto ret = libssl_ticket_key_callback(s, *df->d_ticketKeys, keyName, iv, ectx, hctx, enc);
+  if (enc == 0) {
+    if (ret == 0) {
+      ++df->d_dsc->cs->tlsUnknownTicketKey;
+    }
+    else if (ret == 2) {
+      ++df->d_dsc->cs->tlsInactiveTicketKey;
+    }
+  }
+
+  return ret;
 }
 
 static std::unique_ptr<SSL_CTX, void(*)(SSL_CTX*)> getTLSContext(DOHFrontend& df,

diff --git a/pdns/dnsdistdist/tcpiohandler.cc b/pdns/dnsdistdist/tcpiohandler.cc
index dd98fc2176e839214c8d62244df01e1b382497dd..dcf85209961f12de533ca933053c2decc5e3d02c 100644 (file)
--- a/pdns/dnsdistdist/tcpiohandler.cc
+++ b/pdns/dnsdistdist/tcpiohandler.cc
@@ -25,6 +25,14 @@ public:
   {
     d_socket = socket;
 
+    if (!s_initTLSConnIndex.test_and_set()) {
+      /* not initialized yet */
+      s_tlsConnIndex = SSL_get_ex_new_index(0, nullptr, nullptr, nullptr, nullptr);
+      if (s_tlsConnIndex == -1) {
+        throw std::runtime_error("Error getting an index for TLS connection data");
+      }
+    }
+
     if (!d_conn) {
       vinfolog("Error creating TLS object");
       if (g_verbose) {
@@ -36,6 +44,8 @@ public:
     if (!SSL_set_fd(d_conn.get(), d_socket)) {
       throw std::runtime_error("Error assigning socket");
     }
+
+    SSL_set_ex_data(d_conn.get(), s_tlsConnIndex, this);
   }
 
   IOState convertIORequestToIOState(int res) const
@@ -232,12 +242,18 @@ public:
     return false;
   }
 
+  static int s_tlsConnIndex;
 
 private:
+  static std::atomic_flag s_initTLSConnIndex;
+
   std::unique_ptr<SSL, void(*)(SSL*)> d_conn;
   unsigned int d_timeout;
 };
 
+std::atomic_flag OpenSSLTLSConnection::s_initTLSConnIndex = ATOMIC_FLAG_INIT;
+int OpenSSLTLSConnection::s_tlsConnIndex = -1;
+
 class OpenSSLTLSIOCtx: public TLSCtx
 {
 public:
@@ -376,7 +392,22 @@ public:
       return -1;
     }
 
-    return libssl_ticket_key_callback(s, ctx->d_ticketKeys, keyName, iv, ectx, hctx, enc);
+    int ret = libssl_ticket_key_callback(s, ctx->d_ticketKeys, keyName, iv, ectx, hctx, enc);
+    if (enc == 0) {
+      if (ret == 0 || ret == 2) {
+        OpenSSLTLSConnection* conn = reinterpret_cast<OpenSSLTLSConnection*>(SSL_get_ex_data(s, OpenSSLTLSConnection::s_tlsConnIndex));
+        if (conn) {
+          if (ret == 0) {
+            conn->setUnknownTicketKey();
+          }
+          else if (ret == 2) {
+            conn->setResumedFromInactiveTicketKey();
+          }
+        }
+      }
+    }
+
+    return ret;
   }
 
   static int ocspStaplingCb(SSL* ssl, void* arg)

diff --git a/pdns/doh.hh b/pdns/doh.hh
index 289b883614fc874e9fe0e1cab9eab8a75ef2ea86..6fc3d2b2c742088a9d2a0b5f21796f36df185bd4 100644 (file)
--- a/pdns/doh.hh
+++ b/pdns/doh.hh
@@ -123,6 +123,22 @@ struct DOHFrontend
 
 #endif /* HAVE_DNS_OVER_HTTPS */
 
+  time_t getNextTicketsKeyRotation() const
+  {
+    return d_ticketsKeyNextRotation;
+  }
+
+  size_t getTicketsKeysCount() const
+  {
+    size_t res = 0;
+#ifdef HAVE_DNS_OVER_HTTPS
+    if (d_ticketKeys) {
+      res = d_ticketKeys->getKeysCount();
+    }
+#endif /* HAVE_DNS_OVER_HTTPS */
+    return res;
+  }
+
 private:
   time_t d_ticketsKeyNextRotation{0};
   std::atomic_flag d_rotatingTicketsKey;

diff --git a/pdns/tcpiohandler.hh b/pdns/tcpiohandler.hh
index c5735729a96502229aa4db708f2ff9a64a6b6962..76ac5330a4c928d256ac290d15954d3fdaa2fd7f 100644 (file)
--- a/pdns/tcpiohandler.hh
+++ b/pdns/tcpiohandler.hh
@@ -22,8 +22,30 @@ public:
   virtual bool hasSessionBeenResumed() const = 0;
   virtual void close() = 0;
 
+  void setUnknownTicketKey()
+  {
+    d_unknownTicketKey = true;
+  }
+
+  bool getUnknownTicketKey() const
+  {
+    return d_unknownTicketKey;
+  }
+
+  void setResumedFromInactiveTicketKey()
+  {
+    d_resumedFromInactiveTicketKey = true;
+  }
+
+  bool getResumedFromInactiveTicketKey() const
+  {
+    return d_resumedFromInactiveTicketKey;
+  }
+
 protected:
   int d_socket{-1};
+  bool d_unknownTicketKey{false};
+  bool d_resumedFromInactiveTicketKey{false};
 };
 
 class TLSCtx
@@ -308,6 +330,16 @@ public:
     return d_conn && d_conn->hasSessionBeenResumed();
   }
 
+  bool getResumedFromInactiveTicketKey() const
+  {
+    return d_conn && d_conn->getResumedFromInactiveTicketKey();
+  }
+
+    bool getUnknownTicketKey() const
+  {
+    return d_conn && d_conn->getUnknownTicketKey();
+  }
+
 private:
   std::unique_ptr<TLSConnection> d_conn{nullptr};
   int d_socket{-1};