]> granicus.if.org Git - libevent/commitdiff
Possible fix to 100% cpu usage with epoll and openssl
authorMike Smellie <mike.j.smellie@gmail.com>
Mon, 19 Jul 2010 01:44:56 +0000 (13:44 +1200)
committerNick Mathewson <nickm@torproject.org>
Mon, 19 Jul 2010 12:55:15 +0000 (14:55 +0200)
I'm running a fairly simple bit of test code using libevent2 with epoll and
openssl bufferevents and I've run into a 100% cpu usage problem.

Looking into it 100% usage was caused by epoll_wait constantly
returning write events on the openssl socket when it shouldn't really have
been looking for write events at all (N_ACTIVE_CALLBACKS() was returning 0
also).

Looking a bit deeper eventbuffer_openssl socket seems to be requesting
that the EV_WRITE event be removed when it should, but the event isn't
actually being removed from epoll.

Continuing to follow this I think I've found a bug in
event_changelist_del.

For evpoll event_del calls event_changelist_del which caches the change
which is then actioned later when evpoll_dispatch is called.

In event_changlist_del there is a check so that if the currently changed
action is an add then the cached action is changed to a no-op rather than a
delete (which makes sense). The problem arises if there are more than
two add or delete operations between calls to dispatch, in this case it's
possible that the delete is turned into a no-op when it shouldn't have
been.

For example starting with the event on, a delete followed by an add and
then another delete results in a no-op when it should have been a delete (I
added a fair bit of debug output that seems to confirm this behaviour).

I've applied a small change that checks the original old_event stored with
the change and only converts the delete to a no-op if the event isn't on in
old_event. This seems to have fixed my problem.

evmap.c
test/Makefile.am
test/Makefile.nmake
test/test-changelist.c [new file with mode: 0644]

diff --git a/evmap.c b/evmap.c
index cb28002a4295f8ed95e560014f3c6c6e65371fd5..904eefdf38328305dfb10ec26c35d451fc8fddb6 100644 (file)
--- a/evmap.c
+++ b/evmap.c
@@ -689,20 +689,30 @@ event_changelist_del(struct event_base *base, evutil_socket_t fd, short old, sho
           on those platforms where "add, delete, dispatch" is not the same
           as "no-op, dispatch", we want the no-op behavior.
 
+          As well as checking the current operation we should also check
+          the original set of events to make sure were not ignoring
+          the case where the add operation is present on an event that 
+          was already set.
+
           If we have a no-op item, we could remove it it from the list
           entirely, but really there's not much point: skipping the no-op
           change when we do the dispatch later is far cheaper than rejuggling
           the array now.
+
+          As this stands, it also lets through deletions of events that are 
+          not currently set.
         */
 
        if (events & (EV_READ|EV_SIGNAL)) {
-               if (change->read_change & EV_CHANGE_ADD)
+                if (!(change->old_events & (EV_READ | EV_SIGNAL)) && 
+                    (change->read_change & EV_CHANGE_ADD))
                        change->read_change = 0;
                else
                        change->read_change = EV_CHANGE_DEL;
        }
        if (events & EV_WRITE) {
-               if (change->write_change & EV_CHANGE_ADD)
+                if (!(change->old_events & EV_WRITE) && 
+                    (change->write_change & EV_CHANGE_ADD))
                        change->write_change = 0;
                else
                        change->write_change = EV_CHANGE_DEL;
index fd67bc1d948609392d185643290ff2df751a5188..c95aa49f5ed82ea76849ce7f427fcb133d8242de 100644 (file)
@@ -5,7 +5,8 @@ AM_CFLAGS = -I$(top_srcdir) -I$(top_srcdir)/compat -I$(top_srcdir)/include
 EXTRA_DIST = regress.rpc regress.gen.h regress.gen.c test.sh
 
 noinst_PROGRAMS = test-init test-eof test-weof test-time regress \
-       bench bench_cascade bench_http bench_httpclient test-ratelim
+       bench bench_cascade bench_http bench_httpclient test-ratelim \
+       test-changelist
 noinst_HEADERS = tinytest.h tinytest_macros.h regress.h
 
 TESTS = $(top_srcdir)/test/test.sh
@@ -15,6 +16,8 @@ test_init_SOURCES = test-init.c
 test_init_LDADD = ../libevent_core.la
 test_eof_SOURCES = test-eof.c
 test_eof_LDADD = ../libevent_core.la
+test_changelist_SOURCES = test-changelist.c
+test_changelist_LDADD = ../libevent_core.la
 test_weof_SOURCES = test-weof.c
 test_weof_LDADD = ../libevent_core.la
 test_time_SOURCES = test-time.c
@@ -65,4 +68,4 @@ DISTCLEANFILES = *~
 
 verify: check
 
-bench test-init test-eof test-weof test-time: ../libevent.la
+bench test-init test-eof test-weof test-time test-changelist: ../libevent.la
index a66ed7e4c6f5e9d172f8e589d46d2094f438a207..6e1326412d65262263b8d23c143863286557745f 100644 (file)
@@ -11,10 +11,12 @@ REGRESS_OBJS=regress.obj regress_buffer.obj regress_http.obj regress_dns.obj \
        regress_main.obj regress_minheap.obj regress_iocp.obj
 
 OTHER_OBJS=test-init.obj test-eof.obj test-weof.obj test-time.obj \
-       bench.obj bench_cascade.obj bench_http.obj bench_httpclient.obj
+       bench.obj bench_cascade.obj bench_http.obj bench_httpclient.obj \
+       test-changelist.obj
 
 PROGRAMS=regress.exe \
-       test-init.exe test-eof.exe test-weof.exe test-time.exe
+       test-init.exe test-eof.exe test-weof.exe test-time.exe \
+       test-changelist.exe
 
 # Disabled for now:
 #      bench.exe bench_cascade.exe bench_http.exe bench_httpclient.exe
@@ -31,6 +33,8 @@ test-init.exe: test-init.obj
        $(CC) $(CFLAGS) $(LIBS) test-init.obj
 test-eof.exe: test-eof.obj
        $(CC) $(CFLAGS) $(LIBS) test-eof.obj
+test-changelist.exe: test-changelist.obj
+       $(CC) $(CFLAGS) $(LIBS) test-changelist.obj
 test-weof.exe: test-weof.obj
        $(CC) $(CFLAGS) $(LIBS) test-weof.obj
 test-time.exe: test-time.obj
diff --git a/test/test-changelist.c b/test/test-changelist.c
new file mode 100644 (file)
index 0000000..1bb3dbe
--- /dev/null
@@ -0,0 +1,136 @@
+/*
+ *  based on test-eof.c
+ */
+
+#include "event-config.h"
+
+#ifdef WIN32
+#include <winsock2.h>
+#else
+#include <unistd.h>
+#endif
+#include <sys/types.h>
+#include <sys/stat.h>
+#ifdef _EVENT_HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+
+
+#ifdef _EVENT_HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include <event2/event.h>
+#include <event2/util.h>
+#include <time.h>
+
+
+static void
+write_cb(evutil_socket_t fd, short event, void *arg)
+{  
+       printf("write callback. should only see this once\n");
+        
+       /* got what we want remove the event */
+       event_del(*(struct event**)arg);
+
+       /* opps changed my mind add it back again */
+       event_add(*(struct event**)arg,NULL);
+
+       /* not a good day for decisiveness, I really didn't want it after all */
+       event_del(*(struct event**)arg);
+
+}
+
+static void
+timeout_cb(evutil_socket_t fd, short event, void *arg)
+{  
+
+       char buf[256];
+       int len;
+
+       printf("timeout fired, time to end test\n");
+       event_del(*(struct event**)arg);
+       return;
+}
+
+int
+main(int argc, char **argv)
+{
+       struct event* ev;
+       struct event* timeout;
+       struct event_base* base;
+
+       int pair[2];
+       int res;
+       int tickspassed;
+       struct timeval timeBegin;
+       struct timeval timeEnd;
+       struct timeval tv;
+
+       clock_t ticksBegin;
+       clock_t ticksEnd;
+       double usage;
+
+#ifdef WIN32
+       WORD wVersionRequested;
+       WSADATA wsaData;
+       int     err;
+
+       wVersionRequested = MAKEWORD(2, 2);
+
+       err = WSAStartup(wVersionRequested, &wsaData);
+#endif
+       if (evutil_socketpair(AF_UNIX, SOCK_STREAM, 0, pair) == -1)
+               return (1);
+
+       /* Initalize the event library */
+       base = event_base_new();
+
+       /* Initalize a timeout to terminate the test */
+       timeout = evtimer_new(base,timeout_cb,&timeout);
+       /* and watch for writability on one end of the pipe */
+       ev = event_new(base,pair[1],EV_WRITE | EV_PERSIST, write_cb, &ev);
+
+       tv.tv_sec  = 5;
+       tv.tv_usec = 0;
+
+       evtimer_add(timeout, &tv);
+
+       event_add(ev, NULL);
+
+       ticksBegin = clock();
+
+       evutil_gettimeofday(&timeBegin,NULL);
+
+       res = event_base_dispatch(base);
+
+       evutil_gettimeofday(&timeEnd,NULL);
+
+       ticksEnd = clock();
+
+       /* attempt to calculate our cpu usage over the test should be
+          virtually nil */
+       
+       tickspassed = ((((timeEnd.tv_sec - timeBegin.tv_sec) * 1000000.0) +
+                       (timeEnd.tv_usec - timeBegin.tv_usec)) *
+                      ((1.0 * CLOCKS_PER_SEC) / 1000000));
+       
+       usage = 100.0 * (((int)(ticksEnd-ticksBegin) * 1.0) / tickspassed);
+
+       printf("ticks used=%d, ticks passed=%d, cpu usage=%.2f%%\n",
+              (int)(ticksEnd-ticksBegin),
+              tickspassed,
+              usage); 
+
+       if (usage > 50.0) /* way too high */
+         return 1;
+
+       return 0;
+}
+