2 * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu>
3 * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 #include "event2/event-config.h"
29 #include "evconfig-private.h"
37 #include <sys/types.h>
39 #ifdef EVENT__HAVE_SYS_TIME_H
43 #ifdef EVENT__HAVE_SYS_SOCKET_H
44 #include <sys/socket.h>
47 #ifdef EVENT__HAVE_SYS_UIO_H
51 #ifdef EVENT__HAVE_SYS_IOCTL_H
52 #include <sys/ioctl.h>
55 #ifdef EVENT__HAVE_SYS_MMAN_H
59 #ifdef EVENT__HAVE_SYS_SENDFILE_H
60 #include <sys/sendfile.h>
62 #ifdef EVENT__HAVE_SYS_STAT_H
71 #ifdef EVENT__HAVE_STDARG_H
74 #ifdef EVENT__HAVE_UNISTD_H
79 #include "event2/event.h"
80 #include "event2/buffer.h"
81 #include "event2/buffer_compat.h"
82 #include "event2/bufferevent.h"
83 #include "event2/bufferevent_compat.h"
84 #include "event2/bufferevent_struct.h"
85 #include "event2/thread.h"
86 #include "log-internal.h"
87 #include "mm-internal.h"
88 #include "util-internal.h"
89 #include "evthread-internal.h"
90 #include "evbuffer-internal.h"
91 #include "bufferevent-internal.h"
92 #include "event-internal.h"
94 /* some systems do not have MAP_FAILED */
96 #define MAP_FAILED ((void *)-1)
99 /* send file support */
100 #if defined(EVENT__HAVE_SYS_SENDFILE_H) && defined(EVENT__HAVE_SENDFILE) && defined(__linux__)
101 #define USE_SENDFILE 1
102 #define SENDFILE_IS_LINUX 1
103 #elif defined(EVENT__HAVE_SENDFILE) && defined(__FreeBSD__)
104 #define USE_SENDFILE 1
105 #define SENDFILE_IS_FREEBSD 1
106 #elif defined(EVENT__HAVE_SENDFILE) && defined(__APPLE__)
107 #define USE_SENDFILE 1
108 #define SENDFILE_IS_MACOSX 1
109 #elif defined(EVENT__HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__)
110 #define USE_SENDFILE 1
111 #define SENDFILE_IS_SOLARIS 1
114 /* Mask of user-selectable callback flags. */
115 #define EVBUFFER_CB_USER_FLAGS 0xffff
116 /* Mask of all internal-use-only flags. */
117 #define EVBUFFER_CB_INTERNAL_FLAGS 0xffff0000
119 /* Flag set if the callback is using the cb_obsolete function pointer */
120 #define EVBUFFER_CB_OBSOLETE 0x00040000
122 /* evbuffer_chain support */
123 #define CHAIN_SPACE_PTR(ch) ((ch)->buffer + (ch)->misalign + (ch)->off)
124 #define CHAIN_SPACE_LEN(ch) ((ch)->flags & EVBUFFER_IMMUTABLE ? \
125 0 : (ch)->buffer_len - ((ch)->misalign + (ch)->off))
127 #define CHAIN_PINNED(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0)
128 #define CHAIN_PINNED_R(ch) (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0)
130 /* evbuffer_ptr support */
131 #define PTR_NOT_FOUND(ptr) do { \
133 (ptr)->internal_.chain = NULL; \
134 (ptr)->internal_.pos_in_chain = 0; \
137 #define EVBUFFER_MAX_READ_DEFAULT 4096
139 static void evbuffer_chain_align(struct evbuffer_chain *chain);
140 static int evbuffer_chain_should_realign(struct evbuffer_chain *chain,
142 static void evbuffer_deferred_callback(struct event_callback *cb, void *arg);
143 static int evbuffer_ptr_memcmp(const struct evbuffer *buf,
144 const struct evbuffer_ptr *pos, const char *mem, size_t len);
145 static struct evbuffer_chain *evbuffer_expand_singlechain(struct evbuffer *buf,
147 static int evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos,
149 static int evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg);
150 static inline void evbuffer_chain_incref(struct evbuffer_chain *chain);
152 static struct evbuffer_chain *
153 evbuffer_chain_new(size_t size)
155 struct evbuffer_chain *chain;
158 if (size > EVBUFFER_CHAIN_MAX - EVBUFFER_CHAIN_SIZE)
161 to_alloc = size + EVBUFFER_CHAIN_SIZE;
163 /* we get everything in one chunk */
164 if ((chain = mm_malloc(to_alloc)) == NULL)
167 memset(chain, 0, EVBUFFER_CHAIN_SIZE);
169 chain->buffer_len = to_alloc - EVBUFFER_CHAIN_SIZE;
171 /* this way we can manipulate the buffer to different addresses,
172 * which is required for mmap for example.
174 chain->buffer = EVBUFFER_CHAIN_EXTRA(unsigned char, chain);
181 static struct evbuffer_chain *
182 evbuffer_chain_new_membuf(size_t size)
186 if (size > EVBUFFER_CHAIN_MAX - EVBUFFER_CHAIN_SIZE)
189 size += EVBUFFER_CHAIN_SIZE;
191 /* get the next largest memory that can hold the buffer */
192 if (size < EVBUFFER_CHAIN_MAX / 2) {
193 to_alloc = MIN_BUFFER_SIZE;
194 while (to_alloc < size) {
201 return evbuffer_chain_new(to_alloc - EVBUFFER_CHAIN_SIZE);
205 evbuffer_chain_free(struct evbuffer_chain *chain)
207 EVUTIL_ASSERT(chain->refcnt > 0);
208 if (--chain->refcnt > 0) {
209 /* chain is still referenced by other chains */
213 if (CHAIN_PINNED(chain)) {
214 /* will get freed once no longer dangling */
216 chain->flags |= EVBUFFER_DANGLING;
220 /* safe to release chain, it's either a referencing
221 * chain or all references to it have been freed */
222 if (chain->flags & EVBUFFER_REFERENCE) {
223 struct evbuffer_chain_reference *info =
224 EVBUFFER_CHAIN_EXTRA(
225 struct evbuffer_chain_reference,
228 (*info->cleanupfn)(chain->buffer,
232 if (chain->flags & EVBUFFER_FILESEGMENT) {
233 struct evbuffer_chain_file_segment *info =
234 EVBUFFER_CHAIN_EXTRA(
235 struct evbuffer_chain_file_segment,
239 if (info->segment->is_mapping)
240 UnmapViewOfFile(chain->buffer);
242 evbuffer_file_segment_free(info->segment);
245 if (chain->flags & EVBUFFER_MULTICAST) {
246 struct evbuffer_multicast_parent *info =
247 EVBUFFER_CHAIN_EXTRA(
248 struct evbuffer_multicast_parent,
250 /* referencing chain is being freed, decrease
251 * refcounts of source chain and associated
252 * evbuffer (which get freed once both reach
254 EVUTIL_ASSERT(info->source != NULL);
255 EVUTIL_ASSERT(info->parent != NULL);
256 EVBUFFER_LOCK(info->source);
257 evbuffer_chain_free(info->parent);
258 evbuffer_decref_and_unlock_(info->source);
265 evbuffer_free_all_chains(struct evbuffer_chain *chain)
267 struct evbuffer_chain *next;
268 for (; chain; chain = next) {
270 evbuffer_chain_free(chain);
276 evbuffer_chains_all_empty(struct evbuffer_chain *chain)
278 for (; chain; chain = chain->next) {
285 /* The definition is needed for EVUTIL_ASSERT, which uses sizeof to avoid
286 "unused variable" warnings. */
287 static inline int evbuffer_chains_all_empty(struct evbuffer_chain *chain) {
292 /* Free all trailing chains in 'buf' that are neither pinned nor empty, prior
293 * to replacing them all with a new chain. Return a pointer to the place
294 * where the new chain will go.
296 * Internal; requires lock. The caller must fix up buf->last and buf->first
297 * as needed; they might have been freed.
299 static struct evbuffer_chain **
300 evbuffer_free_trailing_empty_chains(struct evbuffer *buf)
302 struct evbuffer_chain **ch = buf->last_with_datap;
303 /* Find the first victim chain. It might be *last_with_datap */
304 while ((*ch) && ((*ch)->off != 0 || CHAIN_PINNED(*ch)))
307 EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch));
308 evbuffer_free_all_chains(*ch);
314 /* Add a single chain 'chain' to the end of 'buf', freeing trailing empty
315 * chains as necessary. Requires lock. Does not schedule callbacks.
318 evbuffer_chain_insert(struct evbuffer *buf,
319 struct evbuffer_chain *chain)
321 ASSERT_EVBUFFER_LOCKED(buf);
322 if (*buf->last_with_datap == NULL) {
323 /* There are no chains data on the buffer at all. */
324 EVUTIL_ASSERT(buf->last_with_datap == &buf->first);
325 EVUTIL_ASSERT(buf->first == NULL);
326 buf->first = buf->last = chain;
328 struct evbuffer_chain **chp;
329 chp = evbuffer_free_trailing_empty_chains(buf);
332 buf->last_with_datap = chp;
335 buf->total_len += chain->off;
338 static inline struct evbuffer_chain *
339 evbuffer_chain_insert_new(struct evbuffer *buf, size_t datlen)
341 struct evbuffer_chain *chain;
342 if ((chain = evbuffer_chain_new_membuf(datlen)) == NULL)
344 evbuffer_chain_insert(buf, chain);
349 evbuffer_chain_pin_(struct evbuffer_chain *chain, unsigned flag)
351 EVUTIL_ASSERT((chain->flags & flag) == 0);
352 chain->flags |= flag;
356 evbuffer_chain_unpin_(struct evbuffer_chain *chain, unsigned flag)
358 EVUTIL_ASSERT((chain->flags & flag) != 0);
359 chain->flags &= ~flag;
360 if (chain->flags & EVBUFFER_DANGLING)
361 evbuffer_chain_free(chain);
365 evbuffer_chain_incref(struct evbuffer_chain *chain)
373 struct evbuffer *buffer;
375 buffer = mm_calloc(1, sizeof(struct evbuffer));
379 LIST_INIT(&buffer->callbacks);
381 buffer->last_with_datap = &buffer->first;
382 buffer->max_read = EVBUFFER_MAX_READ_DEFAULT;
388 evbuffer_set_flags(struct evbuffer *buf, ev_uint64_t flags)
391 buf->flags |= (ev_uint32_t)flags;
392 EVBUFFER_UNLOCK(buf);
397 evbuffer_clear_flags(struct evbuffer *buf, ev_uint64_t flags)
400 buf->flags &= ~(ev_uint32_t)flags;
401 EVBUFFER_UNLOCK(buf);
406 evbuffer_incref_(struct evbuffer *buf)
410 EVBUFFER_UNLOCK(buf);
414 evbuffer_incref_and_lock_(struct evbuffer *buf)
421 evbuffer_defer_callbacks(struct evbuffer *buffer, struct event_base *base)
423 EVBUFFER_LOCK(buffer);
424 buffer->cb_queue = base;
425 buffer->deferred_cbs = 1;
426 event_deferred_cb_init_(&buffer->deferred,
427 event_base_get_npriorities(base) / 2,
428 evbuffer_deferred_callback, buffer);
429 EVBUFFER_UNLOCK(buffer);
434 evbuffer_enable_locking(struct evbuffer *buf, void *lock)
436 #ifdef EVENT__DISABLE_THREAD_SUPPORT
443 EVTHREAD_ALLOC_LOCK(lock, EVTHREAD_LOCKTYPE_RECURSIVE);
458 evbuffer_set_parent_(struct evbuffer *buf, struct bufferevent *bev)
462 EVBUFFER_UNLOCK(buf);
466 evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred)
468 struct evbuffer_cb_entry *cbent, *next;
469 struct evbuffer_cb_info info;
471 ev_uint32_t mask, masked_val;
474 if (running_deferred) {
475 mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
476 masked_val = EVBUFFER_CB_ENABLED;
477 } else if (buffer->deferred_cbs) {
478 mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
479 masked_val = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
480 /* Don't zero-out n_add/n_del, since the deferred callbacks
481 will want to see them. */
484 mask = EVBUFFER_CB_ENABLED;
485 masked_val = EVBUFFER_CB_ENABLED;
488 ASSERT_EVBUFFER_LOCKED(buffer);
490 if (LIST_EMPTY(&buffer->callbacks)) {
491 buffer->n_add_for_cb = buffer->n_del_for_cb = 0;
494 if (buffer->n_add_for_cb == 0 && buffer->n_del_for_cb == 0)
497 new_size = buffer->total_len;
498 info.orig_size = new_size + buffer->n_del_for_cb - buffer->n_add_for_cb;
499 info.n_added = buffer->n_add_for_cb;
500 info.n_deleted = buffer->n_del_for_cb;
502 buffer->n_add_for_cb = 0;
503 buffer->n_del_for_cb = 0;
505 for (cbent = LIST_FIRST(&buffer->callbacks);
506 cbent != LIST_END(&buffer->callbacks);
508 /* Get the 'next' pointer now in case this callback decides
509 * to remove itself or something. */
510 next = LIST_NEXT(cbent, next);
512 if ((cbent->flags & mask) != masked_val)
515 if ((cbent->flags & EVBUFFER_CB_OBSOLETE))
516 cbent->cb.cb_obsolete(buffer,
517 info.orig_size, new_size, cbent->cbarg);
519 cbent->cb.cb_func(buffer, &info, cbent->cbarg);
524 evbuffer_invoke_callbacks_(struct evbuffer *buffer)
526 if (LIST_EMPTY(&buffer->callbacks)) {
527 buffer->n_add_for_cb = buffer->n_del_for_cb = 0;
531 if (buffer->deferred_cbs) {
532 if (event_deferred_cb_schedule_(buffer->cb_queue, &buffer->deferred)) {
533 evbuffer_incref_and_lock_(buffer);
535 bufferevent_incref_(buffer->parent);
536 EVBUFFER_UNLOCK(buffer);
540 evbuffer_run_callbacks(buffer, 0);
544 evbuffer_deferred_callback(struct event_callback *cb, void *arg)
546 struct bufferevent *parent = NULL;
547 struct evbuffer *buffer = arg;
549 /* XXXX It would be better to run these callbacks without holding the
551 EVBUFFER_LOCK(buffer);
552 parent = buffer->parent;
553 evbuffer_run_callbacks(buffer, 1);
554 evbuffer_decref_and_unlock_(buffer);
556 bufferevent_decref_(parent);
560 evbuffer_remove_all_callbacks(struct evbuffer *buffer)
562 struct evbuffer_cb_entry *cbent;
564 while ((cbent = LIST_FIRST(&buffer->callbacks))) {
565 LIST_REMOVE(cbent, next);
571 evbuffer_decref_and_unlock_(struct evbuffer *buffer)
573 struct evbuffer_chain *chain, *next;
574 ASSERT_EVBUFFER_LOCKED(buffer);
576 EVUTIL_ASSERT(buffer->refcnt > 0);
578 if (--buffer->refcnt > 0) {
579 EVBUFFER_UNLOCK(buffer);
583 for (chain = buffer->first; chain != NULL; chain = next) {
585 evbuffer_chain_free(chain);
587 evbuffer_remove_all_callbacks(buffer);
588 if (buffer->deferred_cbs)
589 event_deferred_cb_cancel_(buffer->cb_queue, &buffer->deferred);
591 EVBUFFER_UNLOCK(buffer);
592 if (buffer->own_lock)
593 EVTHREAD_FREE_LOCK(buffer->lock, EVTHREAD_LOCKTYPE_RECURSIVE);
598 evbuffer_free(struct evbuffer *buffer)
600 EVBUFFER_LOCK(buffer);
601 evbuffer_decref_and_unlock_(buffer);
604 int evbuffer_set_max_read(struct evbuffer *buf, size_t max)
612 EVBUFFER_UNLOCK(buf);
615 size_t evbuffer_get_max_read(struct evbuffer *buf)
619 result = buf->max_read;
620 EVBUFFER_UNLOCK(buf);
625 evbuffer_lock(struct evbuffer *buf)
631 evbuffer_unlock(struct evbuffer *buf)
633 EVBUFFER_UNLOCK(buf);
637 evbuffer_get_length(const struct evbuffer *buffer)
640 EVBUFFER_LOCK(buffer);
641 result = buffer->total_len;
642 EVBUFFER_UNLOCK(buffer);
647 evbuffer_get_contiguous_space(const struct evbuffer *buf)
649 struct evbuffer_chain *chain;
654 result = (chain != NULL ? chain->off : 0);
655 EVBUFFER_UNLOCK(buf);
661 evbuffer_add_iovec(struct evbuffer * buf, struct evbuffer_iovec * vec, int n_vec) {
670 for (n = 0; n < n_vec; n++) {
671 to_alloc += vec[n].iov_len;
674 if (evbuffer_expand_fast_(buf, to_alloc, 2) < 0) {
678 for (n = 0; n < n_vec; n++) {
679 /* XXX each 'add' call here does a bunch of setup that's
680 * obviated by evbuffer_expand_fast_, and some cleanup that we
681 * would like to do only once. Instead we should just extract
682 * the part of the code that's needed. */
684 if (evbuffer_add(buf, vec[n].iov_base, vec[n].iov_len) < 0) {
688 res += vec[n].iov_len;
692 EVBUFFER_UNLOCK(buf);
697 evbuffer_reserve_space(struct evbuffer *buf, ev_ssize_t size,
698 struct evbuffer_iovec *vec, int n_vecs)
700 struct evbuffer_chain *chain, **chainp;
709 if ((chain = evbuffer_expand_singlechain(buf, size)) == NULL)
712 vec[0].iov_base = (void *)CHAIN_SPACE_PTR(chain);
713 vec[0].iov_len = (size_t)CHAIN_SPACE_LEN(chain);
714 EVUTIL_ASSERT(size<0 || (size_t)vec[0].iov_len >= (size_t)size);
717 if (evbuffer_expand_fast_(buf, size, n_vecs)<0)
719 n = evbuffer_read_setup_vecs_(buf, size, vec, n_vecs,
724 EVBUFFER_UNLOCK(buf);
730 advance_last_with_data(struct evbuffer *buf)
733 struct evbuffer_chain **chainp = buf->last_with_datap;
735 ASSERT_EVBUFFER_LOCKED(buf);
740 while ((*chainp)->next) {
741 chainp = &(*chainp)->next;
743 buf->last_with_datap = chainp;
750 evbuffer_commit_space(struct evbuffer *buf,
751 struct evbuffer_iovec *vec, int n_vecs)
753 struct evbuffer_chain *chain, **firstchainp, **chainp;
765 } else if (n_vecs == 1 &&
766 (buf->last && vec[0].iov_base == (void *)CHAIN_SPACE_PTR(buf->last))) {
767 /* The user only got or used one chain; it might not
768 * be the first one with space in it. */
769 if ((size_t)vec[0].iov_len > (size_t)CHAIN_SPACE_LEN(buf->last))
771 buf->last->off += vec[0].iov_len;
772 added = vec[0].iov_len;
774 advance_last_with_data(buf);
778 /* Advance 'firstchain' to the first chain with space in it. */
779 firstchainp = buf->last_with_datap;
782 if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
783 firstchainp = &(*firstchainp)->next;
786 chain = *firstchainp;
787 /* pass 1: make sure that the pointers and lengths of vecs[] are in
788 * bounds before we try to commit anything. */
789 for (i=0; i<n_vecs; ++i) {
792 if (vec[i].iov_base != (void *)CHAIN_SPACE_PTR(chain) ||
793 (size_t)vec[i].iov_len > CHAIN_SPACE_LEN(chain))
797 /* pass 2: actually adjust all the chains. */
798 chainp = firstchainp;
799 for (i=0; i<n_vecs; ++i) {
800 (*chainp)->off += vec[i].iov_len;
801 added += vec[i].iov_len;
802 if (vec[i].iov_len) {
803 buf->last_with_datap = chainp;
805 chainp = &(*chainp)->next;
809 buf->total_len += added;
810 buf->n_add_for_cb += added;
812 evbuffer_invoke_callbacks_(buf);
815 EVBUFFER_UNLOCK(buf);
820 HAS_PINNED_R(struct evbuffer *buf)
822 return (buf->last && CHAIN_PINNED_R(buf->last));
826 ZERO_CHAIN(struct evbuffer *dst)
828 ASSERT_EVBUFFER_LOCKED(dst);
831 dst->last_with_datap = &(dst)->first;
835 /* Prepares the contents of src to be moved to another buffer by removing
836 * read-pinned chains. The first pinned chain is saved in first, and the
837 * last in last. If src has no read-pinned chains, first and last are set
840 PRESERVE_PINNED(struct evbuffer *src, struct evbuffer_chain **first,
841 struct evbuffer_chain **last)
843 struct evbuffer_chain *chain, **pinned;
845 ASSERT_EVBUFFER_LOCKED(src);
847 if (!HAS_PINNED_R(src)) {
848 *first = *last = NULL;
852 pinned = src->last_with_datap;
853 if (!CHAIN_PINNED_R(*pinned))
854 pinned = &(*pinned)->next;
855 EVUTIL_ASSERT(CHAIN_PINNED_R(*pinned));
856 chain = *first = *pinned;
859 /* If there's data in the first pinned chain, we need to allocate
860 * a new chain and copy the data over. */
862 struct evbuffer_chain *tmp;
864 EVUTIL_ASSERT(pinned == src->last_with_datap);
865 tmp = evbuffer_chain_new_membuf(chain->off);
868 memcpy(tmp->buffer, chain->buffer + chain->misalign,
870 tmp->off = chain->off;
871 *src->last_with_datap = tmp;
873 chain->misalign += chain->off;
876 src->last = *src->last_with_datap;
884 RESTORE_PINNED(struct evbuffer *src, struct evbuffer_chain *pinned,
885 struct evbuffer_chain *last)
887 ASSERT_EVBUFFER_LOCKED(src);
896 src->last_with_datap = &src->first;
901 COPY_CHAIN(struct evbuffer *dst, struct evbuffer *src)
903 ASSERT_EVBUFFER_LOCKED(dst);
904 ASSERT_EVBUFFER_LOCKED(src);
905 dst->first = src->first;
906 if (src->last_with_datap == &src->first)
907 dst->last_with_datap = &dst->first;
909 dst->last_with_datap = src->last_with_datap;
910 dst->last = src->last;
911 dst->total_len = src->total_len;
915 APPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
917 struct evbuffer_chain **chp;
919 ASSERT_EVBUFFER_LOCKED(dst);
920 ASSERT_EVBUFFER_LOCKED(src);
922 chp = evbuffer_free_trailing_empty_chains(dst);
925 if (src->last_with_datap == &src->first)
926 dst->last_with_datap = chp;
928 dst->last_with_datap = src->last_with_datap;
929 dst->last = src->last;
930 dst->total_len += src->total_len;
934 APPEND_CHAIN_MULTICAST(struct evbuffer *dst, struct evbuffer *src)
936 struct evbuffer_chain *tmp;
937 struct evbuffer_chain *chain = src->first;
938 struct evbuffer_multicast_parent *extra;
940 ASSERT_EVBUFFER_LOCKED(dst);
941 ASSERT_EVBUFFER_LOCKED(src);
943 for (; chain; chain = chain->next) {
944 if (!chain->off || chain->flags & EVBUFFER_DANGLING) {
945 /* skip empty chains */
949 tmp = evbuffer_chain_new(sizeof(struct evbuffer_multicast_parent));
951 event_warn("%s: out of memory", __func__);
954 extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_multicast_parent, tmp);
955 /* reference evbuffer containing source chain so it
956 * doesn't get released while the chain is still
957 * being referenced to */
958 evbuffer_incref_(src);
960 /* reference source chain which now becomes immutable */
961 evbuffer_chain_incref(chain);
962 extra->parent = chain;
963 chain->flags |= EVBUFFER_IMMUTABLE;
964 tmp->buffer_len = chain->buffer_len;
965 tmp->misalign = chain->misalign;
966 tmp->off = chain->off;
967 tmp->flags |= EVBUFFER_MULTICAST|EVBUFFER_IMMUTABLE;
968 tmp->buffer = chain->buffer;
969 evbuffer_chain_insert(dst, tmp);
974 PREPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
976 ASSERT_EVBUFFER_LOCKED(dst);
977 ASSERT_EVBUFFER_LOCKED(src);
978 src->last->next = dst->first;
979 dst->first = src->first;
980 dst->total_len += src->total_len;
981 if (*dst->last_with_datap == NULL) {
982 if (src->last_with_datap == &(src)->first)
983 dst->last_with_datap = &dst->first;
985 dst->last_with_datap = src->last_with_datap;
986 } else if (dst->last_with_datap == &dst->first) {
987 dst->last_with_datap = &src->last->next;
992 evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
994 struct evbuffer_chain *pinned, *last;
995 size_t in_total_len, out_total_len;
998 EVBUFFER_LOCK2(inbuf, outbuf);
999 in_total_len = inbuf->total_len;
1000 out_total_len = outbuf->total_len;
1002 if (in_total_len == 0 || outbuf == inbuf)
1005 if (outbuf->freeze_end || inbuf->freeze_start) {
1010 if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
1015 if (out_total_len == 0) {
1016 /* There might be an empty chain at the start of outbuf; free
1018 evbuffer_free_all_chains(outbuf->first);
1019 COPY_CHAIN(outbuf, inbuf);
1021 APPEND_CHAIN(outbuf, inbuf);
1024 RESTORE_PINNED(inbuf, pinned, last);
1026 inbuf->n_del_for_cb += in_total_len;
1027 outbuf->n_add_for_cb += in_total_len;
1029 evbuffer_invoke_callbacks_(inbuf);
1030 evbuffer_invoke_callbacks_(outbuf);
1033 EVBUFFER_UNLOCK2(inbuf, outbuf);
1038 evbuffer_add_buffer_reference(struct evbuffer *outbuf, struct evbuffer *inbuf)
1040 size_t in_total_len, out_total_len;
1041 struct evbuffer_chain *chain;
1044 EVBUFFER_LOCK2(inbuf, outbuf);
1045 in_total_len = inbuf->total_len;
1046 out_total_len = outbuf->total_len;
1047 chain = inbuf->first;
1049 if (in_total_len == 0)
1052 if (outbuf->freeze_end || outbuf == inbuf) {
1057 for (; chain; chain = chain->next) {
1058 if ((chain->flags & (EVBUFFER_FILESEGMENT|EVBUFFER_SENDFILE|EVBUFFER_MULTICAST)) != 0) {
1059 /* chain type can not be referenced */
1065 if (out_total_len == 0) {
1066 /* There might be an empty chain at the start of outbuf; free
1068 evbuffer_free_all_chains(outbuf->first);
1070 APPEND_CHAIN_MULTICAST(outbuf, inbuf);
1072 outbuf->n_add_for_cb += in_total_len;
1073 evbuffer_invoke_callbacks_(outbuf);
1076 EVBUFFER_UNLOCK2(inbuf, outbuf);
1081 evbuffer_prepend_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
1083 struct evbuffer_chain *pinned, *last;
1084 size_t in_total_len, out_total_len;
1087 EVBUFFER_LOCK2(inbuf, outbuf);
1089 in_total_len = inbuf->total_len;
1090 out_total_len = outbuf->total_len;
1092 if (!in_total_len || inbuf == outbuf)
1095 if (outbuf->freeze_start || inbuf->freeze_start) {
1100 if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
1105 if (out_total_len == 0) {
1106 /* There might be an empty chain at the start of outbuf; free
1108 evbuffer_free_all_chains(outbuf->first);
1109 COPY_CHAIN(outbuf, inbuf);
1111 PREPEND_CHAIN(outbuf, inbuf);
1114 RESTORE_PINNED(inbuf, pinned, last);
1116 inbuf->n_del_for_cb += in_total_len;
1117 outbuf->n_add_for_cb += in_total_len;
1119 evbuffer_invoke_callbacks_(inbuf);
1120 evbuffer_invoke_callbacks_(outbuf);
1122 EVBUFFER_UNLOCK2(inbuf, outbuf);
1127 evbuffer_drain(struct evbuffer *buf, size_t len)
1129 struct evbuffer_chain *chain, *next;
1130 size_t remaining, old_len;
1134 old_len = buf->total_len;
1139 if (buf->freeze_start) {
1144 if (len >= old_len && !HAS_PINNED_R(buf)) {
1146 for (chain = buf->first; chain != NULL; chain = next) {
1148 evbuffer_chain_free(chain);
1156 buf->total_len -= len;
1158 for (chain = buf->first;
1159 remaining >= chain->off;
1162 remaining -= chain->off;
1164 if (chain == *buf->last_with_datap) {
1165 buf->last_with_datap = &buf->first;
1167 if (&chain->next == buf->last_with_datap)
1168 buf->last_with_datap = &buf->first;
1170 if (CHAIN_PINNED_R(chain)) {
1171 EVUTIL_ASSERT(remaining == 0);
1172 chain->misalign += chain->off;
1176 evbuffer_chain_free(chain);
1180 EVUTIL_ASSERT(remaining <= chain->off);
1181 chain->misalign += remaining;
1182 chain->off -= remaining;
1185 buf->n_del_for_cb += len;
1186 /* Tell someone about changes in this buffer */
1187 evbuffer_invoke_callbacks_(buf);
1190 EVBUFFER_UNLOCK(buf);
1194 /* Reads data from an event buffer and drains the bytes read */
1196 evbuffer_remove(struct evbuffer *buf, void *data_out, size_t datlen)
1200 n = evbuffer_copyout_from(buf, NULL, data_out, datlen);
1202 if (evbuffer_drain(buf, n)<0)
1205 EVBUFFER_UNLOCK(buf);
1210 evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen)
1212 return evbuffer_copyout_from(buf, NULL, data_out, datlen);
1216 evbuffer_copyout_from(struct evbuffer *buf, const struct evbuffer_ptr *pos,
1217 void *data_out, size_t datlen)
1219 /*XXX fails badly on sendfile case. */
1220 struct evbuffer_chain *chain;
1221 char *data = data_out;
1223 ev_ssize_t result = 0;
1224 size_t pos_in_chain;
1229 if (datlen > (size_t)(EV_SSIZE_MAX - pos->pos)) {
1233 chain = pos->internal_.chain;
1234 pos_in_chain = pos->internal_.pos_in_chain;
1235 if (datlen + pos->pos > buf->total_len)
1236 datlen = buf->total_len - pos->pos;
1240 if (datlen > buf->total_len)
1241 datlen = buf->total_len;
1248 if (buf->freeze_start) {
1255 while (datlen && datlen >= chain->off - pos_in_chain) {
1256 size_t copylen = chain->off - pos_in_chain;
1258 chain->buffer + chain->misalign + pos_in_chain,
1263 chain = chain->next;
1265 EVUTIL_ASSERT(chain || datlen==0);
1269 EVUTIL_ASSERT(chain);
1270 EVUTIL_ASSERT(datlen+pos_in_chain <= chain->off);
1272 memcpy(data, chain->buffer + chain->misalign + pos_in_chain,
1278 EVBUFFER_UNLOCK(buf);
1282 /* reads data from the src buffer to the dst buffer, avoids memcpy as
1284 /* XXXX should return ev_ssize_t */
1286 evbuffer_remove_buffer(struct evbuffer *src, struct evbuffer *dst,
1289 /*XXX We should have an option to force this to be zero-copy.*/
1291 /*XXX can fail badly on sendfile case. */
1292 struct evbuffer_chain *chain, *previous;
1296 EVBUFFER_LOCK2(src, dst);
1298 chain = previous = src->first;
1300 if (datlen == 0 || dst == src) {
1305 if (dst->freeze_end || src->freeze_start) {
1310 /* short-cut if there is no more data buffered */
1311 if (datlen >= src->total_len) {
1312 datlen = src->total_len;
1313 evbuffer_add_buffer(dst, src);
1314 result = (int)datlen; /*XXXX should return ev_ssize_t*/
1318 /* removes chains if possible */
1319 while (chain->off <= datlen) {
1320 /* We can't remove the last with data from src unless we
1321 * remove all chains, in which case we would have done the if
1323 EVUTIL_ASSERT(chain != *src->last_with_datap);
1324 nread += chain->off;
1325 datlen -= chain->off;
1327 if (src->last_with_datap == &chain->next)
1328 src->last_with_datap = &src->first;
1329 chain = chain->next;
1332 if (chain != src->first) {
1333 /* we can remove the chain */
1334 struct evbuffer_chain **chp;
1335 chp = evbuffer_free_trailing_empty_chains(dst);
1337 if (dst->first == NULL) {
1338 dst->first = src->first;
1342 dst->last = previous;
1343 previous->next = NULL;
1345 advance_last_with_data(dst);
1347 dst->total_len += nread;
1348 dst->n_add_for_cb += nread;
1351 /* we know that there is more data in the src buffer than
1352 * we want to read, so we manually drain the chain */
1353 evbuffer_add(dst, chain->buffer + chain->misalign, datlen);
1354 chain->misalign += datlen;
1355 chain->off -= datlen;
1358 /* You might think we would want to increment dst->n_add_for_cb
1359 * here too. But evbuffer_add above already took care of that.
1361 src->total_len -= nread;
1362 src->n_del_for_cb += nread;
1365 evbuffer_invoke_callbacks_(dst);
1366 evbuffer_invoke_callbacks_(src);
1368 result = (int)nread;/*XXXX should change return type */
1371 EVBUFFER_UNLOCK2(src, dst);
1376 evbuffer_pullup(struct evbuffer *buf, ev_ssize_t size)
1378 struct evbuffer_chain *chain, *next, *tmp, *last_with_data;
1379 unsigned char *buffer, *result = NULL;
1380 ev_ssize_t remaining;
1381 int removed_last_with_data = 0;
1382 int removed_last_with_datap = 0;
1389 size = buf->total_len;
1390 /* if size > buf->total_len, we cannot guarantee to the user that she
1391 * is going to have a long enough buffer afterwards; so we return
1393 if (size == 0 || (size_t)size > buf->total_len)
1396 /* No need to pull up anything; the first size bytes are
1398 if (chain->off >= (size_t)size) {
1399 result = chain->buffer + chain->misalign;
1403 /* Make sure that none of the chains we need to copy from is pinned. */
1404 remaining = size - chain->off;
1405 EVUTIL_ASSERT(remaining >= 0);
1406 for (tmp=chain->next; tmp; tmp=tmp->next) {
1407 if (CHAIN_PINNED(tmp))
1409 if (tmp->off >= (size_t)remaining)
1411 remaining -= tmp->off;
1414 if (CHAIN_PINNED(chain)) {
1415 size_t old_off = chain->off;
1416 if (CHAIN_SPACE_LEN(chain) < size - chain->off) {
1417 /* not enough room at end of chunk. */
1420 buffer = CHAIN_SPACE_PTR(chain);
1424 chain = chain->next;
1425 } else if (chain->buffer_len - chain->misalign >= (size_t)size) {
1426 /* already have enough space in the first chain */
1427 size_t old_off = chain->off;
1428 buffer = chain->buffer + chain->misalign + chain->off;
1432 chain = chain->next;
1434 if ((tmp = evbuffer_chain_new_membuf(size)) == NULL) {
1435 event_warn("%s: out of memory", __func__);
1438 buffer = tmp->buffer;
1443 /* TODO(niels): deal with buffers that point to NULL like sendfile */
1445 /* Copy and free every chunk that will be entirely pulled into tmp */
1446 last_with_data = *buf->last_with_datap;
1447 for (; chain != NULL && (size_t)size >= chain->off; chain = next) {
1450 if (chain->buffer) {
1451 memcpy(buffer, chain->buffer + chain->misalign, chain->off);
1453 buffer += chain->off;
1455 if (chain == last_with_data)
1456 removed_last_with_data = 1;
1457 if (&chain->next == buf->last_with_datap)
1458 removed_last_with_datap = 1;
1460 evbuffer_chain_free(chain);
1463 if (chain != NULL) {
1464 memcpy(buffer, chain->buffer + chain->misalign, size);
1465 chain->misalign += size;
1473 if (removed_last_with_data) {
1474 buf->last_with_datap = &buf->first;
1475 } else if (removed_last_with_datap) {
1476 if (buf->first->next && buf->first->next->off)
1477 buf->last_with_datap = &buf->first->next;
1479 buf->last_with_datap = &buf->first;
1482 result = (tmp->buffer + tmp->misalign);
1485 EVBUFFER_UNLOCK(buf);
1490 * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'.
1491 * The returned buffer needs to be freed by the called.
1494 evbuffer_readline(struct evbuffer *buffer)
1496 return evbuffer_readln(buffer, NULL, EVBUFFER_EOL_ANY);
1499 static inline ev_ssize_t
1500 evbuffer_strchr(struct evbuffer_ptr *it, const char chr)
1502 struct evbuffer_chain *chain = it->internal_.chain;
1503 size_t i = it->internal_.pos_in_chain;
1504 while (chain != NULL) {
1505 char *buffer = (char *)chain->buffer + chain->misalign;
1506 char *cp = memchr(buffer+i, chr, chain->off-i);
1508 it->internal_.chain = chain;
1509 it->internal_.pos_in_chain = cp - buffer;
1510 it->pos += (cp - buffer - i);
1513 it->pos += chain->off - i;
1515 chain = chain->next;
1521 static inline char *
1522 find_eol_char(char *s, size_t len)
1524 #define CHUNK_SZ 128
1525 /* Lots of benchmarking found this approach to be faster in practice
1526 * than doing two memchrs over the whole buffer, doin a memchr on each
1527 * char of the buffer, or trying to emulate memchr by hand. */
1528 char *s_end, *cr, *lf;
1531 size_t chunk = (s + CHUNK_SZ < s_end) ? CHUNK_SZ : (s_end - s);
1532 cr = memchr(s, '\r', chunk);
1533 lf = memchr(s, '\n', chunk);
1549 evbuffer_find_eol_char(struct evbuffer_ptr *it)
1551 struct evbuffer_chain *chain = it->internal_.chain;
1552 size_t i = it->internal_.pos_in_chain;
1553 while (chain != NULL) {
1554 char *buffer = (char *)chain->buffer + chain->misalign;
1555 char *cp = find_eol_char(buffer+i, chain->off-i);
1557 it->internal_.chain = chain;
1558 it->internal_.pos_in_chain = cp - buffer;
1559 it->pos += (cp - buffer) - i;
1562 it->pos += chain->off - i;
1564 chain = chain->next;
1570 static inline size_t
1572 struct evbuffer_ptr *ptr, const char *chrset)
1575 struct evbuffer_chain *chain = ptr->internal_.chain;
1576 size_t i = ptr->internal_.pos_in_chain;
1582 char *buffer = (char *)chain->buffer + chain->misalign;
1583 for (; i < chain->off; ++i) {
1584 const char *p = chrset;
1586 if (buffer[i] == *p++)
1589 ptr->internal_.chain = chain;
1590 ptr->internal_.pos_in_chain = i;
1598 if (! chain->next) {
1599 ptr->internal_.chain = chain;
1600 ptr->internal_.pos_in_chain = i;
1605 chain = chain->next;
1611 evbuffer_getchr(struct evbuffer_ptr *it)
1613 struct evbuffer_chain *chain = it->internal_.chain;
1614 size_t off = it->internal_.pos_in_chain;
1619 return (unsigned char)chain->buffer[chain->misalign + off];
1623 evbuffer_search_eol(struct evbuffer *buffer,
1624 struct evbuffer_ptr *start, size_t *eol_len_out,
1625 enum evbuffer_eol_style eol_style)
1627 struct evbuffer_ptr it, it2;
1628 size_t extra_drain = 0;
1631 /* Avoid locking in trivial edge cases */
1632 if (start && start->internal_.chain == NULL) {
1635 *eol_len_out = extra_drain;
1639 EVBUFFER_LOCK(buffer);
1642 memcpy(&it, start, sizeof(it));
1645 it.internal_.chain = buffer->first;
1646 it.internal_.pos_in_chain = 0;
1649 /* the eol_style determines our first stop character and how many
1650 * characters we are going to drain afterwards. */
1651 switch (eol_style) {
1652 case EVBUFFER_EOL_ANY:
1653 if (evbuffer_find_eol_char(&it) < 0)
1655 memcpy(&it2, &it, sizeof(it));
1656 extra_drain = evbuffer_strspn(&it2, "\r\n");
1658 case EVBUFFER_EOL_CRLF_STRICT: {
1659 it = evbuffer_search(buffer, "\r\n", 2, &it);
1665 case EVBUFFER_EOL_CRLF: {
1666 ev_ssize_t start_pos = it.pos;
1667 /* Look for a LF ... */
1668 if (evbuffer_strchr(&it, '\n') < 0)
1671 /* ... optionally preceded by a CR. */
1672 if (it.pos == start_pos)
1673 break; /* If the first character is \n, don't back up */
1674 /* This potentially does an extra linear walk over the first
1675 * few chains. Probably, that's not too expensive unless you
1676 * have a really pathological setup. */
1677 memcpy(&it2, &it, sizeof(it));
1678 if (evbuffer_ptr_subtract(buffer, &it2, 1)<0)
1680 if (evbuffer_getchr(&it2) == '\r') {
1681 memcpy(&it, &it2, sizeof(it));
1686 case EVBUFFER_EOL_LF:
1687 if (evbuffer_strchr(&it, '\n') < 0)
1691 case EVBUFFER_EOL_NUL:
1692 if (evbuffer_strchr(&it, '\0') < 0)
1702 EVBUFFER_UNLOCK(buffer);
1707 *eol_len_out = extra_drain;
1713 evbuffer_readln(struct evbuffer *buffer, size_t *n_read_out,
1714 enum evbuffer_eol_style eol_style)
1716 struct evbuffer_ptr it;
1718 size_t n_to_copy=0, extra_drain=0;
1719 char *result = NULL;
1721 EVBUFFER_LOCK(buffer);
1723 if (buffer->freeze_start) {
1727 it = evbuffer_search_eol(buffer, NULL, &extra_drain, eol_style);
1732 if ((line = mm_malloc(n_to_copy+1)) == NULL) {
1733 event_warn("%s: out of memory", __func__);
1737 evbuffer_remove(buffer, line, n_to_copy);
1738 line[n_to_copy] = '\0';
1740 evbuffer_drain(buffer, extra_drain);
1743 EVBUFFER_UNLOCK(buffer);
1746 *n_read_out = result ? n_to_copy : 0;
1751 #define EVBUFFER_CHAIN_MAX_AUTO_SIZE 4096
1753 /* Adds data to an event buffer */
1756 evbuffer_add(struct evbuffer *buf, const void *data_in, size_t datlen)
1758 struct evbuffer_chain *chain, *tmp;
1759 const unsigned char *data = data_in;
1760 size_t remain, to_alloc;
1765 if (buf->freeze_end) {
1768 /* Prevent buf->total_len overflow */
1769 if (datlen > EV_SIZE_MAX - buf->total_len) {
1773 if (*buf->last_with_datap == NULL) {
1776 chain = *buf->last_with_datap;
1779 /* If there are no chains allocated for this buffer, allocate one
1780 * big enough to hold all the data. */
1781 if (chain == NULL) {
1782 chain = evbuffer_chain_insert_new(buf, datlen);
1787 if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
1788 /* Always true for mutable buffers */
1789 EVUTIL_ASSERT(chain->misalign >= 0 &&
1790 (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX);
1791 remain = chain->buffer_len - (size_t)chain->misalign - chain->off;
1792 if (remain >= datlen) {
1793 /* there's enough space to hold all the data in the
1794 * current last chain */
1795 memcpy(chain->buffer + chain->misalign + chain->off,
1797 chain->off += datlen;
1798 buf->total_len += datlen;
1799 buf->n_add_for_cb += datlen;
1801 } else if (!CHAIN_PINNED(chain) &&
1802 evbuffer_chain_should_realign(chain, datlen)) {
1803 /* we can fit the data into the misalignment */
1804 evbuffer_chain_align(chain);
1806 memcpy(chain->buffer + chain->off, data, datlen);
1807 chain->off += datlen;
1808 buf->total_len += datlen;
1809 buf->n_add_for_cb += datlen;
1813 /* we cannot write any data to the last chain */
1817 /* we need to add another chain */
1818 to_alloc = chain->buffer_len;
1819 if (to_alloc <= EVBUFFER_CHAIN_MAX_AUTO_SIZE/2)
1821 if (datlen > to_alloc)
1823 tmp = evbuffer_chain_new_membuf(to_alloc);
1828 memcpy(chain->buffer + chain->misalign + chain->off,
1830 chain->off += remain;
1831 buf->total_len += remain;
1832 buf->n_add_for_cb += remain;
1838 memcpy(tmp->buffer, data, datlen);
1840 evbuffer_chain_insert(buf, tmp);
1841 buf->n_add_for_cb += datlen;
1844 evbuffer_invoke_callbacks_(buf);
1847 EVBUFFER_UNLOCK(buf);
1852 evbuffer_prepend(struct evbuffer *buf, const void *data, size_t datlen)
1854 struct evbuffer_chain *chain, *tmp;
1863 if (buf->freeze_start) {
1866 if (datlen > EV_SIZE_MAX - buf->total_len) {
1872 if (chain == NULL) {
1873 chain = evbuffer_chain_insert_new(buf, datlen);
1878 /* we cannot touch immutable buffers */
1879 if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
1880 /* Always true for mutable buffers */
1881 EVUTIL_ASSERT(chain->misalign >= 0 &&
1882 (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX);
1884 /* If this chain is empty, we can treat it as
1885 * 'empty at the beginning' rather than 'empty at the end' */
1886 if (chain->off == 0)
1887 chain->misalign = chain->buffer_len;
1889 if ((size_t)chain->misalign >= datlen) {
1890 /* we have enough space to fit everything */
1891 memcpy(chain->buffer + chain->misalign - datlen,
1893 chain->off += datlen;
1894 chain->misalign -= datlen;
1895 buf->total_len += datlen;
1896 buf->n_add_for_cb += datlen;
1898 } else if (chain->misalign) {
1899 /* we can only fit some of the data. */
1900 memcpy(chain->buffer,
1901 (char*)data + datlen - chain->misalign,
1902 (size_t)chain->misalign);
1903 chain->off += (size_t)chain->misalign;
1904 buf->total_len += (size_t)chain->misalign;
1905 buf->n_add_for_cb += (size_t)chain->misalign;
1906 datlen -= (size_t)chain->misalign;
1907 chain->misalign = 0;
1911 /* we need to add another chain */
1912 if ((tmp = evbuffer_chain_new_membuf(datlen)) == NULL)
1915 if (buf->last_with_datap == &buf->first && chain->off)
1916 buf->last_with_datap = &tmp->next;
1921 EVUTIL_ASSERT(datlen <= tmp->buffer_len);
1922 tmp->misalign = tmp->buffer_len - datlen;
1924 memcpy(tmp->buffer + tmp->misalign, data, datlen);
1925 buf->total_len += datlen;
1926 buf->n_add_for_cb += datlen;
1929 evbuffer_invoke_callbacks_(buf);
1932 EVBUFFER_UNLOCK(buf);
1936 /** Helper: realigns the memory in chain->buffer so that misalign is 0. */
1938 evbuffer_chain_align(struct evbuffer_chain *chain)
1940 EVUTIL_ASSERT(!(chain->flags & EVBUFFER_IMMUTABLE));
1941 EVUTIL_ASSERT(!(chain->flags & EVBUFFER_MEM_PINNED_ANY));
1942 memmove(chain->buffer, chain->buffer + chain->misalign, chain->off);
1943 chain->misalign = 0;
1946 #define MAX_TO_COPY_IN_EXPAND 4096
1947 #define MAX_TO_REALIGN_IN_EXPAND 2048
1949 /** Helper: return true iff we should realign chain to fit datalen bytes of
1952 evbuffer_chain_should_realign(struct evbuffer_chain *chain,
1955 return chain->buffer_len - chain->off >= datlen &&
1956 (chain->off < chain->buffer_len / 2) &&
1957 (chain->off <= MAX_TO_REALIGN_IN_EXPAND);
1960 /* Expands the available space in the event buffer to at least datlen, all in
1961 * a single chunk. Return that chunk. */
1962 static struct evbuffer_chain *
1963 evbuffer_expand_singlechain(struct evbuffer *buf, size_t datlen)
1965 struct evbuffer_chain *chain, **chainp;
1966 struct evbuffer_chain *result = NULL;
1967 ASSERT_EVBUFFER_LOCKED(buf);
1969 chainp = buf->last_with_datap;
1971 /* XXX If *chainp is no longer writeable, but has enough space in its
1972 * misalign, this might be a bad idea: we could still use *chainp, not
1973 * (*chainp)->next. */
1974 if (*chainp && CHAIN_SPACE_LEN(*chainp) == 0)
1975 chainp = &(*chainp)->next;
1977 /* 'chain' now points to the first chain with writable space (if any)
1978 * We will either use it, realign it, replace it, or resize it. */
1981 if (chain == NULL ||
1982 (chain->flags & (EVBUFFER_IMMUTABLE|EVBUFFER_MEM_PINNED_ANY))) {
1983 /* We can't use the last_with_data chain at all. Just add a
1984 * new one that's big enough. */
1988 /* If we can fit all the data, then we don't have to do anything */
1989 if (CHAIN_SPACE_LEN(chain) >= datlen) {
1994 /* If the chain is completely empty, just replace it by adding a new
1996 if (chain->off == 0) {
2000 /* If the misalignment plus the remaining space fulfills our data
2001 * needs, we could just force an alignment to happen. Afterwards, we
2002 * have enough space. But only do this if we're saving a lot of space
2003 * and not moving too much data. Otherwise the space savings are
2004 * probably offset by the time lost in copying.
2006 if (evbuffer_chain_should_realign(chain, datlen)) {
2007 evbuffer_chain_align(chain);
2012 /* At this point, we can either resize the last chunk with space in
2013 * it, use the next chunk after it, or If we add a new chunk, we waste
2014 * CHAIN_SPACE_LEN(chain) bytes in the former last chunk. If we
2015 * resize, we have to copy chain->off bytes.
2018 /* Would expanding this chunk be affordable and worthwhile? */
2019 if (CHAIN_SPACE_LEN(chain) < chain->buffer_len / 8 ||
2020 chain->off > MAX_TO_COPY_IN_EXPAND ||
2021 datlen >= (EVBUFFER_CHAIN_MAX - chain->off)) {
2022 /* It's not worth resizing this chain. Can the next one be
2024 if (chain->next && CHAIN_SPACE_LEN(chain->next) >= datlen) {
2025 /* Yes, we can just use the next chain (which should
2027 result = chain->next;
2030 /* No; append a new chain (which will free all
2031 * terminal empty chains.) */
2035 /* Okay, we're going to try to resize this chain: Not doing so
2036 * would waste at least 1/8 of its current allocation, and we
2037 * can do so without having to copy more than
2038 * MAX_TO_COPY_IN_EXPAND bytes. */
2039 /* figure out how much space we need */
2040 size_t length = chain->off + datlen;
2041 struct evbuffer_chain *tmp = evbuffer_chain_new_membuf(length);
2045 /* copy the data over that we had so far */
2046 tmp->off = chain->off;
2047 memcpy(tmp->buffer, chain->buffer + chain->misalign,
2049 /* fix up the list */
2050 EVUTIL_ASSERT(*chainp == chain);
2051 result = *chainp = tmp;
2053 if (buf->last == chain)
2056 tmp->next = chain->next;
2057 evbuffer_chain_free(chain);
2062 result = evbuffer_chain_insert_new(buf, datlen);
2066 EVUTIL_ASSERT(result);
2067 EVUTIL_ASSERT(CHAIN_SPACE_LEN(result) >= datlen);
2072 /* Make sure that datlen bytes are available for writing in the last n
2073 * chains. Never copies or moves data. */
2075 evbuffer_expand_fast_(struct evbuffer *buf, size_t datlen, int n)
2077 struct evbuffer_chain *chain = buf->last, *tmp, *next;
2081 ASSERT_EVBUFFER_LOCKED(buf);
2082 EVUTIL_ASSERT(n >= 2);
2084 if (chain == NULL || (chain->flags & EVBUFFER_IMMUTABLE)) {
2085 /* There is no last chunk, or we can't touch the last chunk.
2086 * Just add a new chunk. */
2087 chain = evbuffer_chain_insert_new(buf, datlen);
2094 used = 0; /* number of chains we're using space in. */
2095 avail = 0; /* how much space they have. */
2096 /* How many bytes can we stick at the end of buffer as it is? Iterate
2097 * over the chains at the end of the buffer, tring to see how much
2098 * space we have in the first n. */
2099 for (chain = *buf->last_with_datap; chain; chain = chain->next) {
2101 size_t space = (size_t) CHAIN_SPACE_LEN(chain);
2102 EVUTIL_ASSERT(chain == *buf->last_with_datap);
2108 /* No data in chain; realign it. */
2109 chain->misalign = 0;
2110 avail += chain->buffer_len;
2113 if (avail >= datlen) {
2114 /* There is already enough space. Just return */
2121 /* There wasn't enough space in the first n chains with space in
2122 * them. Either add a new chain with enough space, or replace all
2123 * empty chains with one that has enough space, depending on n. */
2125 /* The loop ran off the end of the chains before it hit n
2126 * chains; we can add another. */
2127 EVUTIL_ASSERT(chain == NULL);
2129 tmp = evbuffer_chain_new_membuf(datlen - avail);
2133 buf->last->next = tmp;
2135 /* (we would only set last_with_data if we added the first
2136 * chain. But if the buffer had no chains, we would have
2137 * just allocated a new chain earlier) */
2140 /* Nuke _all_ the empty chains. */
2141 int rmv_all = 0; /* True iff we removed last_with_data. */
2142 chain = *buf->last_with_datap;
2144 EVUTIL_ASSERT(chain == buf->first);
2148 /* can't overflow, since only mutable chains have
2149 * huge misaligns. */
2150 avail = (size_t) CHAIN_SPACE_LEN(chain);
2151 chain = chain->next;
2155 for (; chain; chain = next) {
2157 EVUTIL_ASSERT(chain->off == 0);
2158 evbuffer_chain_free(chain);
2160 EVUTIL_ASSERT(datlen >= avail);
2161 tmp = evbuffer_chain_new_membuf(datlen - avail);
2166 buf->last = *buf->last_with_datap;
2167 (*buf->last_with_datap)->next = NULL;
2173 buf->first = buf->last = tmp;
2174 buf->last_with_datap = &buf->first;
2176 (*buf->last_with_datap)->next = tmp;
2184 evbuffer_expand(struct evbuffer *buf, size_t datlen)
2186 struct evbuffer_chain *chain;
2189 chain = evbuffer_expand_singlechain(buf, datlen);
2190 EVBUFFER_UNLOCK(buf);
2191 return chain ? 0 : -1;
2195 * Reads data from a file descriptor into a buffer.
2198 #if defined(EVENT__HAVE_SYS_UIO_H) || defined(_WIN32)
2199 #define USE_IOVEC_IMPL
2202 #ifdef USE_IOVEC_IMPL
2204 #ifdef EVENT__HAVE_SYS_UIO_H
2205 /* number of iovec we use for writev, fragmentation is going to determine
2206 * how much we end up writing */
2208 #define DEFAULT_WRITE_IOVEC 128
2210 #if defined(UIO_MAXIOV) && UIO_MAXIOV < DEFAULT_WRITE_IOVEC
2211 #define NUM_WRITE_IOVEC UIO_MAXIOV
2212 #elif defined(IOV_MAX) && IOV_MAX < DEFAULT_WRITE_IOVEC
2213 #define NUM_WRITE_IOVEC IOV_MAX
2215 #define NUM_WRITE_IOVEC DEFAULT_WRITE_IOVEC
2218 #define IOV_TYPE struct iovec
2219 #define IOV_PTR_FIELD iov_base
2220 #define IOV_LEN_FIELD iov_len
2221 #define IOV_LEN_TYPE size_t
2223 #define NUM_WRITE_IOVEC 16
2224 #define IOV_TYPE WSABUF
2225 #define IOV_PTR_FIELD buf
2226 #define IOV_LEN_FIELD len
2227 #define IOV_LEN_TYPE unsigned long
2230 #define NUM_READ_IOVEC 4
2232 /** Helper function to figure out which space to use for reading data into
2233 an evbuffer. Internal use only.
2235 @param buf The buffer to read into
2236 @param howmuch How much we want to read.
2237 @param vecs An array of two or more iovecs or WSABUFs.
2238 @param n_vecs_avail The length of vecs
2239 @param chainp A pointer to a variable to hold the first chain we're
2241 @param exact Boolean: if true, we do not provide more than 'howmuch'
2242 space in the vectors, even if more space is available.
2243 @return The number of buffers we're using.
2246 evbuffer_read_setup_vecs_(struct evbuffer *buf, ev_ssize_t howmuch,
2247 struct evbuffer_iovec *vecs, int n_vecs_avail,
2248 struct evbuffer_chain ***chainp, int exact)
2250 struct evbuffer_chain *chain;
2251 struct evbuffer_chain **firstchainp;
2254 ASSERT_EVBUFFER_LOCKED(buf);
2260 /* Let firstchain be the first chain with any space on it */
2261 firstchainp = buf->last_with_datap;
2262 EVUTIL_ASSERT(*firstchainp);
2263 if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
2264 firstchainp = &(*firstchainp)->next;
2267 chain = *firstchainp;
2268 EVUTIL_ASSERT(chain);
2269 for (i = 0; i < n_vecs_avail && so_far < (size_t)howmuch; ++i) {
2270 size_t avail = (size_t) CHAIN_SPACE_LEN(chain);
2271 if (avail > (howmuch - so_far) && exact)
2272 avail = howmuch - so_far;
2273 vecs[i].iov_base = (void *)CHAIN_SPACE_PTR(chain);
2274 vecs[i].iov_len = avail;
2276 chain = chain->next;
2279 *chainp = firstchainp;
2284 get_n_bytes_readable_on_socket(evutil_socket_t fd)
2286 #if defined(FIONREAD) && defined(_WIN32)
2287 unsigned long lng = EVBUFFER_MAX_READ_DEFAULT;
2288 if (ioctlsocket(fd, FIONREAD, &lng) < 0)
2290 /* Can overflow, but mostly harmlessly. XXXX */
2292 #elif defined(FIONREAD)
2293 int n = EVBUFFER_MAX_READ_DEFAULT;
2294 if (ioctl(fd, FIONREAD, &n) < 0)
2298 return EVBUFFER_MAX_READ_DEFAULT;
2302 /* TODO(niels): should this function return ev_ssize_t and take ev_ssize_t
2305 evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch)
2307 struct evbuffer_chain **chainp;
2311 #ifdef USE_IOVEC_IMPL
2312 int nvecs, i, remaining;
2314 struct evbuffer_chain *chain;
2320 if (buf->freeze_end) {
2325 n = get_n_bytes_readable_on_socket(fd);
2326 if (n <= 0 || n > (int)buf->max_read)
2327 n = (int)buf->max_read;
2328 if (howmuch < 0 || howmuch > n)
2331 #ifdef USE_IOVEC_IMPL
2332 /* Since we can use iovecs, we're willing to use the last
2333 * NUM_READ_IOVEC chains. */
2334 if (evbuffer_expand_fast_(buf, howmuch, NUM_READ_IOVEC) == -1) {
2338 IOV_TYPE vecs[NUM_READ_IOVEC];
2339 #ifdef EVBUFFER_IOVEC_IS_NATIVE_
2340 nvecs = evbuffer_read_setup_vecs_(buf, howmuch, vecs,
2341 NUM_READ_IOVEC, &chainp, 1);
2343 /* We aren't using the native struct iovec. Therefore,
2345 struct evbuffer_iovec ev_vecs[NUM_READ_IOVEC];
2346 nvecs = evbuffer_read_setup_vecs_(buf, howmuch, ev_vecs, 2,
2349 for (i=0; i < nvecs; ++i)
2350 WSABUF_FROM_EVBUFFER_IOV(&vecs[i], &ev_vecs[i]);
2357 if (WSARecv(fd, vecs, nvecs, &bytesRead, &flags, NULL, NULL)) {
2358 /* The read failed. It might be a close,
2359 * or it might be an error. */
2360 if (WSAGetLastError() == WSAECONNABORTED)
2368 n = readv(fd, vecs, nvecs);
2372 #else /*!USE_IOVEC_IMPL*/
2373 /* If we don't have FIONREAD, we might waste some space here */
2374 /* XXX we _will_ waste some space here if there is any space left
2375 * over on buf->last. */
2376 if ((chain = evbuffer_expand_singlechain(buf, howmuch)) == NULL) {
2381 /* We can append new data at this point */
2382 p = chain->buffer + chain->misalign + chain->off;
2385 n = read(fd, p, howmuch);
2387 n = recv(fd, p, howmuch, 0);
2389 #endif /* USE_IOVEC_IMPL */
2400 #ifdef USE_IOVEC_IMPL
2402 for (i=0; i < nvecs; ++i) {
2403 /* can't overflow, since only mutable chains have
2404 * huge misaligns. */
2405 size_t space = (size_t) CHAIN_SPACE_LEN(*chainp);
2406 /* XXXX This is a kludge that can waste space in perverse
2408 if (space > EVBUFFER_CHAIN_MAX)
2409 space = EVBUFFER_CHAIN_MAX;
2410 if ((ev_ssize_t)space < remaining) {
2411 (*chainp)->off += space;
2412 remaining -= (int)space;
2414 (*chainp)->off += remaining;
2415 buf->last_with_datap = chainp;
2418 chainp = &(*chainp)->next;
2422 advance_last_with_data(buf);
2424 buf->total_len += n;
2425 buf->n_add_for_cb += n;
2427 /* Tell someone about changes in this buffer */
2428 evbuffer_invoke_callbacks_(buf);
2431 EVBUFFER_UNLOCK(buf);
2435 #ifdef USE_IOVEC_IMPL
2437 evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd,
2440 IOV_TYPE iov[NUM_WRITE_IOVEC];
2441 struct evbuffer_chain *chain = buffer->first;
2447 ASSERT_EVBUFFER_LOCKED(buffer);
2448 /* XXX make this top out at some maximal data length? if the
2449 * buffer has (say) 1MB in it, split over 128 chains, there's
2450 * no way it all gets written in one go. */
2451 while (chain != NULL && i < NUM_WRITE_IOVEC && howmuch) {
2453 /* we cannot write the file info via writev */
2454 if (chain->flags & EVBUFFER_SENDFILE)
2457 iov[i].IOV_PTR_FIELD = (void *) (chain->buffer + chain->misalign);
2458 if ((size_t)howmuch >= chain->off) {
2459 /* XXXcould be problematic when windows supports mmap*/
2460 iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)chain->off;
2461 howmuch -= chain->off;
2463 /* XXXcould be problematic when windows supports mmap*/
2464 iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)howmuch;
2467 chain = chain->next;
2475 if (WSASend(fd, iov, i, &bytesSent, 0, NULL, NULL))
2481 n = writev(fd, iov, i);
2489 evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t dest_fd,
2492 struct evbuffer_chain *chain = buffer->first;
2493 struct evbuffer_chain_file_segment *info =
2494 EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment,
2496 const int source_fd = info->segment->fd;
2497 #if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD)
2499 ev_off_t len = chain->off;
2500 #elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS)
2502 off_t offset = chain->misalign;
2505 ASSERT_EVBUFFER_LOCKED(buffer);
2507 #if defined(SENDFILE_IS_MACOSX)
2508 res = sendfile(source_fd, dest_fd, chain->misalign, &len, NULL, 0);
2509 if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
2513 #elif defined(SENDFILE_IS_FREEBSD)
2514 res = sendfile(source_fd, dest_fd, chain->misalign, chain->off, NULL, &len, 0);
2515 if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
2519 #elif defined(SENDFILE_IS_LINUX)
2520 res = sendfile(dest_fd, source_fd, &offset, chain->off);
2521 if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
2522 /* if this is EAGAIN or EINTR return 0; otherwise, -1 */
2526 #elif defined(SENDFILE_IS_SOLARIS)
2528 const off_t offset_orig = offset;
2529 res = sendfile(dest_fd, source_fd, &offset, chain->off);
2530 if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
2531 if (offset - offset_orig)
2532 return offset - offset_orig;
2533 /* if this is EAGAIN or EINTR and no bytes were
2534 * written, return 0 */
2544 evbuffer_write_atmost(struct evbuffer *buffer, evutil_socket_t fd,
2549 EVBUFFER_LOCK(buffer);
2551 if (buffer->freeze_start) {
2555 if (howmuch < 0 || (size_t)howmuch > buffer->total_len)
2556 howmuch = buffer->total_len;
2560 struct evbuffer_chain *chain = buffer->first;
2561 if (chain != NULL && (chain->flags & EVBUFFER_SENDFILE))
2562 n = evbuffer_write_sendfile(buffer, fd, howmuch);
2565 #ifdef USE_IOVEC_IMPL
2566 n = evbuffer_write_iovec(buffer, fd, howmuch);
2567 #elif defined(_WIN32)
2568 /* XXX(nickm) Don't disable this code until we know if
2569 * the WSARecv code above works. */
2570 void *p = evbuffer_pullup(buffer, howmuch);
2571 EVUTIL_ASSERT(p || !howmuch);
2572 n = send(fd, p, howmuch, 0);
2574 void *p = evbuffer_pullup(buffer, howmuch);
2575 EVUTIL_ASSERT(p || !howmuch);
2576 n = write(fd, p, howmuch);
2584 evbuffer_drain(buffer, n);
2587 EVBUFFER_UNLOCK(buffer);
2592 evbuffer_write(struct evbuffer *buffer, evutil_socket_t fd)
2594 return evbuffer_write_atmost(buffer, fd, -1);
2598 evbuffer_find(struct evbuffer *buffer, const unsigned char *what, size_t len)
2600 unsigned char *search;
2601 struct evbuffer_ptr ptr;
2603 EVBUFFER_LOCK(buffer);
2605 ptr = evbuffer_search(buffer, (const char *)what, len, NULL);
2609 search = evbuffer_pullup(buffer, ptr.pos + len);
2613 EVBUFFER_UNLOCK(buffer);
2617 /* Subract <b>howfar</b> from the position of <b>pos</b> within
2618 * <b>buf</b>. Returns 0 on success, -1 on failure.
2620 * This isn't exposed yet, because of potential inefficiency issues.
2621 * Maybe it should be. */
2623 evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos,
2628 if (howfar > (size_t)pos->pos)
2630 if (pos->internal_.chain && howfar <= pos->internal_.pos_in_chain) {
2631 pos->internal_.pos_in_chain -= howfar;
2635 const size_t newpos = pos->pos - howfar;
2636 /* Here's the inefficient part: it walks over the
2637 * chains until we hit newpos. */
2638 return evbuffer_ptr_set(buf, pos, newpos, EVBUFFER_PTR_SET);
2643 evbuffer_ptr_set(struct evbuffer *buf, struct evbuffer_ptr *pos,
2644 size_t position, enum evbuffer_ptr_how how)
2646 size_t left = position;
2647 struct evbuffer_chain *chain = NULL;
2653 case EVBUFFER_PTR_SET:
2655 pos->pos = position;
2658 case EVBUFFER_PTR_ADD:
2659 /* this avoids iterating over all previous chains if
2660 we just want to advance the position */
2661 if (pos->pos < 0 || EV_SIZE_MAX - position < (size_t)pos->pos) {
2662 EVBUFFER_UNLOCK(buf);
2665 chain = pos->internal_.chain;
2666 pos->pos += position;
2667 position = pos->internal_.pos_in_chain;
2671 EVUTIL_ASSERT(EV_SIZE_MAX - left >= position);
2672 while (chain && position + left >= chain->off) {
2673 left -= chain->off - position;
2674 chain = chain->next;
2678 pos->internal_.chain = chain;
2679 pos->internal_.pos_in_chain = position + left;
2680 } else if (left == 0) {
2681 /* The first byte in the (nonexistent) chain after the last chain */
2682 pos->internal_.chain = NULL;
2683 pos->internal_.pos_in_chain = 0;
2689 EVBUFFER_UNLOCK(buf);
2695 Compare the bytes in buf at position pos to the len bytes in mem. Return
2696 less than 0, 0, or greater than 0 as memcmp.
2699 evbuffer_ptr_memcmp(const struct evbuffer *buf, const struct evbuffer_ptr *pos,
2700 const char *mem, size_t len)
2702 struct evbuffer_chain *chain;
2706 ASSERT_EVBUFFER_LOCKED(buf);
2709 EV_SIZE_MAX - len < (size_t)pos->pos ||
2710 pos->pos + len > buf->total_len)
2713 chain = pos->internal_.chain;
2714 position = pos->internal_.pos_in_chain;
2715 while (len && chain) {
2716 size_t n_comparable;
2717 if (len + position > chain->off)
2718 n_comparable = chain->off - position;
2721 r = memcmp(chain->buffer + chain->misalign + position, mem,
2725 mem += n_comparable;
2726 len -= n_comparable;
2728 chain = chain->next;
2735 evbuffer_search(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start)
2737 return evbuffer_search_range(buffer, what, len, start, NULL);
2741 evbuffer_search_range(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start, const struct evbuffer_ptr *end)
2743 struct evbuffer_ptr pos;
2744 struct evbuffer_chain *chain, *last_chain = NULL;
2745 const unsigned char *p;
2748 EVBUFFER_LOCK(buffer);
2751 memcpy(&pos, start, sizeof(pos));
2752 chain = pos.internal_.chain;
2755 chain = pos.internal_.chain = buffer->first;
2756 pos.internal_.pos_in_chain = 0;
2760 last_chain = end->internal_.chain;
2762 if (!len || len > EV_SSIZE_MAX)
2768 const unsigned char *start_at =
2769 chain->buffer + chain->misalign +
2770 pos.internal_.pos_in_chain;
2771 p = memchr(start_at, first,
2772 chain->off - pos.internal_.pos_in_chain);
2774 pos.pos += p - start_at;
2775 pos.internal_.pos_in_chain += p - start_at;
2776 if (!evbuffer_ptr_memcmp(buffer, &pos, what, len)) {
2777 if (end && pos.pos + (ev_ssize_t)len > end->pos)
2783 ++pos.internal_.pos_in_chain;
2784 if (pos.internal_.pos_in_chain == chain->off) {
2785 chain = pos.internal_.chain = chain->next;
2786 pos.internal_.pos_in_chain = 0;
2789 if (chain == last_chain)
2791 pos.pos += chain->off - pos.internal_.pos_in_chain;
2792 chain = pos.internal_.chain = chain->next;
2793 pos.internal_.pos_in_chain = 0;
2798 PTR_NOT_FOUND(&pos);
2800 EVBUFFER_UNLOCK(buffer);
2805 evbuffer_peek(struct evbuffer *buffer, ev_ssize_t len,
2806 struct evbuffer_ptr *start_at,
2807 struct evbuffer_iovec *vec, int n_vec)
2809 struct evbuffer_chain *chain;
2811 ev_ssize_t len_so_far = 0;
2813 /* Avoid locking in trivial edge cases */
2814 if (start_at && start_at->internal_.chain == NULL)
2817 EVBUFFER_LOCK(buffer);
2820 chain = start_at->internal_.chain;
2821 len_so_far = chain->off
2822 - start_at->internal_.pos_in_chain;
2825 vec[0].iov_base = (void *)(chain->buffer + chain->misalign
2826 + start_at->internal_.pos_in_chain);
2827 vec[0].iov_len = len_so_far;
2829 chain = chain->next;
2831 chain = buffer->first;
2834 if (n_vec == 0 && len < 0) {
2835 /* If no vectors are provided and they asked for "everything",
2836 * pretend they asked for the actual available amount. */
2837 len = buffer->total_len;
2839 len -= start_at->pos;
2844 if (len >= 0 && len_so_far >= len)
2847 vec[idx].iov_base = (void *)(chain->buffer + chain->misalign);
2848 vec[idx].iov_len = chain->off;
2853 len_so_far += chain->off;
2854 chain = chain->next;
2857 EVBUFFER_UNLOCK(buffer);
2864 evbuffer_add_vprintf(struct evbuffer *buf, const char *fmt, va_list ap)
2868 int sz, result = -1;
2870 struct evbuffer_chain *chain;
2875 if (buf->freeze_end) {
2879 /* make sure that at least some space is available */
2880 if ((chain = evbuffer_expand_singlechain(buf, 64)) == NULL)
2885 size_t used = chain->misalign + chain->off;
2886 buffer = (char *)chain->buffer + chain->misalign + chain->off;
2887 EVUTIL_ASSERT(chain->buffer_len >= used);
2888 space = chain->buffer_len - used;
2890 buffer = (char*) CHAIN_SPACE_PTR(chain);
2891 space = (size_t) CHAIN_SPACE_LEN(chain);
2894 #define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list))
2898 sz = evutil_vsnprintf(buffer, space, fmt, aq);
2904 if (INT_MAX >= EVBUFFER_CHAIN_MAX &&
2905 (size_t)sz >= EVBUFFER_CHAIN_MAX)
2907 if ((size_t)sz < space) {
2909 buf->total_len += sz;
2910 buf->n_add_for_cb += sz;
2912 advance_last_with_data(buf);
2913 evbuffer_invoke_callbacks_(buf);
2917 if ((chain = evbuffer_expand_singlechain(buf, sz + 1)) == NULL)
2923 EVBUFFER_UNLOCK(buf);
2928 evbuffer_add_printf(struct evbuffer *buf, const char *fmt, ...)
2934 res = evbuffer_add_vprintf(buf, fmt, ap);
2941 evbuffer_add_reference(struct evbuffer *outbuf,
2942 const void *data, size_t datlen,
2943 evbuffer_ref_cleanup_cb cleanupfn, void *extra)
2945 struct evbuffer_chain *chain;
2946 struct evbuffer_chain_reference *info;
2949 chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_reference));
2952 chain->flags |= EVBUFFER_REFERENCE | EVBUFFER_IMMUTABLE;
2953 chain->buffer = (unsigned char *)data;
2954 chain->buffer_len = datlen;
2955 chain->off = datlen;
2957 info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_reference, chain);
2958 info->cleanupfn = cleanupfn;
2959 info->extra = extra;
2961 EVBUFFER_LOCK(outbuf);
2962 if (outbuf->freeze_end) {
2963 /* don't call chain_free; we do not want to actually invoke
2964 * the cleanup function */
2968 evbuffer_chain_insert(outbuf, chain);
2969 outbuf->n_add_for_cb += datlen;
2971 evbuffer_invoke_callbacks_(outbuf);
2975 EVBUFFER_UNLOCK(outbuf);
2980 /* TODO(niels): we may want to add to automagically convert to mmap, in
2981 * case evbuffer_remove() or evbuffer_pullup() are being used.
2983 struct evbuffer_file_segment *
2984 evbuffer_file_segment_new(
2985 int fd, ev_off_t offset, ev_off_t length, unsigned flags)
2987 struct evbuffer_file_segment *seg =
2988 mm_calloc(sizeof(struct evbuffer_file_segment), 1);
2994 seg->file_offset = offset;
2995 seg->cleanup_cb = NULL;
2996 seg->cleanup_cb_arg = NULL;
2999 #define lseek _lseeki64
3002 #define fstat _fstat
3010 if (fstat(fd, &st) < 0)
3012 length = st.st_size;
3014 seg->length = length;
3016 if (offset < 0 || length < 0 ||
3017 ((ev_uint64_t)length > EVBUFFER_CHAIN_MAX) ||
3018 (ev_uint64_t)offset > (ev_uint64_t)(EVBUFFER_CHAIN_MAX - length))
3021 #if defined(USE_SENDFILE)
3022 if (!(flags & EVBUF_FS_DISABLE_SENDFILE)) {
3023 seg->can_sendfile = 1;
3028 if (evbuffer_file_segment_materialize(seg)<0)
3031 #if defined(USE_SENDFILE)
3034 if (!(flags & EVBUF_FS_DISABLE_LOCKING)) {
3035 EVTHREAD_ALLOC_LOCK(seg->lock, 0);
3043 #ifdef EVENT__HAVE_MMAP
3048 return sysconf(SC_PAGE_SIZE);
3049 #elif defined(_SC_PAGE_SIZE)
3050 return sysconf(_SC_PAGE_SIZE);
3060 evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg)
3062 const unsigned flags = seg->flags;
3063 const int fd = seg->fd;
3064 const ev_off_t length = seg->length;
3065 const ev_off_t offset = seg->file_offset;
3067 if (seg->contents || seg->is_mapping)
3068 return 0; /* already materialized */
3070 #if defined(EVENT__HAVE_MMAP)
3071 if (!(flags & EVBUF_FS_DISABLE_MMAP)) {
3072 off_t offset_rounded = 0, offset_leftover = 0;
3075 /* mmap implementations don't generally like us
3076 * to have an offset that isn't a round */
3077 long page_size = get_page_size();
3078 if (page_size == -1)
3080 offset_leftover = offset % page_size;
3081 offset_rounded = offset - offset_leftover;
3083 #if defined(EVENT__HAVE_MMAP64)
3084 mapped = mmap64(NULL, length + offset_leftover,
3086 mapped = mmap(NULL, length + offset_leftover,
3090 MAP_NOCACHE | /* ??? */
3096 fd, offset_rounded);
3097 if (mapped == MAP_FAILED) {
3098 event_warn("%s: mmap(%d, %d, %zu) failed",
3099 __func__, fd, 0, (size_t)(offset + length));
3101 seg->mapping = mapped;
3102 seg->contents = (char*)mapped+offset_leftover;
3103 seg->mmap_offset = 0;
3104 seg->is_mapping = 1;
3110 if (!(flags & EVBUF_FS_DISABLE_MMAP)) {
3111 intptr_t h = _get_osfhandle(fd);
3113 ev_uint64_t total_size = length+offset;
3114 if ((HANDLE)h == INVALID_HANDLE_VALUE)
3116 m = CreateFileMapping((HANDLE)h, NULL, PAGE_READONLY,
3117 (total_size >> 32), total_size & 0xfffffffful,
3119 if (m != INVALID_HANDLE_VALUE) { /* Does h leak? */
3120 seg->mapping_handle = m;
3121 seg->mmap_offset = offset;
3122 seg->is_mapping = 1;
3128 ev_off_t start_pos = lseek(fd, 0, SEEK_CUR), pos;
3129 ev_off_t read_so_far = 0;
3133 if (!(mem = mm_malloc(length)))
3135 if (start_pos < 0) {
3139 if (lseek(fd, offset, SEEK_SET) < 0) {
3143 while (read_so_far < length) {
3144 n = read(fd, mem+read_so_far, length-read_so_far);
3151 pos = lseek(fd, start_pos, SEEK_SET);
3152 if (n < 0 || (n == 0 && length > read_so_far)) {
3156 } else if (pos < 0) {
3161 seg->contents = mem;
3170 void evbuffer_file_segment_add_cleanup_cb(struct evbuffer_file_segment *seg,
3171 evbuffer_file_segment_cleanup_cb cb, void* arg)
3173 EVUTIL_ASSERT(seg->refcnt > 0);
3174 seg->cleanup_cb = cb;
3175 seg->cleanup_cb_arg = arg;
3179 evbuffer_file_segment_free(struct evbuffer_file_segment *seg)
3182 EVLOCK_LOCK(seg->lock, 0);
3183 refcnt = --seg->refcnt;
3184 EVLOCK_UNLOCK(seg->lock, 0);
3187 EVUTIL_ASSERT(refcnt == 0);
3189 if (seg->is_mapping) {
3191 CloseHandle(seg->mapping_handle);
3192 #elif defined (EVENT__HAVE_MMAP)
3193 off_t offset_leftover;
3194 offset_leftover = seg->file_offset % get_page_size();
3195 if (munmap(seg->mapping, seg->length + offset_leftover) == -1)
3196 event_warn("%s: munmap failed", __func__);
3198 } else if (seg->contents) {
3199 mm_free(seg->contents);
3202 if ((seg->flags & EVBUF_FS_CLOSE_ON_FREE) && seg->fd >= 0) {
3206 if (seg->cleanup_cb) {
3207 (*seg->cleanup_cb)((struct evbuffer_file_segment const*)seg,
3208 seg->flags, seg->cleanup_cb_arg);
3209 seg->cleanup_cb = NULL;
3210 seg->cleanup_cb_arg = NULL;
3213 EVTHREAD_FREE_LOCK(seg->lock, 0);
3218 evbuffer_add_file_segment(struct evbuffer *buf,
3219 struct evbuffer_file_segment *seg, ev_off_t offset, ev_off_t length)
3221 struct evbuffer_chain *chain;
3222 struct evbuffer_chain_file_segment *extra;
3223 int can_use_sendfile = 0;
3226 EVLOCK_LOCK(seg->lock, 0);
3227 if (buf->flags & EVBUFFER_FLAG_DRAINS_TO_FD) {
3228 can_use_sendfile = 1;
3230 if (evbuffer_file_segment_materialize(seg)<0) {
3231 EVLOCK_UNLOCK(seg->lock, 0);
3232 EVBUFFER_UNLOCK(buf);
3236 EVLOCK_UNLOCK(seg->lock, 0);
3238 if (buf->freeze_end)
3242 if (offset > seg->length)
3244 length = seg->length - offset;
3247 /* Can we actually add this? */
3248 if (offset+length > seg->length)
3251 chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_file_segment));
3254 extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment, chain);
3256 chain->flags |= EVBUFFER_IMMUTABLE|EVBUFFER_FILESEGMENT;
3257 if (can_use_sendfile && seg->can_sendfile) {
3258 chain->flags |= EVBUFFER_SENDFILE;
3259 chain->misalign = seg->file_offset + offset;
3260 chain->off = length;
3261 chain->buffer_len = chain->misalign + length;
3262 } else if (seg->is_mapping) {
3264 ev_uint64_t total_offset = seg->mmap_offset+offset;
3265 ev_uint64_t offset_rounded=0, offset_remaining=0;
3269 memset(&si, 0, sizeof(si)); /* cargo cult */
3271 offset_remaining = total_offset % si.dwAllocationGranularity;
3272 offset_rounded = total_offset - offset_remaining;
3274 data = MapViewOfFile(
3275 seg->mapping_handle,
3277 offset_rounded >> 32,
3278 offset_rounded & 0xfffffffful,
3279 length + offset_remaining);
3284 chain->buffer = (unsigned char*) data;
3285 chain->buffer_len = length+offset_remaining;
3286 chain->misalign = offset_remaining;
3287 chain->off = length;
3289 chain->buffer = (unsigned char*)(seg->contents + offset);
3290 chain->buffer_len = length;
3291 chain->off = length;
3294 chain->buffer = (unsigned char*)(seg->contents + offset);
3295 chain->buffer_len = length;
3296 chain->off = length;
3299 EVLOCK_LOCK(seg->lock, 0);
3301 EVLOCK_UNLOCK(seg->lock, 0);
3302 extra->segment = seg;
3303 buf->n_add_for_cb += length;
3304 evbuffer_chain_insert(buf, chain);
3306 evbuffer_invoke_callbacks_(buf);
3308 EVBUFFER_UNLOCK(buf);
3312 EVBUFFER_UNLOCK(buf);
3313 evbuffer_file_segment_free(seg); /* Lowers the refcount */
3318 evbuffer_add_file(struct evbuffer *buf, int fd, ev_off_t offset, ev_off_t length)
3320 struct evbuffer_file_segment *seg;
3321 unsigned flags = EVBUF_FS_CLOSE_ON_FREE;
3324 seg = evbuffer_file_segment_new(fd, offset, length, flags);
3327 r = evbuffer_add_file_segment(buf, seg, 0, length);
3329 evbuffer_file_segment_free(seg);
3334 evbuffer_setcb(struct evbuffer *buffer, evbuffer_cb cb, void *cbarg)
3336 EVBUFFER_LOCK(buffer);
3338 if (!LIST_EMPTY(&buffer->callbacks))
3339 evbuffer_remove_all_callbacks(buffer);
3342 struct evbuffer_cb_entry *ent =
3343 evbuffer_add_cb(buffer, NULL, cbarg);
3345 EVBUFFER_UNLOCK(buffer);
3348 ent->cb.cb_obsolete = cb;
3349 ent->flags |= EVBUFFER_CB_OBSOLETE;
3351 EVBUFFER_UNLOCK(buffer);
3355 struct evbuffer_cb_entry *
3356 evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
3358 struct evbuffer_cb_entry *e;
3359 if (! (e = mm_calloc(1, sizeof(struct evbuffer_cb_entry))))
3361 EVBUFFER_LOCK(buffer);
3364 e->flags = EVBUFFER_CB_ENABLED;
3365 LIST_INSERT_HEAD(&buffer->callbacks, e, next);
3366 EVBUFFER_UNLOCK(buffer);
3371 evbuffer_remove_cb_entry(struct evbuffer *buffer,
3372 struct evbuffer_cb_entry *ent)
3374 EVBUFFER_LOCK(buffer);
3375 LIST_REMOVE(ent, next);
3376 EVBUFFER_UNLOCK(buffer);
3382 evbuffer_remove_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
3384 struct evbuffer_cb_entry *cbent;
3386 EVBUFFER_LOCK(buffer);
3387 LIST_FOREACH(cbent, &buffer->callbacks, next) {
3388 if (cb == cbent->cb.cb_func && cbarg == cbent->cbarg) {
3389 result = evbuffer_remove_cb_entry(buffer, cbent);
3394 EVBUFFER_UNLOCK(buffer);
3399 evbuffer_cb_set_flags(struct evbuffer *buffer,
3400 struct evbuffer_cb_entry *cb, ev_uint32_t flags)
3402 /* the user isn't allowed to mess with these. */
3403 flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
3404 EVBUFFER_LOCK(buffer);
3406 EVBUFFER_UNLOCK(buffer);
3411 evbuffer_cb_clear_flags(struct evbuffer *buffer,
3412 struct evbuffer_cb_entry *cb, ev_uint32_t flags)
3414 /* the user isn't allowed to mess with these. */
3415 flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
3416 EVBUFFER_LOCK(buffer);
3417 cb->flags &= ~flags;
3418 EVBUFFER_UNLOCK(buffer);
3423 evbuffer_freeze(struct evbuffer *buffer, int start)
3425 EVBUFFER_LOCK(buffer);
3427 buffer->freeze_start = 1;
3429 buffer->freeze_end = 1;
3430 EVBUFFER_UNLOCK(buffer);
3435 evbuffer_unfreeze(struct evbuffer *buffer, int start)
3437 EVBUFFER_LOCK(buffer);
3439 buffer->freeze_start = 0;
3441 buffer->freeze_end = 0;
3442 EVBUFFER_UNLOCK(buffer);
3448 evbuffer_cb_suspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
3450 if (!(cb->flags & EVBUFFER_CB_SUSPENDED)) {
3451 cb->size_before_suspend = evbuffer_get_length(buffer);
3452 cb->flags |= EVBUFFER_CB_SUSPENDED;
3457 evbuffer_cb_unsuspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
3459 if ((cb->flags & EVBUFFER_CB_SUSPENDED)) {
3460 unsigned call = (cb->flags & EVBUFFER_CB_CALL_ON_UNSUSPEND);
3461 size_t sz = cb->size_before_suspend;
3462 cb->flags &= ~(EVBUFFER_CB_SUSPENDED|
3463 EVBUFFER_CB_CALL_ON_UNSUSPEND);
3464 cb->size_before_suspend = 0;
3465 if (call && (cb->flags & EVBUFFER_CB_ENABLED)) {
3466 cb->cb(buffer, sz, evbuffer_get_length(buffer), cb->cbarg);
3473 evbuffer_get_callbacks_(struct evbuffer *buffer, struct event_callback **cbs,
3477 EVBUFFER_LOCK(buffer);
3478 if (buffer->deferred_cbs) {
3483 cbs[0] = &buffer->deferred;
3487 EVBUFFER_UNLOCK(buffer);