From: Max Dymond Date: Sun, 27 Aug 2017 14:57:05 +0000 (+0100) Subject: ossfuzz: moving towards the ideal integration X-Git-Tag: curl-7_56_0~140 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=efeb4a317616b0437a26277945bd300eaffe96d7;p=curl ossfuzz: moving towards the ideal integration - Start with the basic code from the ossfuzz project. - Rewrite fuzz corpora to be binary files full of Type-Length-Value data, and write a glue layer in the fuzzing function to convert corpora into CURL options. - Have supporting functions to generate corpora from existing tests - Integrate with Makefile.am --- diff --git a/.travis.yml b/.travis.yml index c5505075a..cd8876e86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -60,6 +60,10 @@ matrix: compiler: gcc dist: trusty env: T=distcheck + - os: linux + compiler: clang + dist: trusty + env: T=fuzzer install: - pip install --user cpp-coveralls @@ -138,6 +142,16 @@ script: cmake .. && \ make) fi + - | + if [ "$T" = "fuzzer" ]; then + export CC=clang + export CFLAGS="-fsanitize=address" + ./configure --disable-shared --enable-debug --enable-maintainer-mode + make + cd tests/fuzz + make clean + make check + fi # whitelist branches to avoid testing feature branches twice (as branch and as pull request) branches: diff --git a/Makefile.am b/Makefile.am index ab8f11cbc..e517ba56e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -210,6 +210,9 @@ test-am: endif +fuzzer: + @(cd tests/fuzz; $(MAKE) all) + examples: @(cd docs/examples; $(MAKE) check) diff --git a/configure.ac b/configure.ac index 1bfb28b1b..04d92d8f4 100755 --- a/configure.ac +++ b/configure.ac @@ -1873,7 +1873,7 @@ if test -z "$ssl_backends" -o "x$OPT_GNUTLS" != xno; then dnl linker doesn't search through, we need to add it to dnl LD_LIBRARY_PATH to prevent further configure tests to fail dnl due to this - if test "x$cross_compiling" != "xyes"; then + if test "x$cross_compiling" != "xyes"; then LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$gtlslib" export LD_LIBRARY_PATH AC_MSG_NOTICE([Added $gtlslib to LD_LIBRARY_PATH]) diff --git a/tests/curl_test_data.py b/tests/curl_test_data.py index bfe1287d8..21747407d 100755 --- a/tests/curl_test_data.py +++ b/tests/curl_test_data.py @@ -24,12 +24,15 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) import os -import xml.etree.ElementTree as ET +import re import logging log = logging.getLogger(__name__) +REPLY_DATA = re.compile("\s*(.*?)", re.MULTILINE | re.DOTALL) + + class TestData(object): def __init__(self, data_folder): self.data_folder = data_folder @@ -39,15 +42,17 @@ class TestData(object): filename = os.path.join(self.data_folder, "test{0}".format(test_number)) - # The user should handle the exception from failing to find the file. - tree = ET.parse(filename) + log.debug("Parsing file %s", filename) + + with open(filename, "rb") as f: + contents = f.read().decode("utf-8") - # We need the text. - reply = tree.find("reply") - data = reply.find("data") + m = REPLY_DATA.search(contents) + if not m: + raise Exception("Couldn't find a section") - # Return the text contents of the data - return data.text + # Left-strip the data so we don't get a newline before our data. + return m.group(1).lstrip() if __name__ == '__main__': diff --git a/tests/fuzz/CMakeLists.txt b/tests/fuzz/CMakeLists.txt deleted file mode 100644 index aefedf26f..000000000 --- a/tests/fuzz/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -# FIXME, probably adapt from file in ../unit diff --git a/tests/fuzz/Makefile.am b/tests/fuzz/Makefile.am index 7a245dda5..270b9783e 100644 --- a/tests/fuzz/Makefile.am +++ b/tests/fuzz/Makefile.am @@ -30,18 +30,21 @@ AUTOMAKE_OPTIONS = foreign nostdinc # $(top_builddir)/lib is for libcurl's generated lib/curl_config.h file # $(top_srcdir)/lib for libcurl's lib/curl_setup.h and other "borrowed" files -AM_CPPFLAGS = -I$(top_srcdir)/include \ - -I$(top_builddir)/lib \ - -I$(top_srcdir)/lib \ - -I$(top_srcdir)/tests/fuzz +AM_CFLAGS = -I$(top_srcdir)/include \ + -I$(top_builddir)/lib \ + -I$(top_srcdir)/lib \ + -I$(top_srcdir)/tests/fuzz -EXTRA_DIST = Makefile.inc CMakeLists.txt +LIBS = -lpthread -lstdc++ -lm -LIBS = -lpthread -lFuzzer -lstdc++ -lm -LDFLAGS = -L/usr/lib/llvm-5.0/lib +# Run e.g. "make all LIB_FUZZING_ENGINE=/path/to/libFuzzer.a" +# to link the fuzzer(s) against a real fuzzing engine. +# +# OSS-Fuzz will define its own value for LIB_FUZZING_ENGINE. +LIB_FUZZING_ENGINE ?= libstandaloneengine.a LDADD = $(top_builddir)/lib/libcurl.la \ - @LDFLAGS@ @LIBCURL_LIBS@ + $(LIB_FUZZING_ENGINE) @LDFLAGS@ @LIBCURL_LIBS@ # Makefile.inc provides neat definitions include Makefile.inc @@ -50,4 +53,4 @@ checksrc: @PERL@ $(top_srcdir)/lib/checksrc.pl $(srcdir)/*.c noinst_PROGRAMS = $(FUZZPROGS) - +noinst_LIBRARIES = $(FUZZLIBS) \ No newline at end of file diff --git a/tests/fuzz/Makefile.inc b/tests/fuzz/Makefile.inc index fb6cdb11a..4d475374b 100644 --- a/tests/fuzz/Makefile.inc +++ b/tests/fuzz/Makefile.inc @@ -1,19 +1,15 @@ -FUZZPROGS = http11 ftp imap pop3 httpupload http2 +FUZZPROGS = curl_fuzzer +FUZZLIBS = libstandaloneengine.a -http11_SOURCES = curl_fuzzer.c -http11_CPPFLAGS = $(AM_CPPFLAGS) +curl_fuzzer_SOURCES = curl_fuzzer.c +curl_fuzzer_CFLAGS = $(AM_CFLAGS) -ftp_SOURCES = curl_fuzzer.c -ftp_CPPFLAGS = -DFUZZER_FTP $(AM_CPPFLAGS) +libstandaloneengine_a_SOURCES = standalone_fuzz_target_runner.c +libstandaloneengine_a_CFLAGS = $(AM_CFLAGS) -imap_SOURCES = curl_fuzzer.c -imap_CPPFLAGS = -DFUZZER_IMAP $(AM_CPPFLAGS) +# Some more targets. +zip: + zip -q -r curl_fuzzer_seed_corpus.zip curl_fuzz_data -pop3_SOURCES = curl_fuzzer.c -pop3_CPPFLAGS = -DFUZZER_POP3 $(AM_CPPFLAGS) - -httpupload_SOURCES = curl_fuzzer.c -httpupload_CPPFLAGS = -DFUZZER_HTTP_UPLOAD $(AM_CPPFLAGS) - -http2_SOURCES = curl_fuzzer.c -http2_CPPFLAGS = -DFUZZER_HTTP2 $(AM_CPPFLAGS) +check: all + ./curl_fuzzer curl_fuzz_data/* \ No newline at end of file diff --git a/tests/fuzz/README b/tests/fuzz/README index 459bda46b..cdb69fe82 100644 --- a/tests/fuzz/README +++ b/tests/fuzz/README @@ -2,13 +2,18 @@ Fuzz tests ========== The goal is to add tests for *ALL* protocols supported in libcurl. -We will need some additional patches in the future, to increase coverage. Building the fuzz target ======================== +From the CURL root directory: -CC=clang-5.0 CFLAGS="-fsanitize=address -fsanitize-address-use-after-scope -fsanitize-coverage=trace-pc-guard,trace-cmp" ./configure --disable-shared --enable-debug --enable-maintainer-mode +export CC=clang-5.0 +export CFLAGS="-fsanitize=address -fsanitize-address-use-after-scope -fsanitize-coverage=trace-pc-guard,trace-cmp" +./configure --disable-shared --enable-debug --enable-maintainer-mode make -sj cd tests/fuzz -make + +(optional) export LIB_FUZZING_ENGINE= + +make check diff --git a/tests/fuzz/corpora/ftp/long1.txt b/tests/fuzz/corpora/ftp/long1.txt deleted file mode 100644 index 47a536e38..000000000 --- a/tests/fuzz/corpora/ftp/long1.txt +++ /dev/null @@ -1,8 +0,0 @@ -200 OK -200 OK -215 UNIX Type: L8 -200 OK -200 OK -200 OK -AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA -220 OK diff --git a/tests/fuzz/corpora/http1_1/200_ok.txt b/tests/fuzz/corpora/http1_1/200_ok.txt deleted file mode 100644 index ca7ef0b23..000000000 --- a/tests/fuzz/corpora/http1_1/200_ok.txt +++ /dev/null @@ -1,13 +0,0 @@ -HTTP/1.1 200 OK -Content-Encoding: gzip -Accept-Ranges: bytes -Cache-Control: max-age=604800 -Content-Type: text/html -Date: Mon, 08 May 2017 19:03:58 GMT -Etag: "359670651+gzip" -Expires: Mon, 15 May 2017 19:03:58 GMT -Last-Modified: Fri, 09 Aug 2013 23:54:35 GMT -Server: ECS (ewr/15BD) -X-Cache: HIT -Content-Length: 606 - diff --git a/tests/fuzz/corpora/http1_1/404_nf.txt b/tests/fuzz/corpora/http1_1/404_nf.txt deleted file mode 100644 index a28311a35..000000000 --- a/tests/fuzz/corpora/http1_1/404_nf.txt +++ /dev/null @@ -1,9 +0,0 @@ -HTTP/1.1 404 Not Found -Cache-Control: max-age=604800 -Content-Type: text/html -Date: Mon, 08 May 2017 19:04:08 GMT -Expires: Mon, 15 May 2017 19:04:08 GMT -Server: EOS (lax004/28A3) -Vary: Accept-Encoding -Content-Length: 1270 - diff --git a/tests/fuzz/curl_fuzz_data/test1 b/tests/fuzz/curl_fuzz_data/test1 new file mode 100644 index 000000000..f7b734a9c Binary files /dev/null and b/tests/fuzz/curl_fuzz_data/test1 differ diff --git a/tests/fuzz/curl_fuzz_data/test2 b/tests/fuzz/curl_fuzz_data/test2 new file mode 100644 index 000000000..8b44d6719 Binary files /dev/null and b/tests/fuzz/curl_fuzz_data/test2 differ diff --git a/tests/fuzz/curl_fuzz_data/test3 b/tests/fuzz/curl_fuzz_data/test3 new file mode 100644 index 000000000..81c6670aa Binary files /dev/null and b/tests/fuzz/curl_fuzz_data/test3 differ diff --git a/tests/fuzz/curl_fuzzer.c b/tests/fuzz/curl_fuzzer.c index 2ccf1b36e..f4a4ec6f9 100644 --- a/tests/fuzz/curl_fuzzer.c +++ b/tests/fuzz/curl_fuzzer.c @@ -1,128 +1,340 @@ -/* -# Copyright 2016 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ -*/ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2017, Max Dymond, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ -#include -#include -#include -#include #include #include #include #include -#include -#include -#include -#include -#include #include - #include +#include "curl_fuzzer.h" + +/** + * Fuzzing entry point. This function is passed a buffer containing a test + * case. This test case should drive the CURL API into making a request. + */ +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + int rc = 0; + int tlv_rc; + FUZZ_DATA fuzz; + TLV tlv; + + if(size < sizeof(TLV_RAW)) { + /* Not enough data */ + goto EXIT_LABEL; + } -static const void *cur_data; -static int cur_size = -1; -static int server_fd = -1; -static int client_fd = -1; -static int wrote = 0; + /* Try to initialize the fuzz data */ + FTRY(fuzz_initialize_fuzz_data(&fuzz, data, size)); + + for(tlv_rc = fuzz_get_first_tlv(&fuzz, &tlv); + tlv_rc == 0; + tlv_rc = fuzz_get_next_tlv(&fuzz, &tlv)) { + /* Have the TLV in hand. Parse the TLV. */ + fuzz_parse_tlv(&fuzz, &tlv); + } + + if(tlv_rc != TLV_RC_NO_MORE_TLVS) { + /* A TLV call failed. Can't continue. */ + goto EXIT_LABEL; + } + + /* Do the CURL stuff! */ + curl_easy_perform(fuzz.easy); + +EXIT_LABEL: + + fuzz_terminate_fuzz_data(&fuzz); + + /* This function must always return 0. Non-zero codes are reserved. */ + return 0; +} + +/** + * Utility function to convert 4 bytes to a u32 predictably. + */ +uint32_t to_u32(uint8_t b[4]) +{ + uint32_t u; + u = (b[0] << 24) + (b[1] << 16) + (b[2] << 8) + b[3]; + return u; +} -static void fail(const char *why) { - perror(why); - exit(1); +/** + * Utility function to convert 2 bytes to a u16 predictably. + */ +uint16_t to_u16(uint8_t b[2]) +{ + uint16_t u; + u = (b[0] << 8) + b[1]; + return u; } -static curl_socket_t open_sock(void *ctx, curlsocktype purpose, - struct curl_sockaddr *address) { - if(cur_size == -1) { - fail("not fuzzing"); +/** + * Initialize the local fuzz data structure. + */ +int fuzz_initialize_fuzz_data(FUZZ_DATA *fuzz, + const uint8_t *data, + size_t data_len) +{ + int rc = 0; + + /* Initialize the fuzz data. */ + memset(fuzz, 0, sizeof(FUZZ_DATA)); + + /* Create an easy handle. This will have all of the settings configured on + it. */ + fuzz->easy = curl_easy_init(); + FCHECK(fuzz->easy != NULL); + + /* Set some standard options on the CURL easy handle. We need to override the + socket function so that we create our own sockets to present to CURL. */ + FTRY(curl_easy_setopt(fuzz->easy, + CURLOPT_OPENSOCKETFUNCTION, + fuzz_open_socket)); + FTRY(curl_easy_setopt(fuzz->easy, CURLOPT_OPENSOCKETDATA, fuzz)); + + /* In case something tries to set a socket option, intercept this. */ + FTRY(curl_easy_setopt(fuzz->easy, + CURLOPT_SOCKOPTFUNCTION, + fuzz_sockopt_callback)); + + /* Can enable verbose mode */ + /* FTRY(curl_easy_setopt(fuzz->easy, CURLOPT_VERBOSE, 1L)); */ + + /* Set up the state parser */ + fuzz->state.data = data; + fuzz->state.data_len = data_len; + +EXIT_LABEL: + + return rc; +} + +/** + * Terminate the fuzz data structure, including freeing any allocated memory. + */ +void fuzz_terminate_fuzz_data(FUZZ_DATA *fuzz) +{ + fuzz_free((void **)&fuzz->url); + fuzz_free((void **)&fuzz->username); + fuzz_free((void **)&fuzz->password); + fuzz_free((void **)&fuzz->postfields); + + if(fuzz->easy != NULL) { + curl_easy_cleanup(fuzz->easy); + fuzz->easy = NULL; } - if(server_fd != -1 || client_fd != -1) { - fail("already connected"); +} + +/** + * If a pointer has been allocated, free that pointer. + */ +void fuzz_free(void **ptr) +{ + if(*ptr != NULL) { + free(*ptr); + *ptr = NULL; } +} + +/** + * Function for providing a socket to CURL already primed with data. + */ +static curl_socket_t fuzz_open_socket(void *ptr, + curlsocktype purpose, + struct curl_sockaddr *address) +{ + FUZZ_DATA *fuzz = (FUZZ_DATA *)ptr; int fds[2]; + curl_socket_t server_fd; + curl_socket_t client_fd; + + /* Handle unused parameters */ + (void)purpose; + (void)address; + if(socketpair(AF_UNIX, SOCK_STREAM, 0, fds)) { - fail("socketpair"); + /* Failed to create a pair of sockets. */ + return CURL_SOCKET_BAD; } + server_fd = fds[0]; client_fd = fds[1]; - if(write(server_fd, cur_data, cur_size) != cur_size) { - fail("write"); + + /* Try and write the response data to the server file descriptor so the + client can read it. */ + if(write(server_fd, + fuzz->rsp1_data, + fuzz->rsp1_data_len) != (ssize_t)fuzz->rsp1_data_len) { + /* Failed to write the data. */ + return CURL_SOCKET_BAD; } + if(shutdown(server_fd, SHUT_WR)) { - fail("shutdown"); + return CURL_SOCKET_BAD; } + return client_fd; } -static int set_opt(void *ctx, curl_socket_t curlfd, curlsocktype purpose) { +/** + * Callback function for setting socket options on the sockets created by + * fuzz_open_socket. In our testbed the sockets are "already connected". + */ +static int fuzz_sockopt_callback(void *ptr, + curl_socket_t curlfd, + curlsocktype purpose) +{ + (void)ptr; + (void)curlfd; + (void)purpose; + return CURL_SOCKOPT_ALREADY_CONNECTED; } -static size_t write_callback(char *ptr, size_t size, size_t n, void *ctx) { - return size * n; +/** + * TLV access function - gets the first TLV from a data stream. + */ +int fuzz_get_first_tlv(FUZZ_DATA *fuzz, + TLV *tlv) +{ + /* Reset the cursor. */ + fuzz->state.data_pos = 0; + return fuzz_get_tlv_comn(fuzz, tlv); } -static size_t read_callback(char *buf, size_t size, size_t n, void *ctx) { - if(wrote || size * n == 0) { - return 0; +/** + * TLV access function - gets the next TLV from a data stream. +*/ +int fuzz_get_next_tlv(FUZZ_DATA *fuzz, + TLV *tlv) +{ + /* Advance the cursor by the full length of the previous TLV. */ + fuzz->state.data_pos += sizeof(TLV_RAW) + tlv->length; + + /* Work out if there's a TLV's worth of data to read */ + if(fuzz->state.data_pos + sizeof(TLV_RAW) > fuzz->state.data_len) { + /* No more TLVs to parse */ + return TLV_RC_NO_MORE_TLVS; } - wrote = 1; - buf[0] = 'a'; - return 1; + + return fuzz_get_tlv_comn(fuzz, tlv); } -int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { - cur_data = Data; - cur_size = Size; - wrote = 0; - CURL *curl = curl_easy_init(); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); - curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); - curl_easy_setopt(curl, CURLOPT_OPENSOCKETFUNCTION, open_sock); - curl_easy_setopt(curl, CURLOPT_SOCKOPTFUNCTION, set_opt); -#if defined(FUZZER_FTP) - curl_easy_setopt(curl, CURLOPT_URL, "ftp://user@localhost/file.txt"); -#elif defined(FUZZER_IMAP) - curl_easy_setopt(curl, CURLOPT_USERNAME, "user"); - curl_easy_setopt(curl, CURLOPT_PASSWORD, "secret"); - curl_easy_setopt(curl, CURLOPT_URL, "imap://localhost"); -#elif defined(FUZZER_POP3) - curl_easy_setopt(curl, CURLOPT_USERNAME, "user"); - curl_easy_setopt(curl, CURLOPT_PASSWORD, "secret"); - curl_easy_setopt(curl, CURLOPT_URL, "pop3://localhost"); -#elif defined(FUZZER_HTTP_UPLOAD) - curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/"); - curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); - curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); -#elif defined(FUZZER_HTTP2) - curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/"); - /* use non-TLS HTTP/2 without HTTP/1.1 Upgrade: */ - curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, - CURL_HTTP_VERSION_2_PRIOR_KNOWLEDGE); -#else - curl_easy_setopt(curl, CURLOPT_URL, "http://localhost/"); - curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); -#endif - curl_easy_perform(curl); - curl_easy_cleanup(curl); - close(server_fd); - close(client_fd); - server_fd = -1; - client_fd = -1; - cur_data = NULL; - cur_size = -1; - return 0; +/** + * Common TLV function for accessing TLVs in a data stream. + */ +int fuzz_get_tlv_comn(FUZZ_DATA *fuzz, + TLV *tlv) +{ + int rc = 0; + size_t data_offset; + TLV_RAW *raw; + + /* Start by casting the data stream to a TLV. */ + raw = (TLV_RAW *)&fuzz->state.data[fuzz->state.data_pos]; + data_offset = fuzz->state.data_pos + sizeof(TLV_RAW); + + /* Set the TLV values. */ + tlv->type = to_u16(raw->raw_type); + tlv->length = to_u32(raw->raw_length); + tlv->value = &fuzz->state.data[data_offset]; + + /* Sanity check that the TLV length is ok. */ + if(data_offset + tlv->length > fuzz->state.data_len) { + rc = TLV_RC_SIZE_ERROR; + } + + return rc; +} + +/** + * Do different actions on the CURL handle for different received TLVs. + */ +int fuzz_parse_tlv(FUZZ_DATA *fuzz, TLV *tlv) +{ + int rc; + + switch(tlv->type) { + case TLV_TYPE_URL: + FCHECK(fuzz->url == NULL); + fuzz->url = fuzz_tlv_to_string(tlv); + FTRY(curl_easy_setopt(fuzz->easy, CURLOPT_URL, fuzz->url)); + break; + + case TLV_TYPE_RESPONSE1: + /* The pointers in the TLV will always be valid as long as the fuzz data + is in scope, which is the entirety of this file. */ + fuzz->rsp1_data = tlv->value; + fuzz->rsp1_data_len = tlv->length; + break; + + case TLV_TYPE_USERNAME: + FCHECK(fuzz->username == NULL); + fuzz->username = fuzz_tlv_to_string(tlv); + FTRY(curl_easy_setopt(fuzz->easy, CURLOPT_USERNAME, fuzz->username)); + break; + + case TLV_TYPE_PASSWORD: + FCHECK(fuzz->password == NULL); + fuzz->password = fuzz_tlv_to_string(tlv); + FTRY(curl_easy_setopt(fuzz->easy, CURLOPT_PASSWORD, fuzz->password)); + break; + + case TLV_TYPE_POSTFIELDS: + FCHECK(fuzz->postfields == NULL); + fuzz->postfields = fuzz_tlv_to_string(tlv); + FTRY(curl_easy_setopt(fuzz->easy, CURLOPT_POSTFIELDS, fuzz->postfields)); + break; + + default: + /* The fuzzer generates lots of unknown TLVs, so don't do anything if + the TLV isn't known. */ + break; + } + + rc = 0; + +EXIT_LABEL: + + return rc; +} + +/** + * Converts a TLV data and length into an allocated string. + */ +char *fuzz_tlv_to_string(TLV *tlv) +{ + char *tlvstr; + + /* Allocate enough space, plus a null terminator */ + tlvstr = malloc(tlv->length + 1); + + if(tlvstr != NULL) { + memcpy(tlvstr, tlv->value, tlv->length); + tlvstr[tlv->length] = 0; + } + + return tlvstr; } diff --git a/tests/fuzz/curl_fuzzer.h b/tests/fuzz/curl_fuzzer.h new file mode 100644 index 000000000..a0c9d596a --- /dev/null +++ b/tests/fuzz/curl_fuzzer.h @@ -0,0 +1,148 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2017, Max Dymond, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include + +/** + * TLV types. + */ +#define TLV_TYPE_URL 1 +#define TLV_TYPE_RESPONSE1 2 +#define TLV_TYPE_USERNAME 3 +#define TLV_TYPE_PASSWORD 4 +#define TLV_TYPE_POSTFIELDS 5 + +/** + * TLV function return codes. + */ +#define TLV_RC_NO_ERROR 0 +#define TLV_RC_NO_MORE_TLVS 1 +#define TLV_RC_SIZE_ERROR 2 + +/** + * Byte stream representation of the TLV header. Casting the byte stream + * to a TLV_RAW allows us to examine the type and length. + */ +typedef struct tlv_raw +{ + /* Type of the TLV - 16 bits. */ + uint8_t raw_type[2]; + + /* Length of the TLV data - 32 bits. */ + uint8_t raw_length[4]; + +} TLV_RAW; + +typedef struct tlv +{ + /* Type of the TLV */ + uint16_t type; + + /* Length of the TLV data */ + uint32_t length; + + /* Pointer to data if length > 0. */ + const uint8_t *value; + +} TLV; + +/** + * Internal state when parsing a TLV data stream. + */ +typedef struct fuzz_parse_state +{ + /* Data stream */ + const uint8_t *data; + size_t data_len; + + /* Current position of our "cursor" in processing the data stream. */ + size_t data_pos; + +} FUZZ_PARSE_STATE; + +/** + * Data local to a fuzzing run. + */ +typedef struct fuzz_data +{ + /* CURL easy object */ + CURL *easy; + + /* Parser state */ + FUZZ_PARSE_STATE state; + + /* Current URL. */ + char *url; + + /* Response data and length */ + const uint8_t *rsp1_data; + size_t rsp1_data_len; + + /* Username and password */ + char *username; + char *password; + + /* Postfields */ + char *postfields; + +} FUZZ_DATA; + +/* Function prototypes */ +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); +uint32_t to_u32(uint8_t b[4]); +uint16_t to_u16(uint8_t b[2]); +int fuzz_initialize_fuzz_data(FUZZ_DATA *fuzz, + const uint8_t *data, + size_t data_len); +void fuzz_terminate_fuzz_data(FUZZ_DATA *fuzz); +void fuzz_free(void **ptr); +static curl_socket_t fuzz_open_socket(void *ptr, + curlsocktype purpose, + struct curl_sockaddr *address); +static int fuzz_sockopt_callback(void *ptr, + curl_socket_t curlfd, + curlsocktype purpose); +int fuzz_get_first_tlv(FUZZ_DATA *fuzz, TLV *tlv); +int fuzz_get_next_tlv(FUZZ_DATA *fuzz, TLV *tlv); +int fuzz_get_tlv_comn(FUZZ_DATA *fuzz, TLV *tlv); +int fuzz_parse_tlv(FUZZ_DATA *fuzz, TLV *tlv); +char *fuzz_tlv_to_string(TLV *tlv); + +/* Macros */ +#define FTRY(FUNC) \ + { \ + int _func_rc = (FUNC); \ + if (_func_rc) \ + { \ + rc = _func_rc; \ + goto EXIT_LABEL; \ + } \ + } + +#define FCHECK(COND) \ + { \ + if (!(COND)) \ + { \ + rc = 1; \ + goto EXIT_LABEL; \ + } \ + } diff --git a/tests/fuzz/generate_corpus.py b/tests/fuzz/generate_corpus.py new file mode 100755 index 000000000..0bb2eda3c --- /dev/null +++ b/tests/fuzz/generate_corpus.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# +# Simple script which generates corpus files. + +import argparse +import logging +import struct +import sys +sys.path.append("..") +import curl_test_data +log = logging.getLogger(__name__) + + +def generate_corpus(options): + td = curl_test_data.TestData("../data") + + with open(options.output, "wb") as f: + enc = TLVEncoder(f) + + # Write the URL to the file. + enc.write_string(enc.TYPE_URL, options.url) + + # Write the first response to the file. + if options.rsp1: + enc.write_bytes(enc.TYPE_RSP1, options.rsp1.encode("utf-8")) + + elif options.rsp1file: + with open(options.rsp1file, "rb") as g: + enc.write_bytes(enc.TYPE_RSP1, g.read()) + + elif options.rsp1test: + wstring = td.get_test_data(options.rsp1test) + enc.write_bytes(enc.TYPE_RSP1, wstring.encode("utf-8")) + + # Write other options to file. + enc.maybe_write_string(enc.TYPE_USERNAME, options.username) + enc.maybe_write_string(enc.TYPE_PASSWORD, options.password) + enc.maybe_write_string(enc.TYPE_POSTFIELDS, options.postfields) + + return ScriptRC.SUCCESS + + +class TLVEncoder(object): + TYPE_URL = 1 + TYPE_RSP1 = 2 + TYPE_USERNAME = 3 + TYPE_PASSWORD = 4 + TYPE_POSTFIELDS = 5 + + def __init__(self, output): + self.output = output + + def write_string(self, tlv_type, wstring): + data = wstring.encode("utf-8") + self.write_tlv(tlv_type, len(data), data) + + def write_bytes(self, tlv_type, bytedata): + self.write_tlv(tlv_type, len(bytedata), bytedata) + + def maybe_write_string(self, tlv_type, wstring): + if wstring: + self.write_string(tlv_type, wstring) + + def write_tlv(self, tlv_type, tlv_length, tlv_data=None): + log.debug("Writing TLV %d, length %d, data %r", + tlv_type, + tlv_length, + tlv_data) + + data = struct.pack("!H", tlv_type) + self.output.write(data) + + data = struct.pack("!L", tlv_length) + self.output.write(data) + + if tlv_data: + self.output.write(tlv_data) + + +def get_options(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + parser.add_argument("--url", required=True) + parser.add_argument("--username") + parser.add_argument("--password") + parser.add_argument("--postfields") + + rsp1 = parser.add_mutually_exclusive_group(required=True) + rsp1.add_argument("--rsp1") + rsp1.add_argument("--rsp1file") + rsp1.add_argument("--rsp1test", type=int) + + return parser.parse_args() + + +def setup_logging(): + """ + Set up logging from the command line options + """ + root_logger = logging.getLogger() + formatter = logging.Formatter("%(asctime)s %(levelname)-5.5s %(message)s") + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setFormatter(formatter) + stdout_handler.setLevel(logging.DEBUG) + root_logger.addHandler(stdout_handler) + root_logger.setLevel(logging.DEBUG) + + +class ScriptRC(object): + """Enum for script return codes""" + SUCCESS = 0 + FAILURE = 1 + EXCEPTION = 2 + + +class ScriptException(Exception): + pass + + +def main(): + # Get the options from the user. + options = get_options() + + setup_logging() + + # Run main script. + try: + rc = generate_corpus(options) + except Exception as e: + log.exception(e) + rc = ScriptRC.EXCEPTION + + log.info("Returning %d", rc) + return rc + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tests/fuzz/standalone_fuzz_target_runner.c b/tests/fuzz/standalone_fuzz_target_runner.c new file mode 100644 index 000000000..c131a2192 --- /dev/null +++ b/tests/fuzz/standalone_fuzz_target_runner.c @@ -0,0 +1,89 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2017, Max Dymond, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +#include +#include +#include + +#include "standalone_fuzz_target_runner.h" + +/** + * Main procedure for standalone fuzzing engine. + * + * Reads filenames from the argument array. For each filename, read the file + * into memory and then call the fuzzing interface with the data. + */ +int main(int argc, char **argv) +{ + int ii; + FILE *infile; + uint8_t *buffer = NULL; + size_t buffer_len; + + for(ii = 1; ii < argc; ii++) { + /* Try and open the file. */ + infile = fopen(argv[ii], "rb"); + if(infile) { + printf("[%s] Open succeeded! \n", argv[ii]); + + /* Get the length of the file. */ + fseek(infile, 0L, SEEK_END); + buffer_len = ftell(infile); + + /* Reset the file indicator to the beginning of the file. */ + fseek(infile, 0L, SEEK_SET); + + /* Allocate a buffer for the file contents. */ + buffer = (uint8_t *)calloc(buffer_len, sizeof(uint8_t)); + if(buffer) { + /* Read all the text from the file into the buffer. */ + fread(buffer, sizeof(uint8_t), buffer_len, infile); + printf("[%s] Read %zu bytes, calling fuzzer\n", argv[ii], buffer_len); + + /* Call the fuzzer with the data. */ + LLVMFuzzerTestOneInput(buffer, buffer_len); + + printf("[%s] Fuzzing complete\n", argv[ii]); + + /* Free the buffer as it's no longer needed. */ + free(buffer); + buffer = NULL; + } + else + { + fprintf(stderr, + "[%s] Failed to allocate %zu bytes \n", + argv[ii], + buffer_len); + } + + /* Close the file as it's no longer needed. */ + fclose(infile); + infile = NULL; + } + else + { + /* Failed to open the file. Maybe wrong name or wrong permissions? */ + fprintf(stderr, "[%s] Open failed. \n", argv[ii]); + } + } +} diff --git a/tests/fuzz/standalone_fuzz_target_runner.h b/tests/fuzz/standalone_fuzz_target_runner.h new file mode 100644 index 000000000..37302618b --- /dev/null +++ b/tests/fuzz/standalone_fuzz_target_runner.h @@ -0,0 +1,23 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 2017, Max Dymond, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); \ No newline at end of file