Changelog
Daniel S (10 Jan 2008)
+- Georg Lippitsch brought CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA to allow
+ libcurl to seek in a given input stream. This is particularly important when
+ doing upload resumes when there's already a huge part of the file present
+ remotely. Before, and still if this callback isn't used, libcurl will read
+ and through away the entire file up to the point to where the resuming
+ begins (which of course can be a slow opereration depending on file size,
+ I/O bandwidth and more). This new function will also be preferred to get
+ used instead of the CURLOPT_IOCTLFUNCTION for seeking back in a stream when
+ doing multi-stage HTTP auth with POST/PUT.
+
- Nikitinskit Dmitriy filed bug report #1868255
(http://curl.haxx.se/bug/view.cgi?id=1868255) with a patch. It identifies
and fixes a problem with parsing WWW-Authenticate: headers with additional
Public curl releases: 103
Command line options: 125
- curl_easy_setopt() options: 148
+ curl_easy_setopt() options: 150
Public functions in libcurl: 56
Public web site mirrors: 42
Known libcurl bindings: 36
This release includes the following changes:
- o --data-urlencode was added
- o CURLOPT_PROXY_TRANSFER_MODE was added
- o --no-keep-alive was added, since starting now curl is doing connections with
- keep-alive enabled by default
+ o --data-urlencode
+ o CURLOPT_PROXY_TRANSFER_MODE
+ o --no-keep-alive - now curl does connections with keep-alive enabled by
+ default
o --socks4a added (proxy type CURLPROXY_SOCKS4A for libcurl)
o --socks5-hostname added (CURLPROXY_SOCKS5_HOSTNAME for libcurl)
- o curl_easy_pause() added
+ o curl_easy_pause()
+ o CURLOPT_SEEKFUNCTION and CURLOPT_SEEKDATA
This release includes the following bugfixes:
Emil Romanus, Alessandro Vesely, Ray Pekowski, Spacen Jasset, Andrew Moise,
Gilles Blanc, David Wright, Vikram Saxena, Mateusz Loskot, Gary Maxwell,
Dmitry Kurochkin, Mohun Biswas, Richard Atterer, Maxim Perenesenko,
- Daniel Egger, Jeff Johnson
+ Daniel Egger, Jeff Johnson, Nikitinskit Dmitriy, Georg Lippitsch
Thanks! (and sorry if I forgot to mention someone)
itself. For now, rewinding the read data stream is the only action it can
request. The rewinding of the read data stream may be necessary when doing a
HTTP PUT or POST with a multi-pass authentication method. (Option added in
-7.12.3)
+7.12.3).
+
+Use \fICURLOPT_SEEKFUNCTION\fP instead to provide seeking!
.IP CURLOPT_IOCTLDATA
Pass a pointer that will be untouched by libcurl and passed as the 3rd
argument in the ioctl callback set with \fICURLOPT_IOCTLFUNCTION\fP. (Option
added in 7.12.3)
+.IP CURLOPT_SEEKFUNCTION
+Function pointer that should match the following prototype: \fIint
+function(void *instream, curl_off_t offset, int origin);\fP This function gets
+called by libcurl to seek to a certain position in the input stream and can be
+used to fast forward a file in a resumed upload (instead of reading all
+uploaded bytes with the normal read function/callback). It is also called to
+rewind a stream when doing a HTTP PUT or POST with a multi-pass authentication
+method. The function shall work like "fseek" or "lseek" and accepted SEEK_SET,
+SEEK_CUR and SEEK_END as argument for origin, although (in 7.18.0) libcurl
+only passes SEEK_SET. The callback must return 0 on success as returning
+non-zero will cause the upload operation to fail.
+
+If you forward the input arguments directly to "fseek" or "lseek", note that
+the data type for \fIoffset\fP is not the same as defined for curl_off_t on
+many systems! (Option added in 7.18.0)
+.IP CURLOPT_SEEKDATA
+Data pointer to pass to the file read function. If you use the
+\fICURLOPT_SEEKFUNCTION\fP option, this is the pointer you'll get as input. If
+you don't specify a seek callback, NULL is passed. (Option added in 7.18.0)
.IP CURLOPT_SOCKOPTFUNCTION
Function pointer that should match the \fIcurl_sockopt_callback\fP prototype
found in \fI<curl/curl.h>\fP. This function gets called by libcurl after the
/* This is a return code for the read callback that, when returned, will
signal libcurl to pause sending data on the current transfer. */
#define CURL_READFUNC_PAUSE 0x10000001
+typedef int (*curl_seek_callback)(void *instream,
+ curl_off_t offset,
+ int origin); /* 'whence' */
+
typedef size_t (*curl_read_callback)(char *buffer,
size_t size,
size_t nitems,
/* set transfer mode (;type=<a|i>) when doing FTP via an HTTP proxy */
CINIT(PROXY_TRANSFER_MODE, LONG, 166),
+ /* Callback function for seeking in the input stream */
+ CINIT(SEEKFUNCTION, FUNCTIONPOINT, 167),
+ CINIT(SEEKDATA, OBJECTPOINT, 168),
+
CURLOPT_LASTENTRY /* the last unused */
} CURLoption;
struct FTP *ftp = conn->data->state.proto.ftp;
struct SessionHandle *data = conn->data;
struct ftp_conn *ftpc = &conn->proto.ftpc;
- curl_off_t passed=0;
if((data->state.resume_from && !sizechecked) ||
((data->state.resume_from > 0) && sizechecked)) {
/* enable append */
data->set.ftp_append = TRUE;
- /* Let's read off the proper amount of bytes from the input. If we knew it
- was a proper file we could've just fseek()ed but we only have a stream
- here */
+ /* Let's read off the proper amount of bytes from the input. */
+ if(conn->seek_func) {
+ curl_off_t readthisamountnow = data->state.resume_from;
- /* TODO: allow the ioctlfunction to provide a fast forward function that
- can be used here and use this method only as a fallback! */
- do {
- curl_off_t readthisamountnow = (data->state.resume_from - passed);
- curl_off_t actuallyread;
-
- if(readthisamountnow > BUFSIZE)
- readthisamountnow = BUFSIZE;
-
- actuallyread = (curl_off_t)
- conn->fread_func(data->state.buffer, 1, (size_t)readthisamountnow,
- conn->fread_in);
-
- passed += actuallyread;
- if((actuallyread <= 0) || (actuallyread > readthisamountnow)) {
- /* this checks for greater-than only to make sure that the
- CURL_READFUNC_ABORT return code still aborts */
- failf(data, "Failed to read data");
+ if(conn->seek_func(conn->seek_client,
+ readthisamountnow, SEEK_SET) != 0) {
+ failf(data, "Could not seek stream");
return CURLE_FTP_COULDNT_USE_REST;
}
- } while(passed < data->state.resume_from);
+ }
+
+ else {
+ curl_off_t passed=0;
+ do {
+ curl_off_t readthisamountnow = (data->state.resume_from - passed);
+ curl_off_t actuallyread;
+
+ if(readthisamountnow > BUFSIZE)
+ readthisamountnow = BUFSIZE;
+
+ actuallyread = (curl_off_t)
+ conn->fread_func(data->state.buffer, 1, (size_t)readthisamountnow,
+ conn->fread_in);
+
+ passed += actuallyread;
+ if((actuallyread <= 0) || (actuallyread > readthisamountnow)) {
+ /* this checks for greater-than only to make sure that the
+ CURL_READFUNC_ABORT return code still aborts */
+ failf(data, "Failed to read data");
+ return CURLE_FTP_COULDNT_USE_REST;
+ }
+ } while(passed < data->state.resume_from);
+ }
/* now, decrease the size of the read */
if(data->set.infilesize>0) {
* | (__| |_| | _ <| |___
* \___|\___/|_| \_\_____|
*
- * Copyright (C) 1998 - 2007, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2008, Daniel Stenberg, <daniel@haxx.se>, et al.
*
* This software is licensed as described in the file COPYING, which
* you should have received as part of this distribution. The terms
/* set the proper values (possibly modified on POST) */
conn->fread_func = data->set.fread_func; /* restore */
conn->fread_in = data->set.in; /* restore */
+ conn->seek_func = data->set.seek_func; /* restore */
+ conn->seek_client = data->set.seek_client; /* restore */
if(http == NULL)
return CURLE_OK;
if(data->state.resume_from && !data->state.this_is_a_follow) {
/* do we still game? */
- curl_off_t passed=0;
/* Now, let's read off the proper amount of bytes from the
- input. If we knew it was a proper file we could've just
- fseek()ed but we only have a stream here */
- do {
- size_t readthisamountnow = (size_t)(data->state.resume_from - passed);
- size_t actuallyread;
-
- if(readthisamountnow > BUFSIZE)
- readthisamountnow = BUFSIZE;
-
- actuallyread =
- data->set.fread_func(data->state.buffer, 1, (size_t)readthisamountnow,
- data->set.in);
-
- passed += actuallyread;
- if(actuallyread != readthisamountnow) {
- failf(data, "Could only read %" FORMAT_OFF_T
- " bytes from the input",
- passed);
+ input. */
+ if(conn->seek_func) {
+ curl_off_t readthisamountnow = data->state.resume_from;
+
+ if(conn->seek_func(conn->seek_client,
+ readthisamountnow, SEEK_SET) != 0) {
+ failf(data, "Could not seek stream");
return CURLE_READ_ERROR;
}
- } while(passed != data->state.resume_from); /* loop until done */
+ }
+ else {
+ curl_off_t passed=0;
+
+ do {
+ size_t readthisamountnow = (size_t)(data->state.resume_from - passed);
+ size_t actuallyread;
+
+ if(readthisamountnow > BUFSIZE)
+ readthisamountnow = BUFSIZE;
+
+ actuallyread = data->set.fread_func(data->state.buffer, 1,
+ (size_t)readthisamountnow,
+ data->set.in);
+
+ passed += actuallyread;
+ if(actuallyread != readthisamountnow) {
+ failf(data, "Could only read %" FORMAT_OFF_T
+ " bytes from the input",
+ passed);
+ return CURLE_READ_ERROR;
+ }
+ } while(passed != data->state.resume_from); /* loop until done */
+ }
/* now, decrease the size of the read */
if(data->set.infilesize>0) {
(data->set.httpreq == HTTPREQ_POST_FORM))
; /* do nothing */
else {
- if(data->set.ioctl_func) {
+ if(data->set.seek_func) {
+ int err;
+
+ err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
+ if(err) {
+ failf(data, "seek callback returned error %d\n", (int)err);
+ return CURLE_SEND_FAIL_REWIND;
+ }
+ }
+ else if(data->set.ioctl_func) {
curlioerr err;
- err = (data->set.ioctl_func) (data, CURLIOCMD_RESTARTREAD,
- data->set.ioctl_client);
+ err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
+ data->set.ioctl_client);
infof(data, "the ioctl callback returned %d\n", (int)err);
if(err) {
/* use fread as default function to read input */
data->set.fread_func = (curl_read_callback)fread;
+ /* don't use a seek function by default */
+ data->set.seek_func = ZERO_NULL;
+ data->set.seek_client = ZERO_NULL;
+
/* conversion callbacks for non-ASCII hosts */
data->set.convfromnetwork = ZERO_NULL;
data->set.convtonetwork = ZERO_NULL;
/* When set to NULL, reset to our internal default function */
data->set.fread_func = (curl_read_callback)fread;
break;
+ case CURLOPT_SEEKFUNCTION:
+ /*
+ * Seek callback. Might be NULL.
+ */
+ data->set.seek_func = va_arg(param, curl_seek_callback);
+ break;
+ case CURLOPT_SEEKDATA:
+ /*
+ * Seek control callback. Might be NULL.
+ */
+ data->set.seek_client = va_arg(param, void *);
+ break;
case CURLOPT_CONV_FROM_NETWORK_FUNCTION:
/*
* "Convert from network encoding" callback
* the persistent connection stuff */
conn->fread_func = data->set.fread_func;
conn->fread_in = data->set.in;
+ conn->seek_func = data->set.seek_func;
+ conn->seek_client = data->set.seek_client;
if((conn->protocol&PROT_HTTP) &&
data->set.upload &&
size_t buf_len; /* Length of the buffer?? */
+ curl_seek_callback seek_func; /* function that seeks the input */
+ void *seek_client; /* pointer to pass to the seek() above */
+
/*************** Request - specific items ************/
/* previously this was in the urldata struct */
bool free_referer; /* set TRUE if 'referer' points to a string we
allocated */
void *postfields; /* if POST, set the fields' values here */
+ curl_seek_callback seek_func; /* function that seeks the input */
curl_off_t postfieldsize; /* if POST, this might have a size to use instead
of strlen(), and then the data *may* be binary
(contain zero bytes) */
the address and opening the socket */
void* opensocket_client;
+ void *seek_client; /* pointer to pass to the seek callback */
/* the 3 curl_conv_callback functions below are used on non-ASCII hosts */
/* function to convert from the network encoding: */
curl_conv_callback convfromnetwork;
struct Configurable *config;
};
+#if 1
static curlioerr my_ioctl(CURL *handle, curliocmd cmd, void *userp)
{
struct InStruct *in=(struct InStruct *)userp;
}
return CURLIOE_OK;
}
+#else
+static int my_seek(void *stream, curl_off_t offset, int whence)
+{
+ struct InStruct *in=(struct InStruct *)stream;
+
+ /* We can't use fseek() here since it can't do 64bit seeks on Windows and
+ possibly elsewhere. We need to switch to the lseek family of tricks. For
+ that to work, we need to switch from fread() to plain read() etc */
+
+ if(-1 == fseek(in->stream, (off_t)offset, whence))
+ /* couldn't rewind, the reason is in errno but errno is just not
+ portable enough and we don't actually care that much why we failed. */
+ return 1;
+
+ return 0;
+}
+
+#endif
static size_t my_fread(void *buffer, size_t sz, size_t nmemb, void *userp)
{
/* what call to read */
my_setopt(curl, CURLOPT_READFUNCTION, my_fread);
+#if 1
/* the ioctl function is at this point only used to rewind files
that are posted when using NTLM etc */
my_setopt(curl, CURLOPT_IOCTLDATA, &input);
my_setopt(curl, CURLOPT_IOCTLFUNCTION, my_ioctl);
+#else
+ /* in 7.18.0, the SEEKFUNCTION/DATA pair is taking over what IOCTL*
+ previously provided for seeking */
+ my_setopt(curl, CURLOPT_SEEKDATA, &input);
+ my_setopt(curl, CURLOPT_SEEKFUNCTION, my_seek);
+#endif
if(config->recvpersecond)
/* tell libcurl to use a smaller sized buffer as it allows us to
projects / curl / commitdiff