/*****************************************************************************
- * _ _ ____ _
- * Project ___| | | | _ \| |
- * / __| | | | |_) | |
- * | (__| |_| | _ <| |___
- * \___|\___/|_| \_\_____|
*
- * $Id$
+ * This example source code introduces a c library buffered I/O interface to
+ * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
+ * rewind(). Supported functions have identical prototypes to their normal c
+ * lib namesakes and are preceaded by url_ .
*
- * This example source code introduces an fopen()/fread()/fclose() emulation
- * for URL reads. Using an approach similar to this, you could replace your
- * program's fopen() with this url_fopen() and fread() with url_fread() and
- * it should be possible to read remote streams instead of (only) local files.
+ * Using this code you can replace your program's fopen() with url_fopen()
+ * and fread() with url_fread() and it become possible to read remote streams
+ * instead of (only) local files. Local files (ie those that can be directly
+ * fopened) will drop back to using the underlying clib implementations
*
- * See the main() function at the bottom that shows a tiny app in action.
+ * See the main() function at the bottom that shows an app that retrives from a
+ * specified url using fgets() and fread() and saves as two output files.
*
- * This source code is a proof of concept. It will need further attention to
- * become production-use useful and solid.
+ * Coyright (c)2003 Simtec Electronics
+ *
+ * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
+ * reference to original curl example code
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This example requires libcurl 7.9.7 or later.
*/
+
#include <stdio.h>
#include <string.h>
#include <sys/time.h>
#include <stdlib.h>
+#include <errno.h>
#include <curl/curl.h>
#error "too old libcurl version, get the latest!"
#endif
-struct data {
- int type;
- union {
- CURL *curl;
- FILE *file;
- } handle;
-
- /* This is the documented biggest possible buffer chunk we can get from
- libcurl in one single callback! */
- char buffer[CURL_MAX_WRITE_SIZE];
- char *readptr; /* read from here */
- int bytes; /* bytes available from read pointer */
+enum fcurl_type_e { CFTYPE_NONE=0, CFTYPE_FILE=1, CFTYPE_CURL=2 };
- CURLMcode m; /* stored from a previous url_fread() */
+struct fcurl_data
+{
+ enum fcurl_type_e type; /* type of handle */
+ union {
+ CURL *curl;
+ FILE *file;
+ } handle; /* handle */
+
+ char *buffer; /* buffer to store cached data*/
+ int buffer_len; /* currently allocated buffers length */
+ int buffer_pos; /* end of data in buffer*/
+ int still_running; /* Is background url fetch still in progress */
};
-typedef struct data URL_FILE;
+typedef struct fcurl_data URL_FILE;
+
+/* exported functions */
+URL_FILE *url_fopen(char *url,const char *operation);
+int url_fclose(URL_FILE *file);
+int url_feof(URL_FILE *file);
+size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
+char * url_fgets(char *ptr, int size, URL_FILE *file);
+void url_rewind(URL_FILE *file);
/* we use a global one for convenience */
CURLM *multi_handle;
-static
-size_t write_callback(char *buffer,
- size_t size,
- size_t nitems,
- void *userp)
+/* curl calls this routine to get more data */
+static size_t
+write_callback(char *buffer,
+ size_t size,
+ size_t nitems,
+ void *userp)
{
- URL_FILE *url = (URL_FILE *)userp;
- size *= nitems;
+ char *newbuff;
+ int rembuff;
+
+ URL_FILE *url = (URL_FILE *)userp;
+ size *= nitems;
+
+ rembuff=url->buffer_len - url->buffer_pos;//remaining space in buffer
+
+ if(size > rembuff)
+ {
+ //not enuf space in buffer
+ newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
+ if(newbuff==NULL)
+ {
+ fprintf(stderr,"callback buffer grow failed\n");
+ size=rembuff;
+ }
+ else
+ {
+ /* realloc suceeded increase buffer size*/
+ url->buffer_len+=size - rembuff;
+ url->buffer=newbuff;
+
+ /*printf("Callback buffer grown to %d bytes\n",url->buffer_len);*/
+ }
+ }
+
+ memcpy(&url->buffer[url->buffer_pos], buffer, size);
+ url->buffer_pos += size;
- memcpy(url->readptr, buffer, size);
- url->readptr += size;
- url->bytes += size;
+ /*fprintf(stderr, "callback %d size bytes\n", size);*/
+
+ return size;
+}
- fprintf(stderr, "callback %d size bytes\n", size);
+/* use to attempt to fill the read buffer up to requested number of bytes */
+static int
+curl_fill_buffer(URL_FILE *file,int want,int waittime)
+{
+ fd_set fdread;
+ fd_set fdwrite;
+ fd_set fdexcep;
+ int maxfd;
+ struct timeval timeout;
+ int rc;
+
+ /* only attempt to fill buffer if transactions still running and buffer
+ * doesnt exceed required size already
+ */
+ if((!file->still_running) || (file->buffer_pos > want))
+ return 0;
+
+ /* attempt to fill buffer */
+ do
+ {
+ FD_ZERO(&fdread);
+ FD_ZERO(&fdwrite);
+ FD_ZERO(&fdexcep);
+
+ /* set a suitable timeout to fail on */
+ timeout.tv_sec = 60; /* 1 minute */
+ timeout.tv_usec = 0;
+
+ /* get file descriptors from the transfers */
+ curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+
+ rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
+
+ switch(rc) {
+ case -1:
+ /* select error */
+ break;
+
+ case 0:
+ break;
+
+ default:
+ /* timeout or readable/writable sockets */
+ /* note we *could* be more efficient and not wait for
+ * CURLM_CALL_MULTI_PERFORM to clear here and check it on re-entry
+ * but that gets messy */
+ while(curl_multi_perform(multi_handle, &file->still_running) ==
+ CURLM_CALL_MULTI_PERFORM);
+
+ break;
+ }
+ } while(file->still_running && (file->buffer_pos < want));
+ return 1;
+}
- return size;
+/* use to remove want bytes from the front of a files buffer */
+static int
+curl_use_buffer(URL_FILE *file,int want)
+{
+ /* sort out buffer */
+ if((file->buffer_pos - want) <=0)
+ {
+ /* ditch buffer - write will recreate */
+ if(file->buffer)
+ free(file->buffer);
+
+ file->buffer=NULL;
+ file->buffer_pos=0;
+ file->buffer_len=0;
+ }
+ else
+ {
+ /* move rest down make it available for later */
+ memmove(file->buffer,
+ &file->buffer[want],
+ (file->buffer_pos - want));
+
+ file->buffer_pos -= want;
+ }
+ return 0;
}
-URL_FILE *url_fopen(char *url, char *operation)
+
+
+URL_FILE *
+url_fopen(char *url,const char *operation)
{
- /* this code could check for URLs or types in the 'url' and
- basicly use the real fopen() for standard files */
+ /* this code could check for URLs or types in the 'url' and
+ basicly use the real fopen() for standard files */
+
+ URL_FILE *file;
+ (void)operation;
+
+ file = (URL_FILE *)malloc(sizeof(URL_FILE));
+ if(!file)
+ return NULL;
- URL_FILE *file;
- int still_running;
- (void)operation;
+ memset(file, 0, sizeof(URL_FILE));
+
+ if((file->handle.file=fopen(url,operation)))
+ {
+ file->type = CFTYPE_FILE; /* marked as URL */
+ }
+ else
+ {
+ file->type = CFTYPE_CURL; /* marked as URL */
+ file->handle.curl = curl_easy_init();
- file = (URL_FILE *)malloc(sizeof(URL_FILE));
- if(!file)
- return NULL;
+ curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
+ curl_easy_setopt(file->handle.curl, CURLOPT_FILE, file);
+ curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE);
+ curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
- memset(file, 0, sizeof(URL_FILE));
+ if(!multi_handle)
+ multi_handle = curl_multi_init();
- file->type = 1; /* marked as URL, use 0 for plain file */
- file->handle.curl = curl_easy_init();
+ curl_multi_add_handle(multi_handle, file->handle.curl);
- curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
- curl_easy_setopt(file->handle.curl, CURLOPT_FILE, file);
- curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE);
- curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
+ /* lets start the fetch */
+ while(curl_multi_perform(multi_handle, &file->still_running) ==
+ CURLM_CALL_MULTI_PERFORM );
- if(!multi_handle)
- multi_handle = curl_multi_init();
+ if((file->buffer_pos == 0) && (!file->still_running))
+ {
+ /* if still_running is 0 now, we should return NULL */
- curl_multi_add_handle(multi_handle, file->handle.curl);
+ /* make sure the easy handle is not in the multi handle anymore */
+ curl_multi_remove_handle(multi_handle, file->handle.curl);
- while(CURLM_CALL_MULTI_PERFORM ==
- curl_multi_perform(multi_handle, &still_running));
+ /* cleanup */
+ curl_easy_cleanup(file->handle.curl);
- /* if still_running would be 0 now, we should return NULL */
+ free(file);
- return file;
+ file = NULL;
+ }
+ }
+ return file;
}
-void url_fclose(URL_FILE *file)
+int
+url_fclose(URL_FILE *file)
{
- /* make sure the easy handle is not in the multi handle anymore */
- curl_multi_remove_handle(multi_handle, file->handle.curl);
+ int ret=0;/* default is good return */
- /* cleanup */
- curl_easy_cleanup(file->handle.curl);
-}
+ switch(file->type)
+ {
+ case CFTYPE_FILE:
+ ret=fclose(file->handle.file); /* passthrough */
+ break;
+
+ case CFTYPE_CURL:
+ /* make sure the easy handle is not in the multi handle anymore */
+ curl_multi_remove_handle(multi_handle, file->handle.curl);
+ /* cleanup */
+ curl_easy_cleanup(file->handle.curl);
+ break;
+
+ default: /* unknown or supported type - oh dear */
+ ret=EOF;
+ errno=EBADF;
+ break;
+
+ }
+ if(file->buffer)
+ free(file->buffer);/* free any allocated buffer space */
-size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
+ free(file);
+
+ return ret;
+}
+
+int
+url_feof(URL_FILE *file)
{
- fd_set fdread;
- fd_set fdwrite;
- fd_set fdexcep;
- int maxfd;
- struct timeval timeout;
- int rc;
- int still_running = 0;
-
- if(!file->bytes) { /* no data available at this point */
-
- file->readptr = file->buffer; /* reset read pointer */
-
- if(CURLM_CALL_MULTI_PERFORM == file->m) {
- while(CURLM_CALL_MULTI_PERFORM ==
- curl_multi_perform(multi_handle, &still_running)) {
- if(file->bytes) {
- printf("(fread) WOAH! THis happened!\n");
- break;
- }
- }
- if(!still_running) {
- printf("DONE RUNNING AROUND!\n");
- return 0;
- }
+ int ret=0;
+
+ switch(file->type)
+ {
+ case CFTYPE_FILE:
+ ret=feof(file->handle.file);
+ break;
+
+ case CFTYPE_CURL:
+ if((file->buffer_pos == 0) && (!file->still_running))
+ ret = 1;
+ break;
+ default: /* unknown or supported type - oh dear */
+ ret=-1;
+ errno=EBADF;
+ break;
}
+ return ret;
+}
- do {
+size_t
+url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
+{
+ size_t want;
+
+ switch(file->type)
+ {
+ case CFTYPE_FILE:
+ want=fread(ptr,size,nmemb,file->handle.file);
+ break;
+
+ case CFTYPE_CURL:
+ want = nmemb * size;
+
+ curl_fill_buffer(file,want,1);
+
+ /* check if theres data in the buffer - if not curl_fill_buffer()
+ * either errored or EOF */
+ if(!file->buffer_pos)
+ return 0;
+
+ /* ensure only available data is considered */
+ if(file->buffer_pos < want)
+ want = file->buffer_pos;
- FD_ZERO(&fdread);
- FD_ZERO(&fdwrite);
- FD_ZERO(&fdexcep);
-
- /* set a suitable timeout to fail on */
- timeout.tv_sec = 500; /* 5 minutes */
- timeout.tv_usec = 0;
-
- /* get file descriptors from the transfers */
- curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
-
- rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
-
- switch(rc) {
- case -1:
- /* select error */
- break;
- case 0:
- break;
- default:
- /* timeout or readable/writable sockets */
- printf("select() returned %d!\n", rc);
- do {
- file->m = curl_multi_perform(multi_handle, &still_running);
-
- if(file->bytes)
- /* we have received data, return that now */
- break;
-
- } while(CURLM_CALL_MULTI_PERFORM == file->m);
-
-
- if(!still_running)
- printf("DONE RUNNING AROUND!\n");
-
- break;
- }
- } while(still_running && (file->bytes <= 0));
- }
- else
- printf("(fread) Skip network read\n");
-
- if(file->bytes) {
- /* data already available, return that */
- int want = size * nmemb;
-
- if(file->bytes < want)
- want = file->bytes;
-
- memcpy(ptr, file->readptr, want);
- file->readptr += want;
- file->bytes -= want;
-
- printf("(fread) return %d bytes\n", want);
+ /* xfer data to caller */
+ memcpy(ptr, file->buffer, want);
+ curl_use_buffer(file,want);
+
+ want = want / size; /* number of items - nb correct op - checked
+ * with glibc code*/
+
+ /*printf("(fread) return %d bytes %d left\n", want,file->buffer_pos);*/
+ break;
+
+ default: /* unknown or supported type - oh dear */
+ want=0;
+ errno=EBADF;
+ break;
+
+ }
return want;
- }
- return 0; /* no data available to return */
}
+char *
+url_fgets(char *ptr, int size, URL_FILE *file)
+{
+ int want = size - 1;/* always need to leave room for zero termination */
+ int loop;
+
+ switch(file->type)
+ {
+ case CFTYPE_FILE:
+ ptr = fgets(ptr,size,file->handle.file);
+ break;
+
+ case CFTYPE_CURL:
+ curl_fill_buffer(file,want,1);
+
+ /* check if theres data in the buffer - if not fill either errored or
+ * EOF */
+ if(!file->buffer_pos)
+ return NULL;
+
+ /* ensure only available data is considered */
+ if(file->buffer_pos < want)
+ want = file->buffer_pos;
+
+ /*buffer contains data */
+ /* look for newline or eof */
+ for(loop=0;loop < want;loop++)
+ {
+ if(file->buffer[loop] == '\n')
+ {
+ want=loop+1;/* include newline */
+ break;
+ }
+ }
+
+ /* xfer data to caller */
+ memcpy(ptr, file->buffer, want);
+ ptr[want]=0;/* allways null terminate */
+
+ curl_use_buffer(file,want);
+
+ /*printf("(fgets) return %d bytes %d left\n", want,file->buffer_pos);*/
+ break;
+
+ default: /* unknown or supported type - oh dear */
+ ptr=NULL;
+ errno=EBADF;
+ break;
+ }
+
+ return ptr;/*success */
+}
+
+void
+url_rewind(URL_FILE *file)
+{
+ switch(file->type)
+ {
+ case CFTYPE_FILE:
+ rewind(file->handle.file); /* passthrough */
+ break;
+
+ case CFTYPE_CURL:
+ /* halt transaction */
+ curl_multi_remove_handle(multi_handle, file->handle.curl);
+
+ /* restart */
+ curl_multi_add_handle(multi_handle, file->handle.curl);
+
+ /* ditch buffer - write will recreate - resets stream pos*/
+ if(file->buffer)
+ free(file->buffer);
+
+ file->buffer=NULL;
+ file->buffer_pos=0;
+ file->buffer_len=0;
+
+ break;
-int main(int argc, char *argv[])
+ default: /* unknown or supported type - oh dear */
+ break;
+
+ }
+
+}
+
+
+/* Small main program to retrive from a url using fgets and fread saving the
+ * output to two test files (note the fgets method will corrupt binary files if
+ * they contain 0 chars */
+int
+main(int argc, char *argv[])
{
- URL_FILE *handle;
- int nread;
- char buffer[256];
+ URL_FILE *handle;
+ FILE *outf;
+
+ int nread;
+ char buffer[256];
+ char *url;
+
+ if(argc < 2)
+ {
+ url="http://192.168.7.3/testfile";/* default to testurl */
+ }
+ else
+ {
+ url=argv[1];/* use passed url */
+ }
+
+ /* copy from url line by line with fgets */
+ outf=fopen("fgets.test","w+");
+ if(!outf)
+ {
+ perror("couldnt open fgets output file\n");
+ return 1;
+ }
+
+ handle = url_fopen(url, "r");
+ if(!handle)
+ {
+ printf("couldn't url_fopen()\n");
+ fclose(outf);
+ return 2;
+ }
+
+ while(!url_feof(handle))
+ {
+ url_fgets(buffer,sizeof(buffer),handle);
+ fwrite(buffer,1,strlen(buffer),outf);
+ }
+
+ url_fclose(handle);
+
+ fclose(outf);
+
+
+ /* Copy from url with fread */
+ outf=fopen("fread.test","w+");
+ if(!outf)
+ {
+ perror("couldnt open fread output file\n");
+ return 1;
+ }
+
+ handle = url_fopen("testfile", "r");
+ if(!handle) {
+ printf("couldn't url_fopen()\n");
+ fclose(outf);
+ return 2;
+ }
+
+ do {
+ nread = url_fread(buffer, 1,sizeof(buffer), handle);
+ fwrite(buffer,1,nread,outf);
+ } while(nread);
+
+ url_fclose(handle);
+
+ fclose(outf);
+
+
+ /* Test rewind */
+ outf=fopen("rewind.test","w+");
+ if(!outf)
+ {
+ perror("couldnt open fread output file\n");
+ return 1;
+ }
+
+ handle = url_fopen("testfile", "r");
+ if(!handle) {
+ printf("couldn't url_fopen()\n");
+ fclose(outf);
+ return 2;
+ }
+
+ nread = url_fread(buffer, 1,sizeof(buffer), handle);
+ fwrite(buffer,1,nread,outf);
+ url_rewind(handle);
- (void)argc;
- (void)argv;
+ buffer[0]='\n';
+ fwrite(buffer,1,1,outf);
- handle = url_fopen("http://curl.haxx.se/", "r");
+ nread = url_fread(buffer, 1,sizeof(buffer), handle);
+ fwrite(buffer,1,nread,outf);
- if(!handle) {
- printf("couldn't url_fopen()\n");
- }
- do {
- nread = url_fread(buffer, sizeof(buffer), 1, handle);
+ url_fclose(handle);
- printf("We got: %d bytes\n", nread);
- } while(nread);
+ fclose(outf);
- url_fclose(handle);
- return 0;
+ return 0;/* all done */
}
projects / curl / commitdiff