curl.c File Reference

API for downloading JSON via CURL. More...

#include "platform.h"
#include <jansson.h>
#include <microhttpd.h>
#include "gnunet_curl_lib.h"
#include "curl_internal.h"

Include dependency graph for curl.c:

Go to the source code of this file.

Data Structures
struct	GNUNET_CURL_Job
	Jobs are CURL requests running within a struct GNUNET_CURL_Context. More...
struct	GNUNET_CURL_Context
	Context. More...

Macros
#define	DEBUG   0
	Set to 1 for extra debug logging.
#define	CURL_STRERROR(type, function, code)
	Log error related to CURL operations.
#define	JSON_WARN(error)
	Print JSON parsing related error information.

Functions
void	GNUNET_CURL_set_userpass (struct GNUNET_CURL_Context *ctx, const char *userpass)
	Force use of the provided username and password for client authentication for all operations performed with ctx.
void	GNUNET_CURL_set_tlscert (struct GNUNET_CURL_Context *ctx, const char *certtype, const char *certfile, const char *keyfile, const char *keypass)
	Force use of the provided TLS client certificate for client authentication for all operations performed with ctx.
struct GNUNET_CURL_Context *	GNUNET_CURL_init (GNUNET_CURL_RescheduleCallback cb, void *cb_cls)
	Initialise this library.
void	GNUNET_CURL_enable_async_scope_header (struct GNUNET_CURL_Context *ctx, const char *header_name)
	Enable sending the async scope ID as a header.
enum GNUNET_GenericReturnValue	GNUNET_CURL_is_valid_scope_id (const char *scope_id)
	Return GNUNET_YES if given a valid scope ID and GNUNET_NO otherwise.
static size_t	download_cb (char *bufptr, size_t size, size_t nitems, void *cls)
	Callback used when downloading the reply to an HTTP request.
static struct curl_slist *	setup_job_headers (struct GNUNET_CURL_Context *ctx, const struct curl_slist *job_headers)
	Create the HTTP headers for the request.
static size_t	stream_write_cb (char *bufptr, size_t size, size_t nitems, void *cls)
	Callback used when the caller wants the response body as it arrives, instead of buffered.
static size_t	stream_header_cb (char *bufptr, size_t size, size_t nitems, void *cls)
	Callback used to pass the response's header lines on to a streaming caller.
static bool	setup_job_io (CURL *eh, struct GNUNET_CURL_Job *job)
	Point eh's body and header handling at job, in whichever of the two modes job was set up for.
static struct GNUNET_CURL_Job *	setup_job (CURL *eh, struct GNUNET_CURL_Context *ctx, struct curl_slist *all_headers, const struct GNUNET_CURL_StreamHandlers *sh)
	Create a job.
void	GNUNET_CURL_extend_headers (struct GNUNET_CURL_Job *job, const struct curl_slist *extra_headers)
	Add extra_headers to the HTTP headers for job.
struct GNUNET_CURL_Job *	GNUNET_CURL_job_add_raw (struct GNUNET_CURL_Context *ctx, CURL *eh, const struct curl_slist *job_headers, GNUNET_CURL_RawJobCompletionCallback jcc, void *jcc_cls)
	Schedule a CURL request to be executed and call the given jcc upon its completion.
struct GNUNET_CURL_Job *	GNUNET_CURL_job_add_stream (struct GNUNET_CURL_Context *ctx, CURL *eh, const struct curl_slist *job_headers, const struct GNUNET_CURL_StreamHandlers *sh)
	Schedule a CURL request whose response is delivered incrementally, rather than buffered in full and handed over at the end.
void	GNUNET_CURL_job_set_paused (struct GNUNET_CURL_Job *job, bool recv_paused, bool send_paused)
	Say which directions of job are to be paused from now on.
struct GNUNET_CURL_Job *	GNUNET_CURL_job_add2 (struct GNUNET_CURL_Context *ctx, CURL *eh, const struct curl_slist *job_headers, GNUNET_CURL_JobCompletionCallback jcc, void *jcc_cls)
	Schedule a CURL request to be executed and call the given jcc upon its completion.
struct GNUNET_CURL_Job *	GNUNET_CURL_job_add_with_ct_json (struct GNUNET_CURL_Context *ctx, CURL *eh, GNUNET_CURL_JobCompletionCallback jcc, void *jcc_cls)
	Schedule a CURL request to be executed and call the given jcc upon its completion.
struct GNUNET_CURL_Job *	GNUNET_CURL_job_add (struct GNUNET_CURL_Context *ctx, CURL *eh, GNUNET_CURL_JobCompletionCallback jcc, void *jcc_cls)
	Schedule a CURL request to be executed and call the given jcc upon its completion.
void	GNUNET_CURL_job_cancel (struct GNUNET_CURL_Job *job)
	Cancel a job.
static bool	is_json (const char *ct)
	Test if the given content type ct is JSON.
void *	GNUNET_CURL_download_get_result_ (struct GNUNET_CURL_DownloadBuffer *db, CURL *eh, long *response_code)
enum GNUNET_GenericReturnValue	GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx, const char *header)
	Add custom request header.
void	GNUNET_CURL_perform2 (struct GNUNET_CURL_Context *ctx, GNUNET_CURL_RawParser rp, GNUNET_CURL_ResponseCleaner rc)
	Run the main event loop for the HTTP interaction.
void	GNUNET_CURL_perform (struct GNUNET_CURL_Context *ctx)
	Run the main event loop for the CURL interaction.
void	GNUNET_CURL_get_select_info (struct GNUNET_CURL_Context *ctx, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *except_fd_set, int *max_fd, long *timeout)
	Obtain the information for a select() call to wait until GNUNET_CURL_perform() is ready again.
void	GNUNET_CURL_fini (struct GNUNET_CURL_Context *ctx)
	Cleanup library initialisation resources.
void	GNUNET_CURL_constructor__ (void)
void	__attribute__ ((constructor))
	Initial global setup logic, specifically runs the Curl setup.
void	GNUNET_CURL_destructor__ (void)
void	__attribute__ ((destructor))
	Cleans up after us, specifically runs the Curl cleanup.

Variables
static int	curl_fail
	Failsafe flag.

Detailed Description

API for downloading JSON via CURL.

Author

Sree Harsha Totakura sreeh.nosp@m.arsh.nosp@m.a@tot.nosp@m.akur.nosp@m.a.in

Christian Grothoff

Definition in file curl.c.

Macro Definition Documentation

DEBUG

#define DEBUG   0

Set to 1 for extra debug logging.

Definition at line 39 of file curl.c.

CURL_STRERROR

#define CURL_STRERROR	(		type,
			function,
			code
	)

Value:

        GNUNET_log (type,                                                        \
                    "Curl function `%s' has failed at `%s:%d' with error: %s\n", \
                    function,                                                    \
                    __FILE__,                                                    \
                    __LINE__,                                                    \
                    curl_easy_strerror (code));

Log error related to CURL operations.

Parameters

type	log level
function	which function failed to run
code	what was the curl error code

Definition at line 48 of file curl.c.

                                                         :%d' with error: %s\n", \
                    function,                                                    \
                    __FILE__,                                                    \
                    __LINE__,                                                    \
                    curl_easy_strerror (code));

JSON_WARN

#define JSON_WARN

(

error

)

Value:

        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,                 \
                    "JSON parsing failed at %s:%u: %s (%s)\n", \
                    __FILE__,                                  \
                    __LINE__,                                  \
                    error.text,                                \
                    error.source)

Print JSON parsing related error information.

Definition at line 59 of file curl.c.

                                              :%u: %s (%s)\n", \
                    __FILE__,                                  \
                    __LINE__,                                  \
                    error.text,                                \
                    error.source)

Function Documentation

download_cb()

static size_t download_cb ( char *  bufptr, size_t  size, size_t  nitems, void *  cls  )

static

Callback used when downloading the reply to an HTTP request.

Just appends all of the data to the buf in the struct DownloadBuffer for further processing. The size of the download is limited to GNUNET_MAX_MALLOC_CHECKED, if the download exceeds this size, we abort with an error.

Parameters

bufptr	data downloaded via HTTP
size	size of an item in bufptr
nitems	number of items in bufptr
cls	the struct DownloadBuffer

Returns

number of bytes processed from bufptr

Definition at line 327 of file curl.c.

{
  struct GNUNET_CURL_DownloadBuffer *db = cls;
  size_t msize;
  void *buf;
 
  if (0 == size * nitems)
  {
    /* Nothing (left) to do */
    return 0;
  }
  msize = size * nitems;
  if ((msize + db->buf_size) >= GNUNET_MAX_MALLOC_CHECKED)
  {
    db->eno = ENOMEM;
    return 0;   /* signals an error to curl */
  }
  db->buf = GNUNET_realloc (db->buf,
                            db->buf_size + msize);
  buf = db->buf + db->buf_size;
  GNUNET_memcpy (buf, bufptr, msize);
  db->buf_size += msize;
  return msize;
}

References GNUNET_CURL_DownloadBuffer::buf, db, GNUNET_MAX_MALLOC_CHECKED, GNUNET_memcpy, GNUNET_realloc, and size.

Referenced by setup_job_io().

Here is the caller graph for this function:

setup_job_headers()

static struct curl_slist * setup_job_headers ( struct GNUNET_CURL_Context *  ctx, const struct curl_slist *  job_headers  )

static

Create the HTTP headers for the request.

Parameters

ctx	context we run in
job_headers	job-specific headers

Returns

all headers to use

Definition at line 364 of file curl.c.

{
  struct curl_slist *all_headers = NULL;
 
  for (const struct curl_slist *curr = job_headers;
       NULL != curr;
       curr = curr->next)
  {
    GNUNET_assert (NULL !=
                   (all_headers = curl_slist_append (all_headers,
                                                     curr->data)));
  }
 
  for (const struct curl_slist *curr = ctx->common_headers;
       NULL != curr;
       curr = curr->next)
  {
    GNUNET_assert (NULL !=
                   (all_headers = curl_slist_append (all_headers,
                                                     curr->data)));
  }
 
  if (NULL != ctx->async_scope_id_header)
  {
    struct GNUNET_AsyncScopeSave scope;
 
    GNUNET_async_scope_get (&scope);
    if (GNUNET_YES == scope.have_scope)
    {
      char *aid_header;
 
      aid_header =
        GNUNET_STRINGS_data_to_string_alloc (
          &scope.scope_id,
          sizeof(struct GNUNET_AsyncScopeId));
      GNUNET_assert (NULL != aid_header);
      GNUNET_assert (NULL != curl_slist_append (all_headers,
                                                aid_header));
      GNUNET_free (aid_header);
    }
  }
  return all_headers;
}

References ctx, GNUNET_assert, GNUNET_async_scope_get(), GNUNET_free, GNUNET_STRINGS_data_to_string_alloc(), GNUNET_YES, GNUNET_AsyncScopeSave::have_scope, and GNUNET_AsyncScopeSave::scope_id.

Referenced by GNUNET_CURL_job_add2(), GNUNET_CURL_job_add_raw(), and GNUNET_CURL_job_add_stream().

Here is the call graph for this function:

Here is the caller graph for this function:

stream_write_cb()

static size_t stream_write_cb ( char *  bufptr, size_t  size, size_t  nitems, void *  cls  )

static

Callback used when the caller wants the response body as it arrives, instead of buffered.

Hands the chunk straight to the caller's GNUNET_CURL_StreamCallback, whose return value libcurl understands directly: msize accepts, GNUNET_CURL_STREAM_PAUSE suspends delivery, anything else fails the transfer.

Parameters

bufptr	data downloaded via HTTP
size	size of an item in bufptr
nitems	number of items in bufptr
cls	the struct GNUNET_CURL_Job

Returns

number of bytes processed from bufptr

Definition at line 424 of file curl.c.

{
  struct GNUNET_CURL_Job *job = cls;
  size_t msize = size * nitems;
 
  if (0 == msize)
    return 0; /* nothing (left) to do */
  return job->sh.scb (job->sh.scb_cls,
                      bufptr,
                      msize);
}

References job, and size.

Referenced by setup_job_io().

Here is the caller graph for this function:

stream_header_cb()

static size_t stream_header_cb ( char *  bufptr, size_t  size, size_t  nitems, void *  cls  )

static

Callback used to pass the response's header lines on to a streaming caller.

Note that libcurl routes everything through here that it reads in a header section, including status lines, the blank line ending each section, interim (1xx) responses and trailers; sorting those out is the caller's job, and GNUNET_CURL_StreamHeaderCallback says so.

Parameters

bufptr	one header line, not 0-terminated
size	size of an item in bufptr
nitems	number of items in bufptr
cls	the struct GNUNET_CURL_Job

Returns

size * nitems to continue, anything else to fail the transfer

Definition at line 456 of file curl.c.

{
  struct GNUNET_CURL_Job *job = cls;
  size_t msize = size * nitems;
 
  if (NULL == job->sh.hcb)
    return msize;
  if (GNUNET_OK !=
      job->sh.hcb (job->sh.hcb_cls,
                   bufptr,
                   msize))
    return 0; /* signals an error to curl */
  return msize;
}

References GNUNET_OK, job, and size.

Referenced by setup_job_io().

Here is the caller graph for this function:

setup_job_io()

static bool setup_job_io ( CURL *  eh, struct GNUNET_CURL_Job *  job  )

static

Point eh's body and header handling at job, in whichever of the two modes job was set up for.

Parameters

	eh	easy handle to configure
[in,out]	job	the job eh belongs to

Returns

true on success

Definition at line 484 of file curl.c.

{
  if (! job->streaming)
    return (CURLE_OK ==
            curl_easy_setopt (eh,
                              CURLOPT_WRITEFUNCTION,
                              &download_cb)) &&
           (CURLE_OK ==
            curl_easy_setopt (eh,
                              CURLOPT_WRITEDATA,
                              &job->db));
  return (CURLE_OK ==
          curl_easy_setopt (eh,
                            CURLOPT_WRITEFUNCTION,
                            &stream_write_cb)) &&
         (CURLE_OK ==
          curl_easy_setopt (eh,
                            CURLOPT_WRITEDATA,
                            job)) &&
         (CURLE_OK ==
          curl_easy_setopt (eh,
                            CURLOPT_HEADERFUNCTION,
                            &stream_header_cb)) &&
         (CURLE_OK ==
          curl_easy_setopt (eh,
                            CURLOPT_HEADERDATA,
                            job));
}

References download_cb(), job, stream_header_cb(), and stream_write_cb().

Referenced by setup_job().

Here is the call graph for this function:

Here is the caller graph for this function:

setup_job()

static struct GNUNET_CURL_Job * setup_job ( CURL *  eh, struct GNUNET_CURL_Context *  ctx, struct curl_slist *  all_headers, const struct GNUNET_CURL_StreamHandlers *  sh  )

static

Create a job.

Parameters

eh	easy handle to use
ctx	context to run the job in
all_headers	HTTP client headers to use (free'd)
sh	handlers for a streaming job, or NULL for a buffering one

Returns

NULL on error

Definition at line 525 of file curl.c.

{
  struct GNUNET_CURL_Job *job;
 
  if (CURLE_OK !=
      curl_easy_setopt (eh,
                        CURLOPT_HTTPHEADER,
                        all_headers))
  {
    GNUNET_break (0);
    curl_slist_free_all (all_headers);
    curl_easy_cleanup (eh);
    return NULL;
  }
  job = GNUNET_new (struct GNUNET_CURL_Job);
  job->start_time = GNUNET_TIME_absolute_get ();
  job->job_headers = all_headers;
  if (NULL != sh)
  {
    job->streaming = true;
    job->sh = *sh;
  }
 
  if ( (CURLE_OK !=
        curl_easy_setopt (eh,
                          CURLOPT_PRIVATE,
                          job)) ||
       (! setup_job_io (eh,
                        job)) ||
       (CURLE_OK !=
        curl_easy_setopt (eh,
                          CURLOPT_SHARE,
                          ctx->share)) )
  {
    GNUNET_break (0);
    GNUNET_free (job);
    curl_easy_cleanup (eh);
    return NULL;
  }
  if ( (CURLM_OK !=
        curl_multi_add_handle (ctx->multi,
                               eh)) )
  {
    GNUNET_break (0);
    GNUNET_free (job);
    curl_easy_cleanup (eh);
    return NULL;
  }
  job->easy_handle = eh;
  job->ctx = ctx;
  GNUNET_CONTAINER_DLL_insert (ctx->jobs_head,
                               ctx->jobs_tail,
                               job);
  return job;
}

References ctx, GNUNET_break, GNUNET_CONTAINER_DLL_insert, GNUNET_free, GNUNET_new, GNUNET_TIME_absolute_get(), job, setup_job_io(), and sh.

Referenced by GNUNET_CURL_job_add2(), GNUNET_CURL_job_add_raw(), and GNUNET_CURL_job_add_stream().

Here is the call graph for this function:

Here is the caller graph for this function:

is_json()

static bool is_json ( const char *  ct )

static

Test if the given content type ct is JSON.

Parameters

ct	a content type, e.g. "application/json; charset=UTF-8"

Returns

true if ct denotes JSON

Definition at line 792 of file curl.c.

{
  const char *semi;
 
  /* check for "application/json" exact match */
  if (0 == strcasecmp (ct,
                       "application/json"))
    return true;
  /* check for "application/json;[ANYTHING]" */
  semi = strchr (ct,
                 ';');
  /* also allow "application/json [ANYTHING]" (note the space!) */
  if (NULL == semi)
    semi = strchr (ct,
                   ' ');
  if (NULL == semi)
    return false; /* no delimiter we accept, forget it */
  if (semi - ct != strlen ("application/json"))
    return false; /* delimiter past desired length, forget it */
  if (0 == strncasecmp (ct,
                        "application/json",
                        strlen ("application/json")))
    return true; /* OK */
  return false;
}

Referenced by GNUNET_CURL_download_get_result_().

Here is the caller graph for this function:

GNUNET_CURL_download_get_result_()

void * GNUNET_CURL_download_get_result_	(	struct GNUNET_CURL_DownloadBuffer *	db,
		CURL *	eh,
		long *	response_code
	)

Definition at line 820 of file curl.c.

{
  json_t *json;
  char *ct;
 
#if DEBUG
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Downloaded body: %.*s\n",
              (int) db->buf_size,
              (char *) db->buf);
#endif
  if (CURLE_OK !=
      curl_easy_getinfo (eh,
                         CURLINFO_RESPONSE_CODE,
                         response_code))
  {
    /* unexpected error... */
    GNUNET_break (0);
    *response_code = 0;
  }
  if (MHD_HTTP_NO_CONTENT == *response_code)
    return NULL;
  if ((CURLE_OK !=
       curl_easy_getinfo (eh,
                          CURLINFO_CONTENT_TYPE,
                          &ct)) ||
      (NULL == ct) ||
      (! is_json (ct)))
  {
    /* No content type or explicitly not JSON, refuse to parse
       (but keep response code) */
    if (0 != db->buf_size)
    {
      const char *url;
 
      if (CURLE_OK !=
          curl_easy_getinfo (eh,
                             CURLINFO_EFFECTIVE_URL,
                             &url))
        url = "<unknown URL>";
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  "Request to `%s' was expected to return a body of type `application/json', got `%s'\n",
                  url,
                  ct);
    }
    return NULL;
  }
  if (0 == *response_code)
  {
    const char *url;
 
    if (CURLE_OK !=
        curl_easy_getinfo (eh,
                           CURLINFO_EFFECTIVE_URL,
                           &url))
      url = "<unknown URL>";
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Failed to download response from `%s': \n",
                url);
    return NULL;
  }
  json = NULL;
  if (0 == db->eno)
  {
    json_error_t error;
 
    json = json_loadb (db->buf,
                       db->buf_size,
                       JSON_REJECT_DUPLICATES | JSON_DISABLE_EOF_CHECK,
                       &error);
    if (NULL == json)
    {
      JSON_WARN (error);
      *response_code = 0;
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  "Failed to parse JSON response: %s\n",
                  error.text);
    }
  }
  GNUNET_free (db->buf);
  db->buf = NULL;
  db->buf_size = 0;
  return json;
}

References db, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_WARNING, GNUNET_free, GNUNET_log, is_json(), and JSON_WARN.

Referenced by GNUNET_CURL_gnunet_rc_create(), and GNUNET_CURL_perform().

Here is the call graph for this function:

Here is the caller graph for this function:

GNUNET_CURL_constructor__()

void GNUNET_CURL_constructor__

(

void

)

__attribute__() [1/2]

void __attribute__

(

(constructor)

)

Initial global setup logic, specifically runs the Curl setup.

Definition at line 1095 of file curl.c.

{
  CURLcode ret;
 
  if (CURLE_OK != (ret = curl_global_init (CURL_GLOBAL_DEFAULT)))
  {
    CURL_STRERROR (GNUNET_ERROR_TYPE_ERROR,
                   "curl_global_init",
                   ret);
    curl_fail = 1;
  }
}

References curl_fail, CURL_STRERROR, GNUNET_ERROR_TYPE_ERROR, and ret.

GNUNET_CURL_destructor__()

void GNUNET_CURL_destructor__

(

void

)

__attribute__() [2/2]

void __attribute__

(

(destructor)

)

Cleans up after us, specifically runs the Curl cleanup.

Definition at line 1116 of file curl.c.

{
  if (curl_fail)
    return;
  curl_global_cleanup ();
}

References curl_fail.

Variable Documentation

curl_fail

int curl_fail

static

Failsafe flag.

Raised if our constructor fails to initialize the Curl library.

Definition at line 72 of file curl.c.

Referenced by __attribute__(), __attribute__(), and GNUNET_CURL_init().