curl.c

Go to the documentation of this file.

/*
  This file is part of GNUnet
  Copyright (C) 2014, 2015, 2016, 2018 GNUnet e.V.

  GNUnet is free software: you can redistribute it and/or modify it
  under the terms of the GNU Affero General Public License as published
  by the Free Software Foundation, either version 3 of the License,
  or (at your option) any later version.

  GNUnet is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Affero General Public License for more details.

  You should have received a copy of the GNU Affero General Public License
  along with this program.  If not, see <http://www.gnu.org/licenses/>.

     SPDX-License-Identifier: AGPL3.0-or-later
*/
#include "platform.h"
#include <jansson.h>
#include "gnunet_curl_lib.h"

#if ENABLE_BENCHMARK
#include "../util/benchmark.h"
#endif


#define CURL_STRERROR(type, function, code)                                \
  GNUNET_log (type,                                                        \
              "Curl function `%s' has failed at `%s:%d' with error: %s\n", \
              function,                                                    \
              __FILE__,                                                    \
              __LINE__,                                                    \
              curl_easy_strerror (code));

#define JSON_WARN(error)                                 \
  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,                 \
              "JSON parsing failed at %s:%u: %s (%s)\n", \
              __FILE__,                                  \
              __LINE__,                                  \
              error.text,                                \
              error.source)


static int curl_fail;

struct GNUNET_CURL_Job
{

  struct GNUNET_CURL_Job *next;

  struct GNUNET_CURL_Job *prev;

  CURL *easy_handle;

  struct GNUNET_CURL_Context *ctx;

  GNUNET_CURL_JobCompletionCallback jcc;

  void *jcc_cls;

  struct GNUNET_CURL_DownloadBuffer db;

  struct curl_slist *job_headers;
};


struct GNUNET_CURL_Context
{
  CURLM *multi;

  CURLSH *share;

  struct GNUNET_CURL_Job *jobs_head;

  struct GNUNET_CURL_Job *jobs_tail;

  struct curl_slist *common_headers;

  const char *async_scope_id_header;

  GNUNET_CURL_RescheduleCallback cb;

  void *cb_cls;
};


struct GNUNET_CURL_Context *
GNUNET_CURL_init (GNUNET_CURL_RescheduleCallback cb, void *cb_cls)
{
  struct GNUNET_CURL_Context *ctx;
  CURLM *multi;
  CURLSH *share;

  if (curl_fail)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Curl was not initialised properly\n");
    return NULL;
  }
  if (NULL == (multi = curl_multi_init ()))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to create a Curl multi handle\n");
    return NULL;
  }
  if (NULL == (share = curl_share_init ()))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to create a Curl share handle\n");
    return NULL;
  }
  ctx = GNUNET_new (struct GNUNET_CURL_Context);
  ctx->cb = cb;
  ctx->cb_cls = cb_cls;
  ctx->multi = multi;
  ctx->share = share;
  return ctx;
}


void
GNUNET_CURL_enable_async_scope_header (struct GNUNET_CURL_Context *ctx,
                                       const char *header_name)
{
  ctx->async_scope_id_header = header_name;
}


static size_t
download_cb (char *bufptr, size_t size, size_t nitems, void *cls)
{
  struct GNUNET_CURL_DownloadBuffer *db = cls;
  size_t msize;
  void *buf;

  if (0 == size * nitems)
  {
    /* Nothing (left) to do */
    return 0;
  }
  msize = size * nitems;
  if ((msize + db->buf_size) >= GNUNET_MAX_MALLOC_CHECKED)
  {
    db->eno = ENOMEM;
    return 0; /* signals an error to curl */
  }
  db->buf = GNUNET_realloc (db->buf, db->buf_size + msize);
  buf = db->buf + db->buf_size;
  GNUNET_memcpy (buf, bufptr, msize);
  db->buf_size += msize;
  return msize;
}


struct GNUNET_CURL_Job *
GNUNET_CURL_job_add2 (struct GNUNET_CURL_Context *ctx,
                      CURL *eh,
                      const struct curl_slist *job_headers,
                      GNUNET_CURL_JobCompletionCallback jcc,
                      void *jcc_cls)
{
  struct GNUNET_CURL_Job *job;
  struct curl_slist *all_headers = NULL;

  for (const struct curl_slist *curr = job_headers; curr != NULL;
       curr = curr->next)
  {
    GNUNET_assert (NULL !=
                   (all_headers = curl_slist_append (all_headers, curr->data)));
  }

  for (const struct curl_slist *curr = ctx->common_headers; curr != NULL;
       curr = curr->next)
  {
    GNUNET_assert (NULL !=
                   (all_headers = curl_slist_append (all_headers, curr->data)));
  }

  if (NULL != ctx->async_scope_id_header)
  {
    struct GNUNET_AsyncScopeSave scope;

    GNUNET_async_scope_get (&scope);
    if (GNUNET_YES == scope.have_scope)
    {
      char *aid_header = NULL;
      aid_header =
        GNUNET_STRINGS_data_to_string_alloc (&scope.scope_id,
                                             sizeof (
                                               struct GNUNET_AsyncScopeId));
      GNUNET_assert (NULL != aid_header);
      GNUNET_assert (NULL != curl_slist_append (all_headers, aid_header));
      GNUNET_free (aid_header);
    }
  }

  if (CURLE_OK != curl_easy_setopt (eh, CURLOPT_HTTPHEADER, all_headers))
  {
    GNUNET_break (0);
    curl_slist_free_all (all_headers);
    curl_easy_cleanup (eh);
    return NULL;
  }

  job = GNUNET_new (struct GNUNET_CURL_Job);
  job->job_headers = all_headers;

  if ((CURLE_OK != curl_easy_setopt (eh, CURLOPT_PRIVATE, job)) ||
      (CURLE_OK !=
       curl_easy_setopt (eh, CURLOPT_WRITEFUNCTION, &download_cb)) ||
      (CURLE_OK != curl_easy_setopt (eh, CURLOPT_WRITEDATA, &job->db)) ||
      (CURLE_OK != curl_easy_setopt (eh, CURLOPT_SHARE, ctx->share)) ||
      (CURLM_OK != curl_multi_add_handle (ctx->multi, eh)))
  {
    GNUNET_break (0);
    GNUNET_free (job);
    curl_easy_cleanup (eh);
    return NULL;
  }

  job->easy_handle = eh;
  job->ctx = ctx;
  job->jcc = jcc;
  job->jcc_cls = jcc_cls;
  GNUNET_CONTAINER_DLL_insert (ctx->jobs_head, ctx->jobs_tail, job);
  ctx->cb (ctx->cb_cls);
  return job;
}


struct GNUNET_CURL_Job *
GNUNET_CURL_job_add (struct GNUNET_CURL_Context *ctx,
                     CURL *eh,
                     int add_json,
                     GNUNET_CURL_JobCompletionCallback jcc,
                     void *jcc_cls)
{
  struct GNUNET_CURL_Job *job;
  struct curl_slist *job_headers = NULL;

  if (GNUNET_YES == add_json)
  {
    GNUNET_assert (
      NULL != (job_headers =
                 curl_slist_append (NULL, "Content-Type: application/json")));
  }

  job = GNUNET_CURL_job_add2 (ctx, eh, job_headers, jcc, jcc_cls);
  curl_slist_free_all (job_headers);
  return job;
}


void
GNUNET_CURL_job_cancel (struct GNUNET_CURL_Job *job)
{
  struct GNUNET_CURL_Context *ctx = job->ctx;

  GNUNET_CONTAINER_DLL_remove (ctx->jobs_head, ctx->jobs_tail, job);
  GNUNET_break (CURLM_OK ==
                curl_multi_remove_handle (ctx->multi, job->easy_handle));
  curl_easy_cleanup (job->easy_handle);
  GNUNET_free_non_null (job->db.buf);
  curl_slist_free_all (job->job_headers);
  GNUNET_free (job);
}


void *
GNUNET_CURL_download_get_result_ (struct GNUNET_CURL_DownloadBuffer *db,
                                  CURL *eh,
                                  long *response_code)
{
  json_t *json;
  json_error_t error;
  char *ct;

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Downloaded body: %.*s\n",
              (int) db->buf_size,
              (char *) db->buf);

  if ((CURLE_OK != curl_easy_getinfo (eh, CURLINFO_CONTENT_TYPE, &ct)) ||
      (NULL == ct) || (0 != strcasecmp (ct, "application/json")))
  {
    /* No content type or explicitly not JSON, refuse to parse
       (but keep response code) */
    if (CURLE_OK !=
        curl_easy_getinfo (eh, CURLINFO_RESPONSE_CODE, response_code))
    {
      /* unexpected error... */
      GNUNET_break (0);
      *response_code = 0;
    }
    if (0 != db->buf_size)
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                  "Did NOT detect response as JSON\n");
    return NULL;
  }
  json = NULL;
  if (0 == db->eno)
  {
    json = json_loadb (db->buf,
                       db->buf_size,
                       JSON_REJECT_DUPLICATES | JSON_DISABLE_EOF_CHECK,
                       &error);
    if (NULL == json)
    {
      JSON_WARN (error);
      *response_code = 0;
    }
  }
  GNUNET_free_non_null (db->buf);
  db->buf = NULL;
  db->buf_size = 0;
  if (NULL != json)
  {
    if (CURLE_OK !=
        curl_easy_getinfo (eh, CURLINFO_RESPONSE_CODE, response_code))
    {
      /* unexpected error... */
      GNUNET_break (0);
      *response_code = 0;
    }
  }
  return json;
}


int
GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx, const char *header)
{
  ctx->common_headers = curl_slist_append (ctx->common_headers, header);
  if (NULL == ctx->common_headers)
    return GNUNET_SYSERR;

  return GNUNET_OK;
}


void
GNUNET_CURL_perform2 (struct GNUNET_CURL_Context *ctx,
                      GNUNET_CURL_RawParser rp,
                      GNUNET_CURL_ResponseCleaner rc)
{
  CURLMsg *cmsg;
  int n_running;
  int n_completed;

  (void) curl_multi_perform (ctx->multi, &n_running);
  while (NULL != (cmsg = curl_multi_info_read (ctx->multi, &n_completed)))
  {
    struct GNUNET_CURL_Job *job;
    long response_code;
    void *response;

    /* Only documented return value is CURLMSG_DONE */
    GNUNET_break (CURLMSG_DONE == cmsg->msg);
    GNUNET_assert (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
                                                  CURLINFO_PRIVATE,
                                                  (char **) &job));
    GNUNET_assert (job->ctx == ctx);
    response_code = 0;
    response = rp (&job->db, job->easy_handle, &response_code);
#if ENABLE_BENCHMARK
    {
      char *url = NULL;
      double total_as_double = 0;
      struct GNUNET_TIME_Relative total;
      struct UrlRequestData *urd;
      /* Some care required, as curl is using data types (long vs curl_off_t vs
     * double) inconsistently to store byte count. */
      curl_off_t size_curl = 0;
      long size_long = 0;
      uint64_t bytes_sent = 0;
      uint64_t bytes_received = 0;

      GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
                                                   CURLINFO_TOTAL_TIME,
                                                   &total_as_double));
      total.rel_value_us = total_as_double * 1000 * 1000;

      GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
                                                   CURLINFO_EFFECTIVE_URL,
                                                   &url));

      /* HEADER_SIZE + SIZE_DOWNLOAD_T is hopefully the total
       number of bytes received, not clear from curl docs. */

      GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
                                                   CURLINFO_HEADER_SIZE,
                                                   &size_long));
      bytes_received += size_long;

      GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
                                                   CURLINFO_SIZE_DOWNLOAD_T,
                                                   &size_curl));
      bytes_received += size_curl;

      /* REQUEST_SIZE + SIZE_UPLOAD_T is hopefully the total number of bytes
       sent, again docs are not completely clear. */

      GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
                                                   CURLINFO_REQUEST_SIZE,
                                                   &size_long));
      bytes_sent += size_long;

      /* We obtain this value to check an invariant, but never use it otherwise. */
      GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
                                                   CURLINFO_SIZE_UPLOAD_T,
                                                   &size_curl));

      /* CURLINFO_SIZE_UPLOAD_T <= CURLINFO_REQUEST_SIZE should
       be an invariant.
       As verified with
         curl -w "foo%{size_request} -XPOST --data "ABC" $URL
      the CURLINFO_REQUEST_SIZE should be the whole size of the request
      including headers and body.
     */
      GNUNET_break (size_curl <= size_long);

      urd = get_url_benchmark_data (url, (unsigned int) response_code);
      urd->count++;
      urd->time = GNUNET_TIME_relative_add (urd->time, total);
      urd->time_max = GNUNET_TIME_relative_max (total, urd->time_max);
      urd->bytes_sent += bytes_sent;
      urd->bytes_received += bytes_received;
    }
#endif
    job->jcc (job->jcc_cls, response_code, response);
    rc (response);
    GNUNET_CURL_job_cancel (job);
  }
}


void
GNUNET_CURL_perform (struct GNUNET_CURL_Context *ctx)
{
  GNUNET_CURL_perform2 (ctx,
                        &GNUNET_CURL_download_get_result_,
                        (GNUNET_CURL_ResponseCleaner) &json_decref);
}


void
GNUNET_CURL_get_select_info (struct GNUNET_CURL_Context *ctx,
                             fd_set *read_fd_set,
                             fd_set *write_fd_set,
                             fd_set *except_fd_set,
                             int *max_fd,
                             long *timeout)
{
  long to;
  int m;

  m = -1;
  GNUNET_assert (CURLM_OK == curl_multi_fdset (ctx->multi,
                                               read_fd_set,
                                               write_fd_set,
                                               except_fd_set,
                                               &m));
  to = *timeout;
  *max_fd = GNUNET_MAX (m, *max_fd);
  GNUNET_assert (CURLM_OK == curl_multi_timeout (ctx->multi, &to));

  /* Only if what we got back from curl is smaller than what we
     already had (-1 == infinity!), then update timeout */
  if ((to < *timeout) && (-1 != to))
    *timeout = to;
  if ((-1 == (*timeout)) && (NULL != ctx->jobs_head))
    *timeout = to;
}


void
GNUNET_CURL_fini (struct GNUNET_CURL_Context *ctx)
{
  /* all jobs must have been cancelled at this time, assert this */
  GNUNET_assert (NULL == ctx->jobs_head);
  curl_share_cleanup (ctx->share);
  curl_multi_cleanup (ctx->multi);
  curl_slist_free_all (ctx->common_headers);
  GNUNET_free (ctx);
}


__attribute__ ((constructor)) void
GNUNET_CURL_constructor__ (void)
{
  CURLcode ret;

  if (CURLE_OK != (ret = curl_global_init (CURL_GLOBAL_DEFAULT)))
  {
    CURL_STRERROR (GNUNET_ERROR_TYPE_ERROR, "curl_global_init", ret);
    curl_fail = 1;
  }
}


__attribute__ ((destructor)) void
GNUNET_CURL_destructor__ (void)
{
  if (curl_fail)
    return;
  curl_global_cleanup ();
}

/* end of curl.c */