GNUnet  0.10.x
curl.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2014, 2015, 2016, 2018 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
26 #include "platform.h"
27 #include <jansson.h>
28 #include "gnunet_curl_lib.h"
29 
30 #if ENABLE_BENCHMARK
31 #include "../util/benchmark.h"
32 #endif
33 
34 
42 #define CURL_STRERROR(type, function, code) \
43  GNUNET_log (type, \
44  "Curl function `%s' has failed at `%s:%d' with error: %s\n", \
45  function, \
46  __FILE__, \
47  __LINE__, \
48  curl_easy_strerror (code));
49 
53 #define JSON_WARN(error) \
54  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, \
55  "JSON parsing failed at %s:%u: %s (%s)\n", \
56  __FILE__, \
57  __LINE__, \
58  error.text, \
59  error.source)
60 
61 
66 static int curl_fail;
67 
72 {
77 
82 
86  CURL *easy_handle;
87 
92 
97 
101  void *jcc_cls;
102 
107 
112  struct curl_slist *job_headers;
113 };
114 
115 
120 {
124  CURLM *multi;
125 
129  CURLSH *share;
130 
135 
140 
144  struct curl_slist *common_headers;
145 
151 
157 
161  void *cb_cls;
162 };
163 
164 
173 struct GNUNET_CURL_Context *
175 {
176  struct GNUNET_CURL_Context *ctx;
177  CURLM *multi;
178  CURLSH *share;
179 
180  if (curl_fail)
181  {
182  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Curl was not initialised properly\n");
183  return NULL;
184  }
185  if (NULL == (multi = curl_multi_init ()))
186  {
188  "Failed to create a Curl multi handle\n");
189  return NULL;
190  }
191  if (NULL == (share = curl_share_init ()))
192  {
194  "Failed to create a Curl share handle\n");
195  return NULL;
196  }
197  ctx = GNUNET_new (struct GNUNET_CURL_Context);
198  ctx->cb = cb;
199  ctx->cb_cls = cb_cls;
200  ctx->multi = multi;
201  ctx->share = share;
202  return ctx;
203 }
204 
205 
212 void
214  const char *header_name)
215 {
216  ctx->async_scope_id_header = header_name;
217 }
218 
219 
233 static size_t
234 download_cb (char *bufptr, size_t size, size_t nitems, void *cls)
235 {
236  struct GNUNET_CURL_DownloadBuffer *db = cls;
237  size_t msize;
238  void *buf;
239 
240  if (0 == size * nitems)
241  {
242  /* Nothing (left) to do */
243  return 0;
244  }
245  msize = size * nitems;
246  if ((msize + db->buf_size) >= GNUNET_MAX_MALLOC_CHECKED)
247  {
248  db->eno = ENOMEM;
249  return 0; /* signals an error to curl */
250  }
251  db->buf = GNUNET_realloc (db->buf, db->buf_size + msize);
252  buf = db->buf + db->buf_size;
253  GNUNET_memcpy (buf, bufptr, msize);
254  db->buf_size += msize;
255  return msize;
256 }
257 
258 
278 struct GNUNET_CURL_Job *
280  CURL *eh,
281  const struct curl_slist *job_headers,
283  void *jcc_cls)
284 {
285  struct GNUNET_CURL_Job *job;
286  struct curl_slist *all_headers = NULL;
287 
288  for (const struct curl_slist *curr = job_headers; curr != NULL;
289  curr = curr->next)
290  {
291  GNUNET_assert (NULL !=
292  (all_headers = curl_slist_append (all_headers, curr->data)));
293  }
294 
295  for (const struct curl_slist *curr = ctx->common_headers; curr != NULL;
296  curr = curr->next)
297  {
298  GNUNET_assert (NULL !=
299  (all_headers = curl_slist_append (all_headers, curr->data)));
300  }
301 
302  if (NULL != ctx->async_scope_id_header)
303  {
304  struct GNUNET_AsyncScopeSave scope;
305 
306  GNUNET_async_scope_get (&scope);
307  if (GNUNET_YES == scope.have_scope)
308  {
309  char *aid_header = NULL;
310  aid_header =
312  sizeof(
313  struct GNUNET_AsyncScopeId));
314  GNUNET_assert (NULL != aid_header);
315  GNUNET_assert (NULL != curl_slist_append (all_headers, aid_header));
316  GNUNET_free (aid_header);
317  }
318  }
319 
320  if (CURLE_OK != curl_easy_setopt (eh, CURLOPT_HTTPHEADER, all_headers))
321  {
322  GNUNET_break (0);
323  curl_slist_free_all (all_headers);
324  curl_easy_cleanup (eh);
325  return NULL;
326  }
327 
328  job = GNUNET_new (struct GNUNET_CURL_Job);
329  job->job_headers = all_headers;
330 
331  if ((CURLE_OK != curl_easy_setopt (eh, CURLOPT_PRIVATE, job)) ||
332  (CURLE_OK !=
333  curl_easy_setopt (eh, CURLOPT_WRITEFUNCTION, &download_cb)) ||
334  (CURLE_OK != curl_easy_setopt (eh, CURLOPT_WRITEDATA, &job->db)) ||
335  (CURLE_OK != curl_easy_setopt (eh, CURLOPT_SHARE, ctx->share)) ||
336  (CURLM_OK != curl_multi_add_handle (ctx->multi, eh)))
337  {
338  GNUNET_break (0);
339  GNUNET_free (job);
340  curl_easy_cleanup (eh);
341  return NULL;
342  }
343 
344  job->easy_handle = eh;
345  job->ctx = ctx;
346  job->jcc = jcc;
347  job->jcc_cls = jcc_cls;
349  ctx->cb (ctx->cb_cls);
350  return job;
351 }
352 
353 
370 struct GNUNET_CURL_Job *
372  CURL *eh,
373  int add_json,
375  void *jcc_cls)
376 {
377  struct GNUNET_CURL_Job *job;
378  struct curl_slist *job_headers = NULL;
379 
380  if (GNUNET_YES == add_json)
381  {
382  GNUNET_assert (
383  NULL != (job_headers =
384  curl_slist_append (NULL, "Content-Type: application/json")));
385  }
386 
387  job = GNUNET_CURL_job_add2 (ctx, eh, job_headers, jcc, jcc_cls);
388  curl_slist_free_all (job_headers);
389  return job;
390 }
391 
392 
399 void
401 {
402  struct GNUNET_CURL_Context *ctx = job->ctx;
403 
405  GNUNET_break (CURLM_OK ==
406  curl_multi_remove_handle (ctx->multi, job->easy_handle));
407  curl_easy_cleanup (job->easy_handle);
408  GNUNET_free_non_null (job->db.buf);
409  curl_slist_free_all (job->job_headers);
410  GNUNET_free (job);
411 }
412 
413 
432 void *
434  CURL *eh,
435  long *response_code)
436 {
437  json_t *json;
438  json_error_t error;
439  char *ct;
440 
442  "Downloaded body: %.*s\n",
443  (int) db->buf_size,
444  (char *) db->buf);
445 
446  if ((CURLE_OK != curl_easy_getinfo (eh, CURLINFO_CONTENT_TYPE, &ct)) ||
447  (NULL == ct) || (0 != strcasecmp (ct, "application/json")))
448  {
449  /* No content type or explicitly not JSON, refuse to parse
450  (but keep response code) */
451  if (CURLE_OK !=
452  curl_easy_getinfo (eh, CURLINFO_RESPONSE_CODE, response_code))
453  {
454  /* unexpected error... */
455  GNUNET_break (0);
456  *response_code = 0;
457  }
458  if (0 != db->buf_size)
460  "Did NOT detect response as JSON\n");
461  return NULL;
462  }
463  json = NULL;
464  if (0 == db->eno)
465  {
466  json = json_loadb (db->buf,
467  db->buf_size,
468  JSON_REJECT_DUPLICATES | JSON_DISABLE_EOF_CHECK,
469  &error);
470  if (NULL == json)
471  {
472  JSON_WARN (error);
473  *response_code = 0;
474  }
475  }
477  db->buf = NULL;
478  db->buf_size = 0;
479  if (NULL != json)
480  {
481  if (CURLE_OK !=
482  curl_easy_getinfo (eh, CURLINFO_RESPONSE_CODE, response_code))
483  {
484  /* unexpected error... */
485  GNUNET_break (0);
486  *response_code = 0;
487  }
488  }
489  return json;
490 }
491 
492 
500 int
501 GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx, const char *header)
502 {
503  ctx->common_headers = curl_slist_append (ctx->common_headers, header);
504  if (NULL == ctx->common_headers)
505  return GNUNET_SYSERR;
506 
507  return GNUNET_OK;
508 }
509 
510 
519 void
523 {
524  CURLMsg *cmsg;
525  int n_running;
526  int n_completed;
527 
528  (void) curl_multi_perform (ctx->multi, &n_running);
529  while (NULL != (cmsg = curl_multi_info_read (ctx->multi, &n_completed)))
530  {
531  struct GNUNET_CURL_Job *job;
532  long response_code;
533  void *response;
534 
535  /* Only documented return value is CURLMSG_DONE */
536  GNUNET_break (CURLMSG_DONE == cmsg->msg);
537  GNUNET_assert (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
538  CURLINFO_PRIVATE,
539  (char **) &job));
540  GNUNET_assert (job->ctx == ctx);
541  response_code = 0;
542  response = rp (&job->db, job->easy_handle, &response_code);
543 #if ENABLE_BENCHMARK
544  {
545  char *url = NULL;
546  double total_as_double = 0;
547  struct GNUNET_TIME_Relative total;
548  struct UrlRequestData *urd;
549  /* Some care required, as curl is using data types (long vs curl_off_t vs
550  * double) inconsistently to store byte count. */
551  curl_off_t size_curl = 0;
552  long size_long = 0;
553  uint64_t bytes_sent = 0;
554  uint64_t bytes_received = 0;
555 
556  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
557  CURLINFO_TOTAL_TIME,
558  &total_as_double));
559  total.rel_value_us = total_as_double * 1000 * 1000;
560 
561  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
562  CURLINFO_EFFECTIVE_URL,
563  &url));
564 
565  /* HEADER_SIZE + SIZE_DOWNLOAD_T is hopefully the total
566  number of bytes received, not clear from curl docs. */
567 
568  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
569  CURLINFO_HEADER_SIZE,
570  &size_long));
571  bytes_received += size_long;
572 
573  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
574  CURLINFO_SIZE_DOWNLOAD_T,
575  &size_curl));
576  bytes_received += size_curl;
577 
578  /* REQUEST_SIZE + SIZE_UPLOAD_T is hopefully the total number of bytes
579  sent, again docs are not completely clear. */
580 
581  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
582  CURLINFO_REQUEST_SIZE,
583  &size_long));
584  bytes_sent += size_long;
585 
586  /* We obtain this value to check an invariant, but never use it otherwise. */
587  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
588  CURLINFO_SIZE_UPLOAD_T,
589  &size_curl));
590 
591  /* CURLINFO_SIZE_UPLOAD_T <= CURLINFO_REQUEST_SIZE should
592  be an invariant.
593  As verified with
594  curl -w "foo%{size_request} -XPOST --data "ABC" $URL
595  the CURLINFO_REQUEST_SIZE should be the whole size of the request
596  including headers and body.
597  */
598  GNUNET_break (size_curl <= size_long);
599 
600  urd = get_url_benchmark_data (url, (unsigned int) response_code);
601  urd->count++;
602  urd->time = GNUNET_TIME_relative_add (urd->time, total);
603  urd->time_max = GNUNET_TIME_relative_max (total, urd->time_max);
604  urd->bytes_sent += bytes_sent;
606  }
607 #endif
608  job->jcc (job->jcc_cls, response_code, response);
609  rc (response);
611  }
612 }
613 
614 
620 void
622 {
625  (GNUNET_CURL_ResponseCleaner) & json_decref);
626 }
627 
628 
657 void
659  fd_set *read_fd_set,
660  fd_set *write_fd_set,
661  fd_set *except_fd_set,
662  int *max_fd,
663  long *timeout)
664 {
665  long to;
666  int m;
667 
668  m = -1;
669  GNUNET_assert (CURLM_OK == curl_multi_fdset (ctx->multi,
670  read_fd_set,
671  write_fd_set,
672  except_fd_set,
673  &m));
674  to = *timeout;
675  *max_fd = GNUNET_MAX (m, *max_fd);
676  GNUNET_assert (CURLM_OK == curl_multi_timeout (ctx->multi, &to));
677 
678  /* Only if what we got back from curl is smaller than what we
679  already had (-1 == infinity!), then update timeout */
680  if ((to < *timeout) && (-1 != to))
681  *timeout = to;
682  if ((-1 == (*timeout)) && (NULL != ctx->jobs_head))
683  *timeout = to;
684 }
685 
686 
694 void
696 {
697  /* all jobs must have been cancelled at this time, assert this */
698  GNUNET_assert (NULL == ctx->jobs_head);
699  curl_share_cleanup (ctx->share);
700  curl_multi_cleanup (ctx->multi);
701  curl_slist_free_all (ctx->common_headers);
702  GNUNET_free (ctx);
703 }
704 
705 
709 __attribute__ ((constructor)) void
710 GNUNET_CURL_constructor__ (void)
711 {
712  CURLcode ret;
713 
714  if (CURLE_OK != (ret = curl_global_init (CURL_GLOBAL_DEFAULT)))
715  {
716  CURL_STRERROR (GNUNET_ERROR_TYPE_ERROR, "curl_global_init", ret);
717  curl_fail = 1;
718  }
719 }
720 
721 
725 __attribute__ ((destructor)) void
726 GNUNET_CURL_destructor__ (void)
727 {
728  if (curl_fail)
729  return;
730  curl_global_cleanup ();
731 }
732 
733 /* end of curl.c */
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
#define CURL_STRERROR(type, function, code)
Log error related to CURL operations.
Definition: curl.c:42
struct GNUNET_CURL_Job * GNUNET_CURL_job_add2(struct GNUNET_CURL_Context *ctx, CURL *eh, const struct curl_slist *job_headers, GNUNET_CURL_JobCompletionCallback jcc, void *jcc_cls)
Schedule a CURL request to be executed and call the given jcc upon its completion.
Definition: curl.c:279
uint64_t rel_value_us
The actual value.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
library to make it easy to download JSON replies over HTTP
struct GNUNET_TIME_Relative GNUNET_TIME_relative_max(struct GNUNET_TIME_Relative t1, struct GNUNET_TIME_Relative t2)
Return the maximum of two relative time values.
Definition: time.c:287
struct GNUNET_CURL_Job * prev
We keep jobs in a DLL.
Definition: curl.c:81
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int GNUNET_CURL_append_header(struct GNUNET_CURL_Context *ctx, const char *header)
Add custom request header.
Definition: curl.c:501
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
void GNUNET_CURL_fini(struct GNUNET_CURL_Context *ctx)
Cleanup library initialisation resources.
Definition: curl.c:695
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
Identifier for an asynchronous execution context.
Struct for benchmark data for one URL.
Definition: benchmark.h:61
GNUNET_CURL_RescheduleCallback cb
Function we need to call whenever the event loop&#39;s socket set changed.
Definition: curl.c:156
static int ret
Final status code.
Definition: gnunet-arm.c:89
struct GNUNET_TIME_Relative time_max
Slowest time to response.
Definition: benchmark.h:95
void(* GNUNET_CURL_JobCompletionCallback)(void *cls, long response_code, const void *response)
Function to call upon completion of a job.
void GNUNET_CURL_job_cancel(struct GNUNET_CURL_Job *job)
Cancel a job.
Definition: curl.c:400
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
uint64_t count
How often was the URL requested?
Definition: benchmark.h:75
void * jcc_cls
Closure for jcc.
Definition: curl.c:101
static struct GNUNET_ARM_MonitorHandle * m
Monitor connection with ARM.
Definition: gnunet-arm.c:99
void *(* GNUNET_CURL_RawParser)(struct GNUNET_CURL_DownloadBuffer *db, CURL *eh, long *response_code)
Parses the raw response we got from the Web server.
GNUNET_CURL_JobCompletionCallback jcc
Function to call upon completion.
Definition: curl.c:96
static struct GNUNET_TIME_Relative timeout
User defined timestamp for completing operations.
Definition: gnunet-arm.c:114
struct GNUNET_CURL_Job * jobs_head
We keep jobs in a DLL.
Definition: curl.c:134
#define GNUNET_realloc(ptr, size)
Wrapper around realloc.
struct GNUNET_CURL_Job * next
We keep jobs in a DLL.
Definition: curl.c:76
#define JSON_WARN(error)
Print JSON parsing related error information.
Definition: curl.c:53
#define GNUNET_MAX(a, b)
Definition: gnunet_common.h:82
static CURLM * multi
Current multi-CURL handle.
CURLSH * share
Curl share handle.
Definition: curl.c:129
static char buf[2048]
static int curl_fail
Failsafe flag.
Definition: curl.c:66
Context.
Definition: curl.c:119
uint64_t bytes_sent
How many bytes were sent in total to request the URL.
Definition: benchmark.h:80
struct GNUNET_TIME_Relative time
Total time spent requesting this URL.
Definition: benchmark.h:90
void GNUNET_async_scope_get(struct GNUNET_AsyncScopeSave *scope_ret)
Get the current async scope.
void(* GNUNET_CURL_ResponseCleaner)(void *response)
Deallocate the response.
__attribute__((constructor))
Initial global setup logic, specifically runs the Curl setup.
Definition: curl.c:709
CURLM * multi
Curl multi handle.
Definition: curl.c:124
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:66
uint64_t bytes_received
How many bytes were received in total as response to requesting this URL.
Definition: benchmark.h:85
int have_scope
GNUNET_YES unless this saved scope is the unnamed root scope.
Saved async scope identifier or root scope.
CURL * easy_handle
Easy handle of the job.
Definition: curl.c:86
#define GNUNET_MAX_MALLOC_CHECKED
Maximum allocation with GNUNET_malloc macro.
static struct GNUNET_SCHEDULER_Task * job
Task for main job.
Definition: gnunet-cadet.c:137
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:577
void GNUNET_CURL_get_select_info(struct GNUNET_CURL_Context *ctx, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *except_fd_set, int *max_fd, long *timeout)
Obtain the information for a select() call to wait until GNUNET_CURL_perform() is ready again...
Definition: curl.c:658
Buffer data structure we use to buffer the HTTP download before giving it to the JSON parser...
struct GNUNET_AsyncScopeId scope_id
Saved scope.
void GNUNET_CURL_perform(struct GNUNET_CURL_Context *ctx)
Run the main event loop for the Taler interaction.
Definition: curl.c:621
void * buf
Download buffer.
void(* GNUNET_CURL_RescheduleCallback)(void *cls)
Function called by the context to ask for the event loop to be rescheduled, that is the application s...
void * cb_cls
Closure for cb.
Definition: curl.c:161
Jobs are CURL requests running within a struct GNUNET_CURL_Context.
Definition: curl.c:71
#define GNUNET_log(kind,...)
struct GNUNET_CURL_Context * GNUNET_CURL_init(GNUNET_CURL_RescheduleCallback cb, void *cb_cls)
Initialise this library.
Definition: curl.c:174
const char * async_scope_id_header
If non-NULL, the async scope ID is sent in a request header of this name.
Definition: curl.c:150
static char * rp
Relying party.
struct GNUNET_CURL_Job * jobs_tail
We keep jobs in a DLL.
Definition: curl.c:139
static struct MHD_Response * response
Our canonical response.
#define GNUNET_YES
Definition: gnunet_common.h:77
struct GNUNET_CURL_DownloadBuffer db
Buffer for response received from CURL.
Definition: curl.c:106
struct UrlRequestData * get_url_benchmark_data(char *url, unsigned int status)
Get benchmark data for a URL.
Definition: benchmark.c:236
struct GNUNET_CURL_Job * GNUNET_CURL_job_add(struct GNUNET_CURL_Context *ctx, CURL *eh, int add_json, GNUNET_CURL_JobCompletionCallback jcc, void *jcc_cls)
Schedule a CURL request to be executed and call the given jcc upon its completion.
Definition: curl.c:371
char * GNUNET_STRINGS_data_to_string_alloc(const void *buf, size_t size)
Return the base32crockford encoding of the given buffer.
Definition: strings.c:921
size_t buf_size
The size of the download buffer.
void GNUNET_CURL_perform2(struct GNUNET_CURL_Context *ctx, GNUNET_CURL_RawParser rp, GNUNET_CURL_ResponseCleaner rc)
Run the main event loop for the Taler interaction.
Definition: curl.c:520
struct curl_slist * job_headers
Headers used for this job, the list needs to be freed after the job has finished. ...
Definition: curl.c:112
void GNUNET_CURL_enable_async_scope_header(struct GNUNET_CURL_Context *ctx, const char *header_name)
Enable sending the async scope ID as a header.
Definition: curl.c:213
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_CURL_Context * ctx
Context this job runs in.
Definition: curl.c:91
struct curl_slist * common_headers
Headers common for all requests in the context.
Definition: curl.c:144
Time for relative time used by GNUnet, in microseconds.
int eno
Error code (based on libc errno) if we failed to download (i.e.
void * GNUNET_CURL_download_get_result_(struct GNUNET_CURL_DownloadBuffer *db, CURL *eh, long *response_code)
Obtain information about the final result about the HTTP download.
Definition: curl.c:433
static size_t download_cb(char *bufptr, size_t size, size_t nitems, void *cls)
Callback used when downloading the reply to an HTTP request.
Definition: curl.c:234