GNUnet  0.10.x
curl.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2014, 2015, 2016, 2018 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19 */
26 #include "platform.h"
27 #include <jansson.h>
28 #include "gnunet_curl_lib.h"
29 
30 #if ENABLE_BENCHMARK
31 #include "../util/benchmark.h"
32 #endif
33 
34 
42 #define CURL_STRERROR(type, function, code) \
43  GNUNET_log (type, \
44  "Curl function `%s' has failed at `%s:%d' with error: %s\n", \
45  function, \
46  __FILE__, \
47  __LINE__, \
48  curl_easy_strerror (code));
49 
53 #define JSON_WARN(error) \
54  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, \
55  "JSON parsing failed at %s:%u: %s (%s)\n", \
56  __FILE__, \
57  __LINE__, \
58  error.text, \
59  error.source)
60 
61 
66 static int curl_fail;
67 
72 {
73 
78 
83 
87  CURL *easy_handle;
88 
93 
98 
102  void *jcc_cls;
103 
108 
113  struct curl_slist *job_headers;
114 };
115 
116 
121 {
125  CURLM *multi;
126 
130  CURLSH *share;
131 
136 
141 
145  struct curl_slist *common_headers;
146 
152 
158 
162  void *cb_cls;
163 };
164 
165 
174 struct GNUNET_CURL_Context *
176 {
177  struct GNUNET_CURL_Context *ctx;
178  CURLM *multi;
179  CURLSH *share;
180 
181  if (curl_fail)
182  {
183  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Curl was not initialised properly\n");
184  return NULL;
185  }
186  if (NULL == (multi = curl_multi_init ()))
187  {
189  "Failed to create a Curl multi handle\n");
190  return NULL;
191  }
192  if (NULL == (share = curl_share_init ()))
193  {
195  "Failed to create a Curl share handle\n");
196  return NULL;
197  }
198  ctx = GNUNET_new (struct GNUNET_CURL_Context);
199  ctx->cb = cb;
200  ctx->cb_cls = cb_cls;
201  ctx->multi = multi;
202  ctx->share = share;
203  return ctx;
204 }
205 
206 
213 void
215  const char *header_name)
216 {
217  ctx->async_scope_id_header = header_name;
218 }
219 
220 
234 static size_t
235 download_cb (char *bufptr, size_t size, size_t nitems, void *cls)
236 {
237  struct GNUNET_CURL_DownloadBuffer *db = cls;
238  size_t msize;
239  void *buf;
240 
241  if (0 == size * nitems)
242  {
243  /* Nothing (left) to do */
244  return 0;
245  }
246  msize = size * nitems;
247  if ((msize + db->buf_size) >= GNUNET_MAX_MALLOC_CHECKED)
248  {
249  db->eno = ENOMEM;
250  return 0; /* signals an error to curl */
251  }
252  db->buf = GNUNET_realloc (db->buf, db->buf_size + msize);
253  buf = db->buf + db->buf_size;
254  GNUNET_memcpy (buf, bufptr, msize);
255  db->buf_size += msize;
256  return msize;
257 }
258 
259 
279 struct GNUNET_CURL_Job *
281  CURL *eh,
282  const struct curl_slist *job_headers,
284  void *jcc_cls)
285 {
286  struct GNUNET_CURL_Job *job;
287  struct curl_slist *all_headers = NULL;
288 
289  for (const struct curl_slist *curr = job_headers; curr != NULL;
290  curr = curr->next)
291  {
292  GNUNET_assert (NULL !=
293  (all_headers = curl_slist_append (all_headers, curr->data)));
294  }
295 
296  for (const struct curl_slist *curr = ctx->common_headers; curr != NULL;
297  curr = curr->next)
298  {
299  GNUNET_assert (NULL !=
300  (all_headers = curl_slist_append (all_headers, curr->data)));
301  }
302 
303  if (NULL != ctx->async_scope_id_header)
304  {
305  struct GNUNET_AsyncScopeSave scope;
306 
307  GNUNET_async_scope_get (&scope);
308  if (GNUNET_YES == scope.have_scope)
309  {
310  char *aid_header = NULL;
311  aid_header =
313  sizeof (
314  struct GNUNET_AsyncScopeId));
315  GNUNET_assert (NULL != aid_header);
316  GNUNET_assert (NULL != curl_slist_append (all_headers, aid_header));
317  GNUNET_free (aid_header);
318  }
319  }
320 
321  if (CURLE_OK != curl_easy_setopt (eh, CURLOPT_HTTPHEADER, all_headers))
322  {
323  GNUNET_break (0);
324  curl_slist_free_all (all_headers);
325  curl_easy_cleanup (eh);
326  return NULL;
327  }
328 
329  job = GNUNET_new (struct GNUNET_CURL_Job);
330  job->job_headers = all_headers;
331 
332  if ((CURLE_OK != curl_easy_setopt (eh, CURLOPT_PRIVATE, job)) ||
333  (CURLE_OK !=
334  curl_easy_setopt (eh, CURLOPT_WRITEFUNCTION, &download_cb)) ||
335  (CURLE_OK != curl_easy_setopt (eh, CURLOPT_WRITEDATA, &job->db)) ||
336  (CURLE_OK != curl_easy_setopt (eh, CURLOPT_SHARE, ctx->share)) ||
337  (CURLM_OK != curl_multi_add_handle (ctx->multi, eh)))
338  {
339  GNUNET_break (0);
340  GNUNET_free (job);
341  curl_easy_cleanup (eh);
342  return NULL;
343  }
344 
345  job->easy_handle = eh;
346  job->ctx = ctx;
347  job->jcc = jcc;
348  job->jcc_cls = jcc_cls;
350  ctx->cb (ctx->cb_cls);
351  return job;
352 }
353 
354 
371 struct GNUNET_CURL_Job *
373  CURL *eh,
374  int add_json,
376  void *jcc_cls)
377 {
378  struct GNUNET_CURL_Job *job;
379  struct curl_slist *job_headers = NULL;
380 
381  if (GNUNET_YES == add_json)
382  {
383  GNUNET_assert (
384  NULL != (job_headers =
385  curl_slist_append (NULL, "Content-Type: application/json")));
386  }
387 
388  job = GNUNET_CURL_job_add2 (ctx, eh, job_headers, jcc, jcc_cls);
389  curl_slist_free_all (job_headers);
390  return job;
391 }
392 
393 
400 void
402 {
403  struct GNUNET_CURL_Context *ctx = job->ctx;
404 
406  GNUNET_break (CURLM_OK ==
407  curl_multi_remove_handle (ctx->multi, job->easy_handle));
408  curl_easy_cleanup (job->easy_handle);
409  GNUNET_free_non_null (job->db.buf);
410  curl_slist_free_all (job->job_headers);
411  GNUNET_free (job);
412 }
413 
414 
433 void *
435  CURL *eh,
436  long *response_code)
437 {
438  json_t *json;
439  json_error_t error;
440  char *ct;
441 
443  "Downloaded body: %.*s\n",
444  (int) db->buf_size,
445  (char *) db->buf);
446 
447  if ((CURLE_OK != curl_easy_getinfo (eh, CURLINFO_CONTENT_TYPE, &ct)) ||
448  (NULL == ct) || (0 != strcasecmp (ct, "application/json")))
449  {
450  /* No content type or explicitly not JSON, refuse to parse
451  (but keep response code) */
452  if (CURLE_OK !=
453  curl_easy_getinfo (eh, CURLINFO_RESPONSE_CODE, response_code))
454  {
455  /* unexpected error... */
456  GNUNET_break (0);
457  *response_code = 0;
458  }
459  if (0 != db->buf_size)
461  "Did NOT detect response as JSON\n");
462  return NULL;
463  }
464  json = NULL;
465  if (0 == db->eno)
466  {
467  json = json_loadb (db->buf,
468  db->buf_size,
469  JSON_REJECT_DUPLICATES | JSON_DISABLE_EOF_CHECK,
470  &error);
471  if (NULL == json)
472  {
473  JSON_WARN (error);
474  *response_code = 0;
475  }
476  }
478  db->buf = NULL;
479  db->buf_size = 0;
480  if (NULL != json)
481  {
482  if (CURLE_OK !=
483  curl_easy_getinfo (eh, CURLINFO_RESPONSE_CODE, response_code))
484  {
485  /* unexpected error... */
486  GNUNET_break (0);
487  *response_code = 0;
488  }
489  }
490  return json;
491 }
492 
493 
501 int
502 GNUNET_CURL_append_header (struct GNUNET_CURL_Context *ctx, const char *header)
503 {
504  ctx->common_headers = curl_slist_append (ctx->common_headers, header);
505  if (NULL == ctx->common_headers)
506  return GNUNET_SYSERR;
507 
508  return GNUNET_OK;
509 }
510 
511 
520 void
524 {
525  CURLMsg *cmsg;
526  int n_running;
527  int n_completed;
528 
529  (void) curl_multi_perform (ctx->multi, &n_running);
530  while (NULL != (cmsg = curl_multi_info_read (ctx->multi, &n_completed)))
531  {
532  struct GNUNET_CURL_Job *job;
533  long response_code;
534  void *response;
535 
536  /* Only documented return value is CURLMSG_DONE */
537  GNUNET_break (CURLMSG_DONE == cmsg->msg);
538  GNUNET_assert (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
539  CURLINFO_PRIVATE,
540  (char **) &job));
541  GNUNET_assert (job->ctx == ctx);
542  response_code = 0;
543  response = rp (&job->db, job->easy_handle, &response_code);
544 #if ENABLE_BENCHMARK
545  {
546  char *url = NULL;
547  double total_as_double = 0;
548  struct GNUNET_TIME_Relative total;
549  struct UrlRequestData *urd;
550  /* Some care required, as curl is using data types (long vs curl_off_t vs
551  * double) inconsistently to store byte count. */
552  curl_off_t size_curl = 0;
553  long size_long = 0;
554  uint64_t bytes_sent = 0;
555  uint64_t bytes_received = 0;
556 
557  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
558  CURLINFO_TOTAL_TIME,
559  &total_as_double));
560  total.rel_value_us = total_as_double * 1000 * 1000;
561 
562  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
563  CURLINFO_EFFECTIVE_URL,
564  &url));
565 
566  /* HEADER_SIZE + SIZE_DOWNLOAD_T is hopefully the total
567  number of bytes received, not clear from curl docs. */
568 
569  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
570  CURLINFO_HEADER_SIZE,
571  &size_long));
572  bytes_received += size_long;
573 
574  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
575  CURLINFO_SIZE_DOWNLOAD_T,
576  &size_curl));
577  bytes_received += size_curl;
578 
579  /* REQUEST_SIZE + SIZE_UPLOAD_T is hopefully the total number of bytes
580  sent, again docs are not completely clear. */
581 
582  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
583  CURLINFO_REQUEST_SIZE,
584  &size_long));
585  bytes_sent += size_long;
586 
587  /* We obtain this value to check an invariant, but never use it otherwise. */
588  GNUNET_break (CURLE_OK == curl_easy_getinfo (cmsg->easy_handle,
589  CURLINFO_SIZE_UPLOAD_T,
590  &size_curl));
591 
592  /* CURLINFO_SIZE_UPLOAD_T <= CURLINFO_REQUEST_SIZE should
593  be an invariant.
594  As verified with
595  curl -w "foo%{size_request} -XPOST --data "ABC" $URL
596  the CURLINFO_REQUEST_SIZE should be the whole size of the request
597  including headers and body.
598  */
599  GNUNET_break (size_curl <= size_long);
600 
601  urd = get_url_benchmark_data (url, (unsigned int) response_code);
602  urd->count++;
603  urd->time = GNUNET_TIME_relative_add (urd->time, total);
604  urd->time_max = GNUNET_TIME_relative_max (total, urd->time_max);
605  urd->bytes_sent += bytes_sent;
607  }
608 #endif
609  job->jcc (job->jcc_cls, response_code, response);
610  rc (response);
612  }
613 }
614 
615 
621 void
623 {
626  (GNUNET_CURL_ResponseCleaner) &json_decref);
627 }
628 
629 
658 void
660  fd_set *read_fd_set,
661  fd_set *write_fd_set,
662  fd_set *except_fd_set,
663  int *max_fd,
664  long *timeout)
665 {
666  long to;
667  int m;
668 
669  m = -1;
670  GNUNET_assert (CURLM_OK == curl_multi_fdset (ctx->multi,
671  read_fd_set,
672  write_fd_set,
673  except_fd_set,
674  &m));
675  to = *timeout;
676  *max_fd = GNUNET_MAX (m, *max_fd);
677  GNUNET_assert (CURLM_OK == curl_multi_timeout (ctx->multi, &to));
678 
679  /* Only if what we got back from curl is smaller than what we
680  already had (-1 == infinity!), then update timeout */
681  if ((to < *timeout) && (-1 != to))
682  *timeout = to;
683  if ((-1 == (*timeout)) && (NULL != ctx->jobs_head))
684  *timeout = to;
685 }
686 
687 
695 void
697 {
698  /* all jobs must have been cancelled at this time, assert this */
699  GNUNET_assert (NULL == ctx->jobs_head);
700  curl_share_cleanup (ctx->share);
701  curl_multi_cleanup (ctx->multi);
702  curl_slist_free_all (ctx->common_headers);
703  GNUNET_free (ctx);
704 }
705 
706 
710 __attribute__ ((constructor)) void
711 GNUNET_CURL_constructor__ (void)
712 {
713  CURLcode ret;
714 
715  if (CURLE_OK != (ret = curl_global_init (CURL_GLOBAL_DEFAULT)))
716  {
717  CURL_STRERROR (GNUNET_ERROR_TYPE_ERROR, "curl_global_init", ret);
718  curl_fail = 1;
719  }
720 }
721 
722 
726 __attribute__ ((destructor)) void
727 GNUNET_CURL_destructor__ (void)
728 {
729  if (curl_fail)
730  return;
731  curl_global_cleanup ();
732 }
733 
734 /* end of curl.c */
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
#define CURL_STRERROR(type, function, code)
Log error related to CURL operations.
Definition: curl.c:42
struct GNUNET_CURL_Job * GNUNET_CURL_job_add2(struct GNUNET_CURL_Context *ctx, CURL *eh, const struct curl_slist *job_headers, GNUNET_CURL_JobCompletionCallback jcc, void *jcc_cls)
Schedule a CURL request to be executed and call the given jcc upon its completion.
Definition: curl.c:280
uint64_t rel_value_us
The actual value.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
library to make it easy to download JSON replies over HTTP
struct GNUNET_TIME_Relative GNUNET_TIME_relative_max(struct GNUNET_TIME_Relative t1, struct GNUNET_TIME_Relative t2)
Return the maximum of two relative time values.
Definition: time.c:286
struct GNUNET_CURL_Job * prev
We keep jobs in a DLL.
Definition: curl.c:82
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int GNUNET_CURL_append_header(struct GNUNET_CURL_Context *ctx, const char *header)
Add custom request header.
Definition: curl.c:502
void GNUNET_CURL_fini(struct GNUNET_CURL_Context *ctx)
Cleanup library initialisation resources.
Definition: curl.c:696
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:78
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
Identifier for an asynchronous execution context.
Struct for benchmark data for one URL.
Definition: benchmark.h:61
GNUNET_CURL_RescheduleCallback cb
Function we need to call whenever the event loop&#39;s socket set changed.
Definition: curl.c:157
static int ret
Final status code.
Definition: gnunet-arm.c:89
struct GNUNET_TIME_Relative time_max
Slowest time to response.
Definition: benchmark.h:96
void(* GNUNET_CURL_JobCompletionCallback)(void *cls, long response_code, const void *response)
Function to call upon completion of a job.
void GNUNET_CURL_job_cancel(struct GNUNET_CURL_Job *job)
Cancel a job.
Definition: curl.c:401
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
uint64_t count
How often was the URL requested?
Definition: benchmark.h:76
void * jcc_cls
Closure for jcc.
Definition: curl.c:102
static struct GNUNET_ARM_MonitorHandle * m
Monitor connection with ARM.
Definition: gnunet-arm.c:99
void *(* GNUNET_CURL_RawParser)(struct GNUNET_CURL_DownloadBuffer *db, CURL *eh, long *response_code)
Parses the raw response we got from the Web server.
GNUNET_CURL_JobCompletionCallback jcc
Function to call upon completion.
Definition: curl.c:97
static struct GNUNET_TIME_Relative timeout
User defined timestamp for completing operations.
Definition: gnunet-arm.c:114
struct GNUNET_CURL_Job * jobs_head
We keep jobs in a DLL.
Definition: curl.c:135
#define GNUNET_memcpy(dst, src, n)
#define GNUNET_realloc(ptr, size)
Wrapper around realloc.
struct GNUNET_CURL_Job * next
We keep jobs in a DLL.
Definition: curl.c:77
#define JSON_WARN(error)
Print JSON parsing related error information.
Definition: curl.c:53
#define GNUNET_MAX(a, b)
Definition: gnunet_common.h:85
static CURLM * multi
Current multi-CURL handle.
CURLSH * share
Curl share handle.
Definition: curl.c:130
static char buf[2048]
static int curl_fail
Failsafe flag.
Definition: curl.c:66
Context.
Definition: curl.c:120
uint64_t bytes_sent
How many bytes were sent in total to request the URL.
Definition: benchmark.h:81
struct GNUNET_TIME_Relative time
Total time spent requesting this URL.
Definition: benchmark.h:91
void GNUNET_async_scope_get(struct GNUNET_AsyncScopeSave *scope_ret)
Get the current async scope.
void(* GNUNET_CURL_ResponseCleaner)(void *response)
Deallocate the response.
__attribute__((constructor))
Initial global setup logic, specifically runs the Curl setup.
Definition: curl.c:710
CURLM * multi
Curl multi handle.
Definition: curl.c:125
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
static unsigned int size
Size of the "table".
Definition: peer.c:67
uint64_t bytes_received
How many bytes were received in total as response to requesting this URL.
Definition: benchmark.h:86
int have_scope
GNUNET_YES unless this saved scope is the unnamed root scope.
Saved async scope identifier or root scope.
CURL * easy_handle
Easy handle of the job.
Definition: curl.c:87
#define GNUNET_MAX_MALLOC_CHECKED
Maximum allocation with GNUNET_malloc macro.
static struct GNUNET_SCHEDULER_Task * job
Task for main job.
Definition: gnunet-cadet.c:137
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:576
void GNUNET_CURL_get_select_info(struct GNUNET_CURL_Context *ctx, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *except_fd_set, int *max_fd, long *timeout)
Obtain the information for a select() call to wait until GNUNET_CURL_perform() is ready again...
Definition: curl.c:659
Buffer data structure we use to buffer the HTTP download before giving it to the JSON parser...
struct GNUNET_AsyncScopeId scope_id
Saved scope.
void GNUNET_CURL_perform(struct GNUNET_CURL_Context *ctx)
Run the main event loop for the Taler interaction.
Definition: curl.c:622
void * buf
Download buffer.
void(* GNUNET_CURL_RescheduleCallback)(void *cls)
Function called by the context to ask for the event loop to be rescheduled, that is the application s...
void * cb_cls
Closure for cb.
Definition: curl.c:162
Jobs are CURL requests running within a struct GNUNET_CURL_Context.
Definition: curl.c:71
#define GNUNET_log(kind,...)
struct GNUNET_CURL_Context * GNUNET_CURL_init(GNUNET_CURL_RescheduleCallback cb, void *cb_cls)
Initialise this library.
Definition: curl.c:175
const char * async_scope_id_header
If non-NULL, the async scope ID is sent in a request header of this name.
Definition: curl.c:151
static char * rp
Relying party.
struct GNUNET_CURL_Job * jobs_tail
We keep jobs in a DLL.
Definition: curl.c:140
static struct MHD_Response * response
Our canonical response.
#define GNUNET_YES
Definition: gnunet_common.h:80
struct GNUNET_CURL_DownloadBuffer db
Buffer for response received from CURL.
Definition: curl.c:107
struct UrlRequestData * get_url_benchmark_data(char *url, unsigned int status)
Get benchmark data for a URL.
Definition: benchmark.c:236
struct GNUNET_CURL_Job * GNUNET_CURL_job_add(struct GNUNET_CURL_Context *ctx, CURL *eh, int add_json, GNUNET_CURL_JobCompletionCallback jcc, void *jcc_cls)
Schedule a CURL request to be executed and call the given jcc upon its completion.
Definition: curl.c:372
char * GNUNET_STRINGS_data_to_string_alloc(const void *buf, size_t size)
Return the base32crockford encoding of the given buffer.
Definition: strings.c:988
size_t buf_size
The size of the download buffer.
void GNUNET_CURL_perform2(struct GNUNET_CURL_Context *ctx, GNUNET_CURL_RawParser rp, GNUNET_CURL_ResponseCleaner rc)
Run the main event loop for the Taler interaction.
Definition: curl.c:521
struct curl_slist * job_headers
Headers used for this job, the list needs to be freed after the job has finished. ...
Definition: curl.c:113
void GNUNET_CURL_enable_async_scope_header(struct GNUNET_CURL_Context *ctx, const char *header_name)
Enable sending the async scope ID as a header.
Definition: curl.c:214
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_CURL_Context * ctx
Context this job runs in.
Definition: curl.c:92
struct curl_slist * common_headers
Headers common for all requests in the context.
Definition: curl.c:145
Time for relative time used by GNUnet, in microseconds.
int eno
Error code (based on libc errno) if we failed to download (i.e.
void * GNUNET_CURL_download_get_result_(struct GNUNET_CURL_DownloadBuffer *db, CURL *eh, long *response_code)
Obtain information about the final result about the HTTP download.
Definition: curl.c:434
static size_t download_cb(char *bufptr, size_t size, size_t nitems, void *cls)
Callback used when downloading the reply to an HTTP request.
Definition: curl.c:235