gnunet-daemon-hostlist_client.c

Go to the documentation of this file.

/*
     This file is part of GNUnet.
     Copyright (C) 2001-2010, 2014, 2016 GNUnet e.V.

     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.

     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.

     SPDX-License-Identifier: AGPL3.0-or-later
 */
#include "platform.h"
#include "gnunet-daemon-hostlist_client.h"
#include "gnunet_hello_lib.h"
#include "gnunet_statistics_service.h"
#include "gnunet_transport_service.h"
#include "gnunet_peerinfo_service.h"
#include "gnunet-daemon-hostlist.h"
/* Just included for the right curl.h */
#include "gnunet_curl_lib.h"


#define MIN_CONNECTIONS 4

#define MAX_NUMBER_HOSTLISTS 30

#define SAVING_INTERVAL \
  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)

#define TESTING_INTERVAL \
  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3)

#define WAITING_INTERVAL \
  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)

#define HOSTLIST_INITIAL 10000

#define HOSTLIST_FAILED_DOWNLOAD 100

#define HOSTLIST_SUCCESSFUL_DOWNLOAD 100

#define HOSTLIST_SUCCESSFUL_HELLO 1


struct Hostlist
{
  struct Hostlist *prev;

  struct Hostlist *next;

  const char *hostlist_uri;

  uint64_t quality;

  struct GNUNET_TIME_Absolute time_creation;

  struct GNUNET_TIME_Absolute time_last_usage;

  uint32_t hello_count;

  uint32_t times_used;
};


static const struct GNUNET_CONFIGURATION_Handle *cfg;

static struct GNUNET_STATISTICS_Handle *stats;

static char *proxy;

static char *proxy_username;

static char *proxy_password;

static curl_proxytype proxy_type;

static size_t download_pos;

static char *current_url;

static CURL *curl;

static CURLM *multi;

static uint32_t stat_bytes_downloaded;

static struct GNUNET_TIME_Relative hostlist_delay;

static struct GNUNET_SCHEDULER_Task *ti_check_download;

static struct GNUNET_SCHEDULER_Task *ti_download;

static struct GNUNET_SCHEDULER_Task *ti_saving_task;

static struct GNUNET_SCHEDULER_Task *ti_download_dispatcher_task;

static struct GNUNET_SCHEDULER_Task *ti_testing_intervall_task;

static struct GNUNET_TIME_Absolute end_time;

static struct Hostlist *linked_list_head;

static struct Hostlist *linked_list_tail;

static struct Hostlist *current_hostlist;

static unsigned int linked_list_size;

static struct Hostlist *hostlist_to_test;

static struct GNUNET_STATISTICS_GetHandle *sget;

static int stat_bogus_url;

static int stat_testing_hostlist;

static int stat_testing_allowed;

static int stat_download_in_progress;

static unsigned int stat_use_bootstrap;

static int stat_learning;

static unsigned int stat_download_successful;

static unsigned int stat_hellos_obtained;

static unsigned int stat_connection_count;

static struct GNUNET_PEERINFO_Handle *pi;


static size_t
callback_download (void *ptr, size_t size, size_t nmemb, void *ctx)
{
  static char download_buffer[GNUNET_MAX_MESSAGE_SIZE - 1];
  const char *cbuf = ptr;
  const struct GNUNET_MessageHeader *msg;
  size_t total;
  size_t cpy;
  size_t left;
  uint16_t msize;

  total = size * nmemb;
  stat_bytes_downloaded += total;
  if ((total == 0) || (stat_bogus_url))
  {
    return total;   /* ok, no data or bogus data */
  }

  GNUNET_STATISTICS_update (stats,
                            gettext_noop (
                              "# bytes downloaded from hostlist servers"),
                            (int64_t) total,
                            GNUNET_NO);
  left = total;
  while ((left > 0) || (download_pos > 0))
  {
    cpy = GNUNET_MIN (left, GNUNET_MAX_MESSAGE_SIZE - 1 - download_pos);
    GNUNET_memcpy (&download_buffer[download_pos], cbuf, cpy);
    cbuf += cpy;
    download_pos += cpy;
    left -= cpy;
    if (download_pos < sizeof(struct GNUNET_MessageHeader))
    {
      GNUNET_assert (0 == left);
      break;
    }
    msg = (const struct GNUNET_MessageHeader *) download_buffer;
    msize = ntohs (msg->size);
    if (msize < sizeof(struct GNUNET_MessageHeader))
    {
      GNUNET_STATISTICS_update (
        stats,
        gettext_noop ("# invalid HELLOs downloaded from hostlist servers"),
        1,
        GNUNET_NO);
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  _ ("Invalid `%s' message received from hostlist at `%s'\n"),
                  "HELLO",
                  current_url);
      stat_hellos_obtained++;
      stat_bogus_url = 1;
      return total;
    }
    if (download_pos < msize)
    {
      GNUNET_assert (left == 0);
      break;
    }
    if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) msg) == msize)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Received valid `%s' message from hostlist server.\n",
                  "HELLO");
      GNUNET_STATISTICS_update (
        stats,
        gettext_noop ("# valid HELLOs downloaded from hostlist servers"),
        1,
        GNUNET_NO);
      stat_hellos_obtained++;
      (void)
      GNUNET_PEERINFO_add_peer (pi,
                                (const struct GNUNET_HELLO_Message *) msg,
                                NULL,
                                NULL);
    }
    else
    {
      GNUNET_STATISTICS_update (
        stats,
        gettext_noop ("# invalid HELLOs downloaded from hostlist servers"),
        1,
        GNUNET_NO);
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  _ ("Invalid `%s' message received from hostlist at `%s'\n"),
                  "HELLO",
                  current_url);
      stat_bogus_url = GNUNET_YES;
      stat_hellos_obtained++;
      return total;
    }
    memmove (download_buffer, &download_buffer[msize], download_pos - msize);
    download_pos -= msize;
  }
  return total;
}


static char *
get_bootstrap_server ()
{
  char *servers;
  char *ret;
  size_t urls;
  size_t pos;

  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg,
                                                          "HOSTLIST",
                                                          "SERVERS",
                                                          &servers))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist",
                               "SERVERS");
    return NULL;
  }

  urls = 0;
  if (strlen (servers) > 0)
  {
    urls++;
    pos = strlen (servers) - 1;
    while (pos > 0)
    {
      if (servers[pos] == ' ')
        urls++;
      pos--;
    }
  }
  if (urls == 0)
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist",
                               "SERVERS");
    GNUNET_free (servers);
    return NULL;
  }

  urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
  pos = strlen (servers) - 1;
  while (pos > 0)
  {
    if (servers[pos] == ' ')
    {
      urls--;
      servers[pos] = '\0';
    }
    if (urls == 0)
    {
      pos++;
      break;
    }
    pos--;
  }
  ret = GNUNET_strdup (&servers[pos]);
  GNUNET_free (servers);
  return ret;
}


static char *
download_get_url ()
{
  uint32_t index;
  unsigned int counter;
  struct Hostlist *pos;

  if (GNUNET_NO == stat_learning)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Using preconfigured bootstrap server\n");
    current_hostlist = NULL;
    return get_bootstrap_server ();
  }

  if ((GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Testing new advertised hostlist if it is obtainable\n");
    current_hostlist = hostlist_to_test;
    return GNUNET_strdup (hostlist_to_test->hostlist_uri);
  }

  if ((GNUNET_YES == stat_use_bootstrap) || (linked_list_size == 0))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Using preconfigured bootstrap server\n");
    current_hostlist = NULL;
    return get_bootstrap_server ();
  }
  index =
    GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
  counter = 0;
  pos = linked_list_head;
  while (counter < index)
  {
    pos = pos->next;
    counter++;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Using learned hostlist `%s'\n",
              pos->hostlist_uri);
  current_hostlist = pos;
  return GNUNET_strdup (pos->hostlist_uri);
}


#define CURL_EASY_SETOPT(c, a, b)                   \
  do                                                \
  {                                                 \
    ret = curl_easy_setopt (c, a, b);               \
    if (CURLE_OK != ret)                            \
      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,        \
                  _ ("%s failed at %s:%d: `%s'\n"), \
                  "curl_easy_setopt",               \
                  __FILE__,                         \
                  __LINE__,                         \
                  curl_easy_strerror (ret));        \
  } while (0)


static void
save_hostlist_file (int shutdown);


static uint64_t
checked_add (uint64_t val1, uint64_t val2)
{
  static uint64_t temp;
  static uint64_t maxv;

  maxv = 0;
  maxv--;

  temp = val1 + val2;
  if (temp < val1)
    return maxv;
  return temp;
}


static uint64_t
checked_sub (uint64_t val1, uint64_t val2)
{
  if (val1 <= val2)
    return 0;
  return(val1 - val2);
}


static int
linked_list_contains (const char *uri)
{
  struct Hostlist *pos;

  pos = linked_list_head;
  while (pos != NULL)
  {
    if (0 == strcmp (pos->hostlist_uri, uri))
      return GNUNET_YES;
    pos = pos->next;
  }
  return GNUNET_NO;
}


static struct Hostlist *
linked_list_get_lowest_quality ()
{
  struct Hostlist *pos;
  struct Hostlist *lowest;

  if (linked_list_size == 0)
    return NULL;
  lowest = linked_list_head;
  pos = linked_list_head->next;
  while (pos != NULL)
  {
    if (pos->quality < lowest->quality)
      lowest = pos;
    pos = pos->next;
  }
  return lowest;
}


static void
insert_hostlist ()
{
  struct Hostlist *lowest_quality;

  if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
  {
    /* No free entries available, replace existing entry  */
    lowest_quality = linked_list_get_lowest_quality ();
    GNUNET_assert (lowest_quality != NULL);
    GNUNET_log (
      GNUNET_ERROR_TYPE_DEBUG,
      "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
      lowest_quality->hostlist_uri,
      (unsigned long long) lowest_quality->quality);
    GNUNET_CONTAINER_DLL_remove (linked_list_head,
                                 linked_list_tail,
                                 lowest_quality);
    linked_list_size--;
    GNUNET_free (lowest_quality);
  }
  GNUNET_CONTAINER_DLL_insert (linked_list_head,
                               linked_list_tail,
                               hostlist_to_test);
  linked_list_size++;
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# advertised hostlist URIs"),
                         linked_list_size,
                         GNUNET_NO);
  stat_testing_hostlist = GNUNET_NO;
}


static void
update_hostlist ()
{
  char *stat;

  if (((stat_use_bootstrap == GNUNET_NO) && (NULL != current_hostlist)) ||
      ((stat_testing_hostlist == GNUNET_YES) && (NULL != current_hostlist)))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Updating hostlist statics for URI `%s'\n",
                current_hostlist->hostlist_uri);
    current_hostlist->hello_count = stat_hellos_obtained;
    current_hostlist->time_last_usage = GNUNET_TIME_absolute_get ();
    current_hostlist->quality =
      checked_add (current_hostlist->quality,
                   (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
    if (GNUNET_YES == stat_download_successful)
    {
      current_hostlist->times_used++;
      current_hostlist->quality =
        checked_add (current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
      GNUNET_asprintf (&stat,
                       gettext_noop ("# advertised URI `%s' downloaded"),
                       current_hostlist->hostlist_uri);

      GNUNET_STATISTICS_update (stats, stat, 1, GNUNET_YES);
      GNUNET_free (stat);
    }
    else
      current_hostlist->quality =
        checked_sub (current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD);
  }
  current_hostlist = NULL;
  /* Alternating the usage of preconfigured and learned hostlists */

  if (stat_testing_hostlist == GNUNET_YES)
    return;

  if (GNUNET_YES == stat_learning)
  {
    if (stat_use_bootstrap == GNUNET_YES)
      stat_use_bootstrap = GNUNET_NO;
    else
      stat_use_bootstrap = GNUNET_YES;
  }
  else
    stat_use_bootstrap = GNUNET_YES;
}


static void
clean_up ()
{
  CURLMcode mret;

  if ((stat_testing_hostlist == GNUNET_YES) &&
      (GNUNET_NO == stat_download_successful) && (NULL != hostlist_to_test))
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_INFO,
      _ (
        "Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
      hostlist_to_test->hostlist_uri);
  }

  if (stat_testing_hostlist == GNUNET_YES)
  {
    stat_testing_hostlist = GNUNET_NO;
  }
  if (NULL != hostlist_to_test)
  {
    GNUNET_free (hostlist_to_test);
    hostlist_to_test = NULL;
  }

  if (NULL != multi)
  {
    mret = curl_multi_remove_handle (multi, curl);
    if (mret != CURLM_OK)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  _ ("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_remove_handle",
                  __FILE__,
                  __LINE__,
                  curl_multi_strerror (mret));
    }
    mret = curl_multi_cleanup (multi);
    if (mret != CURLM_OK)
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  _ ("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_cleanup",
                  __FILE__,
                  __LINE__,
                  curl_multi_strerror (mret));
    multi = NULL;
  }
  if (NULL != curl)
  {
    curl_easy_cleanup (curl);
    curl = NULL;
  }
  GNUNET_free_non_null (current_url);
  current_url = NULL;
  stat_bytes_downloaded = 0;
  stat_download_in_progress = GNUNET_NO;
}


static void
task_download (void *cls);


static void
download_prepare ()
{
  CURLMcode mret;
  fd_set rs;
  fd_set ws;
  fd_set es;
  int max;
  struct GNUNET_NETWORK_FDSet *grs;
  struct GNUNET_NETWORK_FDSet *gws;
  long timeout;
  struct GNUNET_TIME_Relative rtime;

  max = -1;
  FD_ZERO (&rs);
  FD_ZERO (&ws);
  FD_ZERO (&es);
  mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("%s failed at %s:%d: `%s'\n"),
                "curl_multi_fdset",
                __FILE__,
                __LINE__,
                curl_multi_strerror (mret));
    clean_up ();
    return;
  }
  mret = curl_multi_timeout (multi, &timeout);
  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("%s failed at %s:%d: `%s'\n"),
                "curl_multi_timeout",
                __FILE__,
                __LINE__,
                curl_multi_strerror (mret));
    clean_up ();
    return;
  }
  rtime = GNUNET_TIME_relative_min (
    GNUNET_TIME_absolute_get_remaining (end_time),
    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, timeout));
  grs = GNUNET_NETWORK_fdset_create ();
  gws = GNUNET_NETWORK_fdset_create ();
  GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
  GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Scheduling task for hostlist download using cURL\n");
  ti_download = GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
                                             rtime,
                                             grs,
                                             gws,
                                             &task_download,
                                             multi);
  GNUNET_NETWORK_fdset_destroy (gws);
  GNUNET_NETWORK_fdset_destroy (grs);
}


static void
task_download (void *cls)
{
  int running;
  struct CURLMsg *msg;
  CURLMcode mret;

  ti_download = NULL;
  if (0 == GNUNET_TIME_absolute_get_remaining (end_time).rel_value_us)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Timeout trying to download hostlist from `%s'\n"),
                current_url);
    update_hostlist ();
    clean_up ();
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Ready for processing hostlist client request\n");
  do
  {
    running = 0;
    if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  _ (
                    "Download limit of %u bytes exceeded, stopping download\n"),
                  MAX_BYTES_PER_HOSTLISTS);
      clean_up ();
      return;
    }
    mret = curl_multi_perform (multi, &running);
    if (running == 0)
    {
      do
      {
        msg = curl_multi_info_read (multi, &running);
        GNUNET_break (msg != NULL);
        if (msg == NULL)
          break;
        switch (msg->msg)
        {
        case CURLMSG_DONE:
          if ((msg->data.result != CURLE_OK) &&
              (msg->data.result != CURLE_GOT_NOTHING))
            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                        _ ("Download of hostlist from `%s' failed: `%s'\n"),
                        current_url,
                        curl_easy_strerror (msg->data.result));
          else
          {
            GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                        _ ("Download of hostlist `%s' completed.\n"),
                        current_url);
            stat_download_successful = GNUNET_YES;
            update_hostlist ();
            if (GNUNET_YES == stat_testing_hostlist)
            {
              GNUNET_log (
                GNUNET_ERROR_TYPE_INFO,
                _ ("Adding successfully tested hostlist `%s' datastore.\n"),
                current_url);
              insert_hostlist ();
              hostlist_to_test = NULL;
              stat_testing_hostlist = GNUNET_NO;
            }
          }
          clean_up ();
          return;

        default:
          break;
        }
      }
      while ((running > 0));
    }
  }
  while (mret == CURLM_CALL_MULTI_PERFORM);

  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("%s failed at %s:%d: `%s'\n"),
                "curl_multi_perform",
                __FILE__,
                __LINE__,
                curl_multi_strerror (mret));
    clean_up ();
  }
  download_prepare ();
}


static void
download_hostlist ()
{
  CURLcode ret;
  CURLMcode mret;


  current_url = download_get_url ();
  if (current_url == NULL)
    return;
  curl = curl_easy_init ();
  multi = NULL;
  if (curl == NULL)
  {
    GNUNET_break (0);
    clean_up ();
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
              _ ("Bootstrapping using hostlist at `%s'.\n"),
              current_url);

  stat_download_in_progress = GNUNET_YES;
  stat_download_successful = GNUNET_NO;
  stat_hellos_obtained = 0;
  stat_bytes_downloaded = 0;

  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# hostlist downloads initiated"),
                            1,
                            GNUNET_NO);
  if (NULL != proxy)
  {
    CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
    CURL_EASY_SETOPT (curl, CURLOPT_PROXYTYPE, proxy_type);
    if (NULL != proxy_username)
      CURL_EASY_SETOPT (curl, CURLOPT_PROXYUSERNAME, proxy_username);
    if (NULL != proxy_password)
      CURL_EASY_SETOPT (curl, CURLOPT_PROXYPASSWORD, proxy_password);
  }
  download_pos = 0;
  stat_bogus_url = 0;
  CURL_EASY_SETOPT (curl, CURLOPT_WRITEFUNCTION, &callback_download);
  if (ret != CURLE_OK)
  {
    clean_up ();
    return;
  }
  CURL_EASY_SETOPT (curl, CURLOPT_WRITEDATA, NULL);
  if (ret != CURLE_OK)
  {
    clean_up ();
    return;
  }
  CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
  CURL_EASY_SETOPT (curl,
                    CURLOPT_REDIR_PROTOCOLS,
                    CURLPROTO_HTTP | CURLPROTO_HTTPS);
  CURL_EASY_SETOPT (curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
  CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
  /* no need to abort if the above failed */
  CURL_EASY_SETOPT (curl, CURLOPT_URL, current_url);
  if (ret != CURLE_OK)
  {
    clean_up ();
    return;
  }
  CURL_EASY_SETOPT (curl, CURLOPT_FAILONERROR, 1);
#if 0
  CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
#endif
  CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_MAX_MESSAGE_SIZE);
  if (0 == strncmp (current_url, "http", 4))
    CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
  CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
  CURL_EASY_SETOPT (curl, CURLOPT_TIMEOUT, 60L);
  multi = curl_multi_init ();
  if (multi == NULL)
  {
    GNUNET_break (0);
    /* clean_up (); */
    return;
  }
  mret = curl_multi_add_handle (multi, curl);
  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("%s failed at %s:%d: `%s'\n"),
                "curl_multi_add_handle",
                __FILE__,
                __LINE__,
                curl_multi_strerror (mret));
    mret = curl_multi_cleanup (multi);
    if (mret != CURLM_OK)
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  _ ("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_cleanup",
                  __FILE__,
                  __LINE__,
                  curl_multi_strerror (mret));
    multi = NULL;
    clean_up ();
    return;
  }
  end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
  download_prepare ();
}


static void
task_download_dispatcher (void *cls)
{
  ti_download_dispatcher_task = NULL;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download is initiated...\n");
  if (GNUNET_NO == stat_download_in_progress)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download can start immediately...\n");
    download_hostlist ();
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Download in progess, have to wait...\n");
    ti_download_dispatcher_task =
      GNUNET_SCHEDULER_add_delayed (WAITING_INTERVAL,
                                    &task_download_dispatcher,
                                    NULL);
  }
}


static void
task_check (void *cls)
{
  static int once;
  struct GNUNET_TIME_Relative delay;

  ti_check_download = NULL;
  if (stats == NULL)
  {
    curl_global_cleanup ();
    return;   /* in shutdown */
  }
  if ((stat_connection_count < MIN_CONNECTIONS) &&
      (NULL == ti_download_dispatcher_task))
    ti_download_dispatcher_task =
      GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);

  delay = hostlist_delay;
  if (0 == hostlist_delay.rel_value_us)
    hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
  else
    hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
  if (hostlist_delay.rel_value_us >
      GNUNET_TIME_UNIT_HOURS.rel_value_us * (1 + stat_connection_count))
    hostlist_delay =
      GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
                                     (1 + stat_connection_count));
  GNUNET_STATISTICS_set (stats,
                         gettext_noop (
                           "# milliseconds between hostlist downloads"),
                         hostlist_delay.rel_value_us / 1000LL,
                         GNUNET_YES);
  if (0 == once)
  {
    delay = GNUNET_TIME_UNIT_ZERO;
    once = 1;
  }
  GNUNET_log (
    GNUNET_ERROR_TYPE_INFO,
    _ ("Have %u/%u connections.  Will consider downloading hostlist in %s\n"),
    stat_connection_count,
    MIN_CONNECTIONS,
    GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
  ti_check_download = GNUNET_SCHEDULER_add_delayed (delay, &task_check, NULL);
}


static void
task_testing_intervall_reset (void *cls)
{
  ti_testing_intervall_task = NULL;
  stat_testing_allowed = GNUNET_OK;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Testing new hostlist advertisements is allowed again\n");
}


static void
task_hostlist_saving (void *cls)
{
  ti_saving_task = NULL;
  save_hostlist_file (GNUNET_NO);

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlists will be saved to file again in %s\n",
              GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVAL,
                                                      GNUNET_YES));
  ti_saving_task =
    GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL, &task_hostlist_saving, NULL);
}


static void *
handler_connect (void *cls,
                 const struct GNUNET_PeerIdentity *peer,
                 struct GNUNET_MQ_Handle *mq)
{
  GNUNET_assert (stat_connection_count < UINT_MAX);
  stat_connection_count++;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# active connections"),
                            1,
                            GNUNET_NO);
  return NULL;
}


static void
handler_disconnect (void *cls,
                    const struct GNUNET_PeerIdentity *peer,
                    void *internal_cls)
{
  GNUNET_assert (stat_connection_count > 0);
  stat_connection_count--;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# active connections"),
                            -1,
                            GNUNET_NO);
}


static void
handler_advertisement (const char *uri)
{
  size_t uri_size;
  struct Hostlist *hostlist;

  uri_size = strlen (uri) + 1;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlist client recieved advertisement containing URI `%s'\n",
              uri);
  if (GNUNET_NO != linked_list_contains (uri))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "URI `%s' is already known\n", uri);
    return;
  }

  if (GNUNET_NO == stat_testing_allowed)
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_DEBUG,
      "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
    return;
  }
  if (GNUNET_YES == stat_testing_hostlist)
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_DEBUG,
      "Currently not accepting new advertisements: we are already testing a hostlist\n");
    return;
  }

  hostlist = GNUNET_malloc (sizeof(struct Hostlist) + uri_size);
  hostlist->hostlist_uri = (const char *) &hostlist[1];
  GNUNET_memcpy (&hostlist[1], uri, uri_size);
  hostlist->time_creation = GNUNET_TIME_absolute_get ();
  hostlist->quality = HOSTLIST_INITIAL;
  hostlist_to_test = hostlist;

  stat_testing_hostlist = GNUNET_YES;
  stat_testing_allowed = GNUNET_NO;
  ti_testing_intervall_task =
    GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
                                  &task_testing_intervall_reset,
                                  NULL);

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Testing new hostlist advertisements is locked for the next %s\n",
              GNUNET_STRINGS_relative_time_to_string (TESTING_INTERVAL,
                                                      GNUNET_YES));

  ti_download_dispatcher_task =
    GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
}


static void
primary_task (void *cls, int success)
{
  if (NULL != ti_check_download)
  {
    GNUNET_SCHEDULER_cancel (ti_check_download);
    ti_check_download = NULL;
  }
  sget = NULL;
  GNUNET_assert (NULL != stats);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Statistics request done, scheduling hostlist download\n");
  ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
}


static void
stat_timeout_task (void *cls)
{
  GNUNET_STATISTICS_get_cancel (sget);
  sget = NULL;
  ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
}


static int
process_stat (void *cls,
              const char *subsystem,
              const char *name,
              uint64_t value,
              int is_persistent)
{
  hostlist_delay.rel_value_us = value * 1000LL;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Initial time between hostlist downloads is %s\n",
              GNUNET_STRINGS_relative_time_to_string (hostlist_delay,
                                                      GNUNET_YES));
  return GNUNET_OK;
}


static void
load_hostlist_file ()
{
  char *filename;
  char *uri;
  char *emsg;
  struct Hostlist *hostlist;
  uint32_t times_used;
  uint32_t hellos_returned;
  uint64_t quality;
  uint64_t last_used;
  uint64_t created;
  uint32_t counter;
  struct GNUNET_BIO_ReadHandle *rh;

  uri = NULL;
  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                            "HOSTLIST",
                                                            "HOSTLISTFILE",
                                                            &filename))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist",
                               "HOSTLISTFILE");
    return;
  }

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _ ("Loading saved hostlist entries from file `%s' \n"),
              filename);
  if (GNUNET_NO == GNUNET_DISK_file_test (filename))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Hostlist file `%s' does not exist\n"),
                filename);
    GNUNET_free (filename);
    return;
  }

  rh = GNUNET_BIO_read_open (filename);
  if (NULL == rh)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ (
                  "Could not open file `%s' for reading to load hostlists: %s\n"),
                filename,
                strerror (errno));
    GNUNET_free (filename);
    return;
  }

  counter = 0;
  while ((GNUNET_OK == GNUNET_BIO_read_string (rh, "url", &uri, MAX_URL_LEN)) &&
         (NULL != uri) &&
         (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &times_used)) &&
         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
         (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)))
  {
    hostlist = GNUNET_malloc (sizeof(struct Hostlist) + strlen (uri) + 1);
    hostlist->hello_count = hellos_returned;
    hostlist->hostlist_uri = (const char *) &hostlist[1];
    GNUNET_memcpy (&hostlist[1], uri, strlen (uri) + 1);
    hostlist->quality = quality;
    hostlist->time_creation.abs_value_us = created;
    hostlist->time_last_usage.abs_value_us = last_used;
    GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail, hostlist);
    linked_list_size++;
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Added hostlist entry eith URI `%s' \n",
                hostlist->hostlist_uri);
    GNUNET_free (uri);
    uri = NULL;
    counter++;
    if (counter >= MAX_NUMBER_HOSTLISTS)
      break;
  }

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _ ("%u hostlist URIs loaded from file\n"),
              counter);
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# hostlist URIs read from file"),
                         counter,
                         GNUNET_YES);
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# advertised hostlist URIs"),
                         linked_list_size,
                         GNUNET_NO);

  GNUNET_free_non_null (uri);
  emsg = NULL;
  (void) GNUNET_BIO_read_close (rh, &emsg);
  if (emsg != NULL)
    GNUNET_free (emsg);
  GNUNET_free (filename);
}


static void
save_hostlist_file (int shutdown)
{
  char *filename;
  struct Hostlist *pos;
  struct GNUNET_BIO_WriteHandle *wh;
  int ok;
  uint32_t counter;

  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                            "HOSTLIST",
                                                            "HOSTLISTFILE",
                                                            &filename))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist",
                               "HOSTLISTFILE");
    return;
  }
  if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
  {
    GNUNET_free (filename);
    return;
  }
  wh = GNUNET_BIO_write_open (filename);
  if (NULL == wh)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ (
                  "Could not open file `%s' for writing to save hostlists: %s\n"),
                filename,
                strerror (errno));
    GNUNET_free (filename);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _ ("Writing %u hostlist URIs to `%s'\n"),
              linked_list_size,
              filename);
  /* add code to write hostlists to file using bio */
  ok = GNUNET_YES;
  counter = 0;
  while (NULL != (pos = linked_list_head))
  {
    if (GNUNET_YES == shutdown)
    {
      GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
      linked_list_size--;
    }
    if (GNUNET_YES == ok)
    {
      if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
          (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
          (GNUNET_OK != GNUNET_BIO_write_int64 (wh, pos->quality)) ||
          (GNUNET_OK !=
           GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value_us)) ||
          (GNUNET_OK !=
           GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value_us)) ||
          (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->hello_count)))
      {
        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                    _ ("Error writing hostlist URIs to file `%s'\n"),
                    filename);
        ok = GNUNET_NO;
      }
    }

    if (GNUNET_YES == shutdown)
      GNUNET_free (pos);
    counter++;
    if (counter >= MAX_NUMBER_HOSTLISTS)
      break;
  }
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# hostlist URIs written to file"),
                         counter,
                         GNUNET_YES);

  if (GNUNET_OK != GNUNET_BIO_write_close (wh))
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Error writing hostlist URIs to file `%s'\n"),
                filename);
  GNUNET_free (filename);
}


int
GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
                              struct GNUNET_STATISTICS_Handle *st,
                              GNUNET_CORE_ConnectEventHandler *ch,
                              GNUNET_CORE_DisconnectEventHandler *dh,
                              GNUNET_HOSTLIST_UriHandler *msgh,
                              int learn)
{
  char *filename;
  char *proxytype_str;
  int result;

  GNUNET_assert (NULL != st);
  if (0 != curl_global_init (CURL_GLOBAL_WIN32))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  cfg = c;
  stats = st;

  /* Read proxy configuration */
  pi = GNUNET_PEERINFO_connect (c);
  if (GNUNET_OK ==
      GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "PROXY", &proxy))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Found proxy host: `%s'\n", proxy);
    /* proxy username */
    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
                                                            "HOSTLIST",
                                                            "PROXY_USERNAME",
                                                            &proxy_username))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Found proxy username name: `%s'\n",
                  proxy_username);
    }

    /* proxy password */
    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
                                                            "HOSTLIST",
                                                            "PROXY_PASSWORD",
                                                            &proxy_password))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Found proxy password name: `%s'\n",
                  proxy_password);
    }

    /* proxy type */
    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
                                                            "HOSTLIST",
                                                            "PROXY_TYPE",
                                                            &proxytype_str))
    {
      GNUNET_STRINGS_utf8_toupper (proxytype_str, proxytype_str);
      proxy_type = CURLPROXY_HTTP;
      if (0 == strcmp (proxytype_str, "HTTP"))
        proxy_type = CURLPROXY_HTTP;
      else if (0 == strcmp (proxytype_str, "HTTP_1_0"))
        proxy_type = CURLPROXY_HTTP_1_0;
      else if (0 == strcmp (proxytype_str, "SOCKS4"))
        proxy_type = CURLPROXY_SOCKS4;
      else if (0 == strcmp (proxytype_str, "SOCKS5"))
        proxy_type = CURLPROXY_SOCKS5;
      else if (0 == strcmp (proxytype_str, "SOCKS4A"))
        proxy_type = CURLPROXY_SOCKS4A;
      else if (0 == strcmp (proxytype_str, "SOCKS5_HOSTNAME"))
        proxy_type = CURLPROXY_SOCKS5_HOSTNAME;
      else
      {
        GNUNET_log (
          GNUNET_ERROR_TYPE_ERROR,
          _ (
            "Invalid proxy type: `%s', disabling proxy! Check configuration!\n"),
          proxytype_str);
        GNUNET_free (proxytype_str);
        GNUNET_free (proxy);
        proxy = NULL;
        GNUNET_free_non_null (proxy_username);
        proxy_username = NULL;
        GNUNET_free_non_null (proxy_password);
        proxy_password = NULL;

        return GNUNET_SYSERR;
      }
    }
    GNUNET_free_non_null (proxytype_str);
  }

  stat_learning = learn;
  *ch = &handler_connect;
  *dh = &handler_disconnect;
  linked_list_head = NULL;
  linked_list_tail = NULL;
  stat_use_bootstrap = GNUNET_YES;
  stat_testing_hostlist = GNUNET_NO;
  stat_testing_allowed = GNUNET_YES;

  if (GNUNET_YES == stat_learning)
  {
    *msgh = &handler_advertisement;
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Learning is enabled on this peer\n"));
    load_hostlist_file ();
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Hostlists will be saved to file again in %s\n",
                GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVAL,
                                                        GNUNET_YES));
    ti_saving_task = GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL,
                                                   &task_hostlist_saving,
                                                   NULL);
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Learning is not enabled on this peer\n"));
    *msgh = NULL;
    if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                              "HOSTLIST",
                                                              "HOSTLISTFILE",
                                                              &filename))
    {
      if (GNUNET_YES == GNUNET_DISK_file_test (filename))
      {
        result = remove (filename);
        if (0 == result)
          GNUNET_log (
            GNUNET_ERROR_TYPE_INFO,
            _ (
              "Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
            filename);
        else
          GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
                                    "remove",
                                    filename);
      }
    }
    GNUNET_free (filename);
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Loading stats value on hostlist download frequency\n");
  sget = GNUNET_STATISTICS_get (stats,
                                "hostlist",
                                gettext_noop (
                                  "# milliseconds between hostlist downloads"),
                                &primary_task,
                                &process_stat,
                                NULL);
  if (NULL == sget)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Statistics request failed, scheduling hostlist download\n");
    ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
  }
  else
  {
    ti_check_download = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
                                                      &stat_timeout_task,
                                                      NULL);
  }
  return GNUNET_OK;
}


void
GNUNET_HOSTLIST_client_stop ()
{
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Hostlist client shutdown\n");
  if (NULL != sget)
  {
    GNUNET_STATISTICS_get_cancel (sget);
    sget = NULL;
  }
  stats = NULL;
  if (GNUNET_YES == stat_learning)
    save_hostlist_file (GNUNET_YES);
  if (NULL != ti_saving_task)
  {
    GNUNET_SCHEDULER_cancel (ti_saving_task);
    ti_saving_task = NULL;
  }
  if (NULL != ti_download_dispatcher_task)
  {
    GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
    ti_download_dispatcher_task = NULL;
  }
  if (NULL != ti_testing_intervall_task)
  {
    GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
    ti_testing_intervall_task = NULL;
  }
  if (NULL != ti_download)
  {
    GNUNET_SCHEDULER_cancel (ti_download);
    ti_download = NULL;
    update_hostlist ();
    clean_up ();
  }
  if (NULL != ti_check_download)
  {
    GNUNET_SCHEDULER_cancel (ti_check_download);
    ti_check_download = NULL;
    curl_global_cleanup ();
  }
  GNUNET_free_non_null (proxy);
  proxy = NULL;
  GNUNET_free_non_null (proxy_username);
  proxy_username = NULL;
  GNUNET_free_non_null (proxy_password);
  proxy_password = NULL;
  if (NULL != pi)
  {
    GNUNET_PEERINFO_disconnect (pi);
    pi = NULL;
  }
  cfg = NULL;
}


/* end of gnunet-daemon-hostlist_client.c */