gnunet-daemon-hostlist_client.c

Go to the documentation of this file.

/*
     This file is part of GNUnet.
     Copyright (C) 2001-2010, 2014, 2016 GNUnet e.V.

     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.
    
     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.

     SPDX-License-Identifier: AGPL3.0-or-later
*/
#include "platform.h"
#include "gnunet-daemon-hostlist_client.h"
#include "gnunet_hello_lib.h"
#include "gnunet_statistics_service.h"
#include "gnunet_transport_service.h"
#include "gnunet_peerinfo_service.h"
#include "gnunet-daemon-hostlist.h"
/* Just included for the right curl.h */
#include "gnunet_curl_lib.h"



#define MIN_CONNECTIONS 4

#define MAX_NUMBER_HOSTLISTS 30

#define SAVING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)

#define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3)

#define WAITING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)

#define HOSTLIST_INITIAL 10000

#define HOSTLIST_FAILED_DOWNLOAD 100

#define HOSTLIST_SUCCESSFUL_DOWNLOAD 100

#define HOSTLIST_SUCCESSFUL_HELLO 1



struct Hostlist
{
  struct Hostlist *prev;

  struct Hostlist *next;

  const char *hostlist_uri;

  uint64_t quality;

  struct GNUNET_TIME_Absolute time_creation;

  struct GNUNET_TIME_Absolute time_last_usage;

  uint32_t hello_count;

  uint32_t times_used;

};


static const struct GNUNET_CONFIGURATION_Handle *cfg;

static struct GNUNET_STATISTICS_Handle *stats;

static char *proxy;

static char *proxy_username;

static char *proxy_password;

static curl_proxytype proxy_type;

static size_t download_pos;

static char *current_url;

static CURL *curl;

static CURLM *multi;

static uint32_t stat_bytes_downloaded;

static struct GNUNET_TIME_Relative hostlist_delay;

static struct GNUNET_SCHEDULER_Task *ti_check_download;

static struct GNUNET_SCHEDULER_Task *ti_download;

static struct GNUNET_SCHEDULER_Task *ti_saving_task;

static struct GNUNET_SCHEDULER_Task *ti_download_dispatcher_task;

static struct GNUNET_SCHEDULER_Task *ti_testing_intervall_task;

static struct GNUNET_TIME_Absolute end_time;

static struct Hostlist *linked_list_head;

static struct Hostlist *linked_list_tail;

static struct Hostlist *current_hostlist;

static unsigned int linked_list_size;

static struct Hostlist *hostlist_to_test;

static struct GNUNET_STATISTICS_GetHandle *sget;

static int stat_bogus_url;

static int stat_testing_hostlist;

static int stat_testing_allowed;

static int stat_download_in_progress;

static unsigned int stat_use_bootstrap;

static int stat_learning;

static unsigned int stat_download_successful;

static unsigned int stat_hellos_obtained;

static unsigned int stat_connection_count;

static struct GNUNET_PEERINFO_Handle *pi;


static size_t
callback_download (void *ptr,
                   size_t size,
                   size_t nmemb,
                   void *ctx)
{
  static char download_buffer[GNUNET_MAX_MESSAGE_SIZE - 1];
  const char *cbuf = ptr;
  const struct GNUNET_MessageHeader *msg;
  size_t total;
  size_t cpy;
  size_t left;
  uint16_t msize;

  total = size * nmemb;
  stat_bytes_downloaded += total;
  if ((total == 0) || (stat_bogus_url))
  {
    return total;               /* ok, no data or bogus data */
  }

  GNUNET_STATISTICS_update (stats,
                            gettext_noop
                            ("# bytes downloaded from hostlist servers"),
                            (int64_t) total, GNUNET_NO);
  left = total;
  while ((left > 0) || (download_pos > 0))
  {
    cpy = GNUNET_MIN (left, GNUNET_MAX_MESSAGE_SIZE - 1 - download_pos);
    GNUNET_memcpy (&download_buffer[download_pos], cbuf, cpy);
    cbuf += cpy;
    download_pos += cpy;
    left -= cpy;
    if (download_pos < sizeof (struct GNUNET_MessageHeader))
    {
      GNUNET_assert (0 == left);
      break;
    }
    msg = (const struct GNUNET_MessageHeader *) download_buffer;
    msize = ntohs (msg->size);
    if (msize < sizeof (struct GNUNET_MessageHeader))
    {
      GNUNET_STATISTICS_update (stats,
                                gettext_noop
                                ("# invalid HELLOs downloaded from hostlist servers"),
                                1, GNUNET_NO);
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  _("Invalid `%s' message received from hostlist at `%s'\n"),
                  "HELLO", current_url);
      stat_hellos_obtained++;
      stat_bogus_url = 1;
      return total;
    }
    if (download_pos < msize)
    {
      GNUNET_assert (left == 0);
      break;
    }
    if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) msg) == msize)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Received valid `%s' message from hostlist server.\n",
                  "HELLO");
      GNUNET_STATISTICS_update (stats,
                                gettext_noop
                                ("# valid HELLOs downloaded from hostlist servers"),
                                1, GNUNET_NO);
      stat_hellos_obtained++;
      (void) GNUNET_PEERINFO_add_peer (pi,
                                       (const struct GNUNET_HELLO_Message *) msg,
                                       NULL,
                                       NULL);
    }
    else
    {
      GNUNET_STATISTICS_update (stats,
                                gettext_noop
                                ("# invalid HELLOs downloaded from hostlist servers"),
                                1, GNUNET_NO);
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  _("Invalid `%s' message received from hostlist at `%s'\n"),
                  "HELLO", current_url);
      stat_bogus_url = GNUNET_YES;
      stat_hellos_obtained++;
      return total;
    }
    memmove (download_buffer,
             &download_buffer[msize],
             download_pos - msize);
    download_pos -= msize;
  }
  return total;
}


static char *
get_bootstrap_server ()
{
  char *servers;
  char *ret;
  size_t urls;
  size_t pos;

  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "SERVERS",
                                             &servers))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist", "SERVERS");
    return NULL;
  }

  urls = 0;
  if (strlen (servers) > 0)
  {
    urls++;
    pos = strlen (servers) - 1;
    while (pos > 0)
    {
      if (servers[pos] == ' ')
        urls++;
      pos--;
    }
  }
  if (urls == 0)
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist", "SERVERS");
    GNUNET_free (servers);
    return NULL;
  }

  urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
  pos = strlen (servers) - 1;
  while (pos > 0)
  {
    if (servers[pos] == ' ')
    {
      urls--;
      servers[pos] = '\0';
    }
    if (urls == 0)
    {
      pos++;
      break;
    }
    pos--;
  }
  ret = GNUNET_strdup (&servers[pos]);
  GNUNET_free (servers);
  return ret;
}


static char *
download_get_url ()
{
  uint32_t index;
  unsigned int counter;
  struct Hostlist *pos;

  if (GNUNET_NO == stat_learning)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Using preconfigured bootstrap server\n");
    current_hostlist = NULL;
    return get_bootstrap_server ();
  }

  if ((GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Testing new advertised hostlist if it is obtainable\n");
    current_hostlist = hostlist_to_test;
    return GNUNET_strdup (hostlist_to_test->hostlist_uri);
  }

  if ((GNUNET_YES == stat_use_bootstrap) || (linked_list_size == 0))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Using preconfigured bootstrap server\n");
    current_hostlist = NULL;
    return get_bootstrap_server ();
  }
  index =
      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
  counter = 0;
  pos = linked_list_head;
  while (counter < index)
  {
    pos = pos->next;
    counter++;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Using learned hostlist `%s'\n",
              pos->hostlist_uri);
  current_hostlist = pos;
  return GNUNET_strdup (pos->hostlist_uri);
}


#define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt (c, a, b); if (CURLE_OK != ret) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror (ret)); } while (0)


static void
save_hostlist_file (int shutdown);


static uint64_t
checked_add (uint64_t val1,
             uint64_t val2)
{
  static uint64_t temp;
  static uint64_t maxv;

  maxv = 0;
  maxv--;

  temp = val1 + val2;
  if (temp < val1)
    return maxv;
  return temp;
}


static uint64_t
checked_sub (uint64_t val1,
             uint64_t val2)
{
  if (val1 <= val2)
    return 0;
  return (val1 - val2);
}


static int
linked_list_contains (const char *uri)
{
  struct Hostlist *pos;

  pos = linked_list_head;
  while (pos != NULL)
  {
    if (0 == strcmp (pos->hostlist_uri, uri))
      return GNUNET_YES;
    pos = pos->next;
  }
  return GNUNET_NO;
}


static struct Hostlist *
linked_list_get_lowest_quality ()
{
  struct Hostlist *pos;
  struct Hostlist *lowest;

  if (linked_list_size == 0)
    return NULL;
  lowest = linked_list_head;
  pos = linked_list_head->next;
  while (pos != NULL)
  {
    if (pos->quality < lowest->quality)
      lowest = pos;
    pos = pos->next;
  }
  return lowest;
}


static void
insert_hostlist ()
{
  struct Hostlist *lowest_quality;

  if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
  {
    /* No free entries available, replace existing entry  */
    lowest_quality = linked_list_get_lowest_quality ();
    GNUNET_assert (lowest_quality != NULL);
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
                lowest_quality->hostlist_uri,
                (unsigned long long) lowest_quality->quality);
    GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail,
                                 lowest_quality);
    linked_list_size--;
    GNUNET_free (lowest_quality);
  }
  GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail,
                               hostlist_to_test);
  linked_list_size++;
  GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
                         linked_list_size, GNUNET_NO);
  stat_testing_hostlist = GNUNET_NO;
}


static void
update_hostlist ()
{
  char *stat;

  if (((stat_use_bootstrap == GNUNET_NO) && (NULL != current_hostlist)) ||
      ((stat_testing_hostlist == GNUNET_YES) && (NULL != current_hostlist)))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Updating hostlist statics for URI `%s'\n",
                current_hostlist->hostlist_uri);
    current_hostlist->hello_count = stat_hellos_obtained;
    current_hostlist->time_last_usage = GNUNET_TIME_absolute_get ();
    current_hostlist->quality =
        checked_add (current_hostlist->quality,
                     (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
    if (GNUNET_YES == stat_download_successful)
    {
      current_hostlist->times_used++;
      current_hostlist->quality =
          checked_add (current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
      GNUNET_asprintf (&stat, gettext_noop ("# advertised URI `%s' downloaded"),
                       current_hostlist->hostlist_uri);

      GNUNET_STATISTICS_update (stats, stat, 1, GNUNET_YES);
      GNUNET_free (stat);
    }
    else
      current_hostlist->quality =
          checked_sub (current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD);
  }
  current_hostlist = NULL;
  /* Alternating the usage of preconfigured and learned hostlists */

  if (stat_testing_hostlist == GNUNET_YES)
    return;

  if (GNUNET_YES == stat_learning)
  {
    if (stat_use_bootstrap == GNUNET_YES)
      stat_use_bootstrap = GNUNET_NO;
    else
      stat_use_bootstrap = GNUNET_YES;
  }
  else
    stat_use_bootstrap = GNUNET_YES;
}


static void
clean_up ()
{
  CURLMcode mret;

  if ( (stat_testing_hostlist == GNUNET_YES) &&
       (GNUNET_NO == stat_download_successful) &&
       (NULL != hostlist_to_test) )
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _
                ("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
                hostlist_to_test->hostlist_uri);
  }

  if (stat_testing_hostlist == GNUNET_YES)
  {
    stat_testing_hostlist = GNUNET_NO;
  }
  if (NULL != hostlist_to_test)
  {
    GNUNET_free (hostlist_to_test);
    hostlist_to_test = NULL;
  }

  if (NULL != multi)
  {
    mret = curl_multi_remove_handle (multi, curl);
    if (mret != CURLM_OK)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_remove_handle", __FILE__, __LINE__,
                  curl_multi_strerror (mret));
    }
    mret = curl_multi_cleanup (multi);
    if (mret != CURLM_OK)
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_cleanup", __FILE__, __LINE__,
                  curl_multi_strerror (mret));
    multi = NULL;
  }
  if (NULL != curl)
  {
    curl_easy_cleanup (curl);
    curl = NULL;
  }
  GNUNET_free_non_null (current_url);
  current_url = NULL;
  stat_bytes_downloaded = 0;
  stat_download_in_progress = GNUNET_NO;
}


static void
task_download (void *cls);


static void
download_prepare ()
{
  CURLMcode mret;
  fd_set rs;
  fd_set ws;
  fd_set es;
  int max;
  struct GNUNET_NETWORK_FDSet *grs;
  struct GNUNET_NETWORK_FDSet *gws;
  long timeout;
  struct GNUNET_TIME_Relative rtime;

  max = -1;
  FD_ZERO (&rs);
  FD_ZERO (&ws);
  FD_ZERO (&es);
  mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
                "curl_multi_fdset", __FILE__, __LINE__,
                curl_multi_strerror (mret));
    clean_up ();
    return;
  }
  mret = curl_multi_timeout (multi, &timeout);
  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
                "curl_multi_timeout", __FILE__, __LINE__,
                curl_multi_strerror (mret));
    clean_up ();
    return;
  }
  rtime =
      GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
                                GNUNET_TIME_relative_multiply
                                (GNUNET_TIME_UNIT_MILLISECONDS, timeout));
  grs = GNUNET_NETWORK_fdset_create ();
  gws = GNUNET_NETWORK_fdset_create ();
  GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
  GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Scheduling task for hostlist download using cURL\n");
  ti_download =
      GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
                                   rtime, grs, gws,
                                   &task_download, multi);
  GNUNET_NETWORK_fdset_destroy (gws);
  GNUNET_NETWORK_fdset_destroy (grs);
}


static void
task_download (void *cls)
{
  int running;
  struct CURLMsg *msg;
  CURLMcode mret;

  ti_download = NULL;
  if (0 == GNUNET_TIME_absolute_get_remaining (end_time).rel_value_us)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _("Timeout trying to download hostlist from `%s'\n"),
                current_url);
    update_hostlist ();
    clean_up ();
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Ready for processing hostlist client request\n");
  do
  {
    running = 0;
    if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  _("Download limit of %u bytes exceeded, stopping download\n"),
                  MAX_BYTES_PER_HOSTLISTS);
      clean_up ();
      return;
    }
    mret = curl_multi_perform (multi, &running);
    if (running == 0)
    {
      do
      {
        msg = curl_multi_info_read (multi, &running);
        GNUNET_break (msg != NULL);
        if (msg == NULL)
          break;
        switch (msg->msg)
        {
        case CURLMSG_DONE:
          if ((msg->data.result != CURLE_OK) &&
              (msg->data.result != CURLE_GOT_NOTHING))
            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                        _("Download of hostlist from `%s' failed: `%s'\n"),
                        current_url,
                        curl_easy_strerror (msg->data.result));
          else
          {
            GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                        _("Download of hostlist `%s' completed.\n"),
                        current_url);
            stat_download_successful = GNUNET_YES;
            update_hostlist ();
            if (GNUNET_YES == stat_testing_hostlist)
            {
              GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                          _
                          ("Adding successfully tested hostlist `%s' datastore.\n"),
                          current_url);
              insert_hostlist ();
              hostlist_to_test = NULL;
              stat_testing_hostlist = GNUNET_NO;
            }
          }
          clean_up ();
          return;
        default:
          break;
        }

      }
      while ((running > 0));
    }
  }
  while (mret == CURLM_CALL_MULTI_PERFORM);

  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%s failed at %s:%d: `%s'\n"),
                "curl_multi_perform", __FILE__, __LINE__,
                curl_multi_strerror (mret));
    clean_up ();
  }
  download_prepare ();
}


static void
download_hostlist ()
{
  CURLcode ret;
  CURLMcode mret;


  current_url = download_get_url ();
  if (current_url == NULL)
    return;
  curl = curl_easy_init ();
  multi = NULL;
  if (curl == NULL)
  {
    GNUNET_break (0);
    clean_up ();
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
              _("Bootstrapping using hostlist at `%s'.\n"), current_url);

  stat_download_in_progress = GNUNET_YES;
  stat_download_successful = GNUNET_NO;
  stat_hellos_obtained = 0;
  stat_bytes_downloaded = 0;

  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# hostlist downloads initiated"), 1,
                            GNUNET_NO);
  if (NULL != proxy)
  {
    CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
    CURL_EASY_SETOPT (curl, CURLOPT_PROXYTYPE, proxy_type);
    if (NULL != proxy_username)
      CURL_EASY_SETOPT (curl, CURLOPT_PROXYUSERNAME, proxy_username);
    if (NULL != proxy_password)
      CURL_EASY_SETOPT (curl, CURLOPT_PROXYPASSWORD, proxy_password);
  }
  download_pos = 0;
  stat_bogus_url = 0;
  CURL_EASY_SETOPT (curl, CURLOPT_WRITEFUNCTION, &callback_download);
  if (ret != CURLE_OK)
  {
    clean_up ();
    return;
  }
  CURL_EASY_SETOPT (curl, CURLOPT_WRITEDATA, NULL);
  if (ret != CURLE_OK)
  {
    clean_up ();
    return;
  }
  CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
  CURL_EASY_SETOPT (curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
  CURL_EASY_SETOPT (curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
  CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
  /* no need to abort if the above failed */
  CURL_EASY_SETOPT (curl, CURLOPT_URL, current_url);
  if (ret != CURLE_OK)
  {
    clean_up ();
    return;
  }
  CURL_EASY_SETOPT (curl, CURLOPT_FAILONERROR, 1);
#if 0
  CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
#endif
  CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_MAX_MESSAGE_SIZE);
  if (0 == strncmp (current_url, "http", 4))
    CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
  CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
  CURL_EASY_SETOPT (curl, CURLOPT_TIMEOUT, 60L);
  multi = curl_multi_init ();
  if (multi == NULL)
  {
    GNUNET_break (0);
    /* clean_up (); */
    return;
  }
  mret = curl_multi_add_handle (multi, curl);
  if (mret != CURLM_OK)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
                "curl_multi_add_handle", __FILE__, __LINE__,
                curl_multi_strerror (mret));
    mret = curl_multi_cleanup (multi);
    if (mret != CURLM_OK)
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
                  "curl_multi_cleanup", __FILE__, __LINE__,
                  curl_multi_strerror (mret));
    multi = NULL;
    clean_up ();
    return;
  }
  end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
  download_prepare ();
}


static void
task_download_dispatcher (void *cls)
{
  ti_download_dispatcher_task = NULL;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download is initiated...\n");
  if (GNUNET_NO == stat_download_in_progress)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download can start immediately...\n");
    download_hostlist ();
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Download in progess, have to wait...\n");
    ti_download_dispatcher_task =
        GNUNET_SCHEDULER_add_delayed (WAITING_INTERVAL,
                                      &task_download_dispatcher, NULL);
  }
}


static void
task_check (void *cls)
{
  static int once;
  struct GNUNET_TIME_Relative delay;

  ti_check_download = NULL;
  if (stats == NULL)
  {
    curl_global_cleanup ();
    return;                     /* in shutdown */
  }
  if ( (stat_connection_count < MIN_CONNECTIONS) &&
       (NULL == ti_download_dispatcher_task) )
    ti_download_dispatcher_task =
        GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);

  delay = hostlist_delay;
  if (0 == hostlist_delay.rel_value_us)
    hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
  else
    hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
  if (hostlist_delay.rel_value_us >
      GNUNET_TIME_UNIT_HOURS.rel_value_us * (1 + stat_connection_count))
    hostlist_delay =
        GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
                                       (1 + stat_connection_count));
  GNUNET_STATISTICS_set (stats,
                         gettext_noop
                         ("# milliseconds between hostlist downloads"),
                         hostlist_delay.rel_value_us / 1000LL,
                         GNUNET_YES);
  if (0 == once)
  {
    delay = GNUNET_TIME_UNIT_ZERO;
    once = 1;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Have %u/%u connections.  Will consider downloading hostlist in %s\n"),
              stat_connection_count, MIN_CONNECTIONS,
              GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
  ti_check_download = GNUNET_SCHEDULER_add_delayed (delay,
                                                    &task_check, NULL);
}


static void
task_testing_intervall_reset (void *cls)
{
  ti_testing_intervall_task = NULL;
  stat_testing_allowed = GNUNET_OK;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Testing new hostlist advertisements is allowed again\n");
}


static void
task_hostlist_saving (void *cls)
{
  ti_saving_task = NULL;
  save_hostlist_file (GNUNET_NO);

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlists will be saved to file again in %s\n",
              GNUNET_STRINGS_relative_time_to_string(SAVING_INTERVAL, GNUNET_YES));
  ti_saving_task =
      GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL,
                                    &task_hostlist_saving,
                                    NULL);
}


static void *
handler_connect (void *cls,
                 const struct GNUNET_PeerIdentity *peer,
                 struct GNUNET_MQ_Handle *mq)
{
  GNUNET_assert (stat_connection_count < UINT_MAX);
  stat_connection_count++;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# active connections"),
                            1,
                            GNUNET_NO);
  return NULL;
}


static void
handler_disconnect (void *cls,
                    const struct GNUNET_PeerIdentity *peer,
                    void *internal_cls)
{
  GNUNET_assert (stat_connection_count > 0);
  stat_connection_count--;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# active connections"),
                            -1,
                            GNUNET_NO);
}


static void
handler_advertisement (const char *uri)
{
  size_t uri_size;
  struct Hostlist *hostlist;

  uri_size = strlen (uri) + 1;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlist client recieved advertisement containing URI `%s'\n",
              uri);
  if (GNUNET_NO != linked_list_contains (uri))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "URI `%s' is already known\n",
                uri);
    return;
  }

  if (GNUNET_NO == stat_testing_allowed)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
    return;
  }
  if (GNUNET_YES == stat_testing_hostlist)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Currently not accepting new advertisements: we are already testing a hostlist\n");
    return;
  }

  hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
  hostlist->hostlist_uri = (const char *) &hostlist[1];
  GNUNET_memcpy (&hostlist[1],
                 uri,
                 uri_size);
  hostlist->time_creation = GNUNET_TIME_absolute_get ();
  hostlist->quality = HOSTLIST_INITIAL;
  hostlist_to_test = hostlist;

  stat_testing_hostlist = GNUNET_YES;
  stat_testing_allowed = GNUNET_NO;
  ti_testing_intervall_task =
      GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
                                    &task_testing_intervall_reset,
                                    NULL);

  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Testing new hostlist advertisements is locked for the next %s\n",
              GNUNET_STRINGS_relative_time_to_string (TESTING_INTERVAL,
                                                      GNUNET_YES));

  ti_download_dispatcher_task =
      GNUNET_SCHEDULER_add_now (&task_download_dispatcher,
                                NULL);
}


static void
primary_task (void *cls,
              int success)
{
  if (NULL != ti_check_download)
  {
    GNUNET_SCHEDULER_cancel (ti_check_download);
    ti_check_download = NULL;
  }
  sget = NULL;
  GNUNET_assert (NULL != stats);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Statistics request done, scheduling hostlist download\n");
  ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
}


static void
stat_timeout_task (void *cls)
{
  GNUNET_STATISTICS_get_cancel (sget);
  sget = NULL;
  ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
                                                NULL);
}


static int
process_stat (void *cls,
              const char *subsystem,
              const char *name,
              uint64_t value,
              int is_persistent)
{
  hostlist_delay.rel_value_us = value * 1000LL;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Initial time between hostlist downloads is %s\n",
              GNUNET_STRINGS_relative_time_to_string (hostlist_delay,
                                                      GNUNET_YES));
  return GNUNET_OK;
}


static void
load_hostlist_file ()
{
  char *filename;
  char *uri;
  char *emsg;
  struct Hostlist *hostlist;
  uint32_t times_used;
  uint32_t hellos_returned;
  uint64_t quality;
  uint64_t last_used;
  uint64_t created;
  uint32_t counter;
  struct GNUNET_BIO_ReadHandle *rh;

  uri = NULL;
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
                                               &filename))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist", "HOSTLISTFILE");
    return;
  }

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Loading saved hostlist entries from file `%s' \n"),
              filename);
  if (GNUNET_NO == GNUNET_DISK_file_test (filename))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _("Hostlist file `%s' does not exist\n"), filename);
    GNUNET_free (filename);
    return;
  }

  rh = GNUNET_BIO_read_open (filename);
  if (NULL == rh)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _("Could not open file `%s' for reading to load hostlists: %s\n"),
                filename, STRERROR (errno));
    GNUNET_free (filename);
    return;
  }

  counter = 0;
  while ((GNUNET_OK == GNUNET_BIO_read_string (rh, "url", &uri, MAX_URL_LEN)) &&
         (NULL != uri) && (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &times_used))
         && (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
         (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
         (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)))
  {
    hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
    hostlist->hello_count = hellos_returned;
    hostlist->hostlist_uri = (const char *) &hostlist[1];
    GNUNET_memcpy (&hostlist[1], uri, strlen (uri) + 1);
    hostlist->quality = quality;
    hostlist->time_creation.abs_value_us = created;
    hostlist->time_last_usage.abs_value_us = last_used;
    GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail, hostlist);
    linked_list_size++;
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Added hostlist entry eith URI `%s' \n",
                hostlist->hostlist_uri);
    GNUNET_free (uri);
    uri = NULL;
    counter++;
    if (counter >= MAX_NUMBER_HOSTLISTS)
      break;
  }

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("%u hostlist URIs loaded from file\n"),
              counter);
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# hostlist URIs read from file"),
                         counter, GNUNET_YES);
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# advertised hostlist URIs"),
                         linked_list_size, GNUNET_NO);

  GNUNET_free_non_null (uri);
  emsg = NULL;
  (void) GNUNET_BIO_read_close (rh, &emsg);
  if (emsg != NULL)
    GNUNET_free (emsg);
  GNUNET_free (filename);
}


static void
save_hostlist_file (int shutdown)
{
  char *filename;
  struct Hostlist *pos;
  struct GNUNET_BIO_WriteHandle *wh;
  int ok;
  uint32_t counter;

  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
                                               &filename))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
                               "hostlist", "HOSTLISTFILE");
    return;
  }
  if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
  {
    GNUNET_free (filename);
    return;
  }
  wh = GNUNET_BIO_write_open (filename);
  if (NULL == wh)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _("Could not open file `%s' for writing to save hostlists: %s\n"),
                filename,
                STRERROR (errno));
    GNUNET_free (filename);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _("Writing %u hostlist URIs to `%s'\n"),
              linked_list_size, filename);
  /* add code to write hostlists to file using bio */
  ok = GNUNET_YES;
  counter = 0;
  while (NULL != (pos = linked_list_head))
  {
    if (GNUNET_YES == shutdown)
    {
      GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
      linked_list_size--;
    }
    if (GNUNET_YES == ok)
    {
      if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
          (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
          (GNUNET_OK != GNUNET_BIO_write_int64 (wh, pos->quality)) ||
          (GNUNET_OK !=
           GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value_us)) ||
          (GNUNET_OK !=
           GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value_us)) ||
          (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->hello_count)))
      {
        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                    _("Error writing hostlist URIs to file `%s'\n"), filename);
        ok = GNUNET_NO;
      }
    }

    if (GNUNET_YES == shutdown)
      GNUNET_free (pos);
    counter++;
    if (counter >= MAX_NUMBER_HOSTLISTS)
      break;
  }
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# hostlist URIs written to file"),
                         counter, GNUNET_YES);

  if (GNUNET_OK != GNUNET_BIO_write_close (wh))
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _("Error writing hostlist URIs to file `%s'\n"), filename);
  GNUNET_free (filename);
}


int
GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
                              struct GNUNET_STATISTICS_Handle *st,
                              GNUNET_CORE_ConnectEventHandler *ch,
                              GNUNET_CORE_DisconnectEventHandler *dh,
                              GNUNET_HOSTLIST_UriHandler *msgh,
                              int learn)
{
  char *filename;
  char *proxytype_str;
  int result;

  GNUNET_assert (NULL != st);
  if (0 != curl_global_init (CURL_GLOBAL_WIN32))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  cfg = c;
  stats = st;

  /* Read proxy configuration */
  pi = GNUNET_PEERINFO_connect (c);
  if (GNUNET_OK ==
      GNUNET_CONFIGURATION_get_value_string (cfg,
                                             "HOSTLIST",
                                             "PROXY",
                                             &proxy))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                     "Found proxy host: `%s'\n",
                     proxy);
    /* proxy username */
    if (GNUNET_OK ==
        GNUNET_CONFIGURATION_get_value_string (cfg,
                                               "HOSTLIST",
                                               "PROXY_USERNAME",
                                               &proxy_username))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Found proxy username name: `%s'\n",
                  proxy_username);
    }

    /* proxy password */
    if (GNUNET_OK ==
        GNUNET_CONFIGURATION_get_value_string (cfg,
                                               "HOSTLIST",
                                               "PROXY_PASSWORD",
                                               &proxy_password))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       "Found proxy password name: `%s'\n",
                       proxy_password);
    }

    /* proxy type */
    if (GNUNET_OK ==
        GNUNET_CONFIGURATION_get_value_string (cfg,
                                               "HOSTLIST",
                                               "PROXY_TYPE",
                                               &proxytype_str))
    {
      GNUNET_STRINGS_utf8_toupper (proxytype_str,
                                   proxytype_str);
      proxy_type = CURLPROXY_HTTP;
      if (0 == strcmp (proxytype_str, "HTTP"))
        proxy_type = CURLPROXY_HTTP;
      else if (0 == strcmp (proxytype_str, "HTTP_1_0"))
        proxy_type = CURLPROXY_HTTP_1_0;
      else if (0 == strcmp (proxytype_str, "SOCKS4"))
        proxy_type = CURLPROXY_SOCKS4;
      else if (0 == strcmp (proxytype_str, "SOCKS5"))
        proxy_type = CURLPROXY_SOCKS5;
      else if (0 == strcmp (proxytype_str, "SOCKS4A"))
        proxy_type = CURLPROXY_SOCKS4A;
      else if (0 == strcmp (proxytype_str, "SOCKS5_HOSTNAME"))
        proxy_type = CURLPROXY_SOCKS5_HOSTNAME;
      else
      {
        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                    _("Invalid proxy type: `%s', disabling proxy! Check configuration!\n"),
                    proxytype_str);
        GNUNET_free (proxytype_str);
        GNUNET_free (proxy);
        proxy = NULL;
        GNUNET_free_non_null (proxy_username);
        proxy_username = NULL;
        GNUNET_free_non_null (proxy_password);
        proxy_password = NULL;

        return GNUNET_SYSERR;
      }
    }
    GNUNET_free_non_null (proxytype_str);
  }

  stat_learning = learn;
  *ch = &handler_connect;
  *dh = &handler_disconnect;
  linked_list_head = NULL;
  linked_list_tail = NULL;
  stat_use_bootstrap = GNUNET_YES;
  stat_testing_hostlist = GNUNET_NO;
  stat_testing_allowed = GNUNET_YES;

  if (GNUNET_YES == stat_learning)
  {
    *msgh = &handler_advertisement;
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _("Learning is enabled on this peer\n"));
    load_hostlist_file ();
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Hostlists will be saved to file again in %s\n",
                GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVAL,
                                                        GNUNET_YES));
    ti_saving_task =
        GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL,
                                      &task_hostlist_saving,
                                      NULL);
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _("Learning is not enabled on this peer\n"));
    *msgh = NULL;
    if (GNUNET_OK ==
        GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                 "HOSTLIST",
                                                 "HOSTLISTFILE",
                                                 &filename))
    {
      if (GNUNET_YES == GNUNET_DISK_file_test (filename))
      {
        result = remove (filename);
        if (0 == result)
          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                      _("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
                      filename);
        else
          GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
                                    "remove",
                                    filename);
      }
    }
    GNUNET_free (filename);
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Loading stats value on hostlist download frequency\n");
  sget = GNUNET_STATISTICS_get (stats, "hostlist",
                                gettext_noop
                                ("# milliseconds between hostlist downloads"),
                                &primary_task,
                                &process_stat,
                                NULL);
  if (NULL == sget)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Statistics request failed, scheduling hostlist download\n");
    ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
                                                  NULL);
  }
  else
  {
    ti_check_download
      = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
                                      &stat_timeout_task,
                                      NULL);
  }
  return GNUNET_OK;
}


void
GNUNET_HOSTLIST_client_stop ()
{
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Hostlist client shutdown\n");
  if (NULL != sget)
  {
    GNUNET_STATISTICS_get_cancel (sget);
    sget = NULL;
  }
  stats = NULL;
  if (GNUNET_YES == stat_learning)
    save_hostlist_file (GNUNET_YES);
  if (NULL != ti_saving_task)
  {
    GNUNET_SCHEDULER_cancel (ti_saving_task);
    ti_saving_task = NULL;
  }
  if (NULL != ti_download_dispatcher_task)
    {
    GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
    ti_download_dispatcher_task = NULL;
  }
  if (NULL != ti_testing_intervall_task)
  {
    GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
    ti_testing_intervall_task = NULL;
  }
  if (NULL != ti_download)
  {
    GNUNET_SCHEDULER_cancel (ti_download);
    ti_download = NULL;
    update_hostlist ();
    clean_up ();
  }
  if (NULL != ti_check_download)
  {
    GNUNET_SCHEDULER_cancel (ti_check_download);
    ti_check_download = NULL;
    curl_global_cleanup ();
  }
  GNUNET_free_non_null (proxy);
  proxy = NULL;
  GNUNET_free_non_null (proxy_username);
  proxy_username = NULL;
  GNUNET_free_non_null (proxy_password);
  proxy_password = NULL;
  if (NULL != pi)
  {
    GNUNET_PEERINFO_disconnect (pi);
    pi = NULL;
  }
  cfg = NULL;
}

/* end of gnunet-daemon-hostlist_client.c */