gnunet-zoneimport.c

Go to the documentation of this file.

/*
     This file is part of GNUnet
     Copyright (C) 2018 GNUnet e.V.

     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.

     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.

     SPDX-License-Identifier: AGPL3.0-or-later
 */
#include "platform.h"
#include <gnunet_util_lib.h>
#include <gnunet_dnsstub_lib.h>
#include <gnunet_dnsparser_lib.h>
#include <gnunet_gnsrecord_lib.h>
#include <gnunet_namestore_service.h>
#include <gnunet_statistics_service.h>
#include <gnunet_identity_service.h>


#define THRESH 100

#define TIME_THRESH 10

#define MAX_RETRIES 5

#define MAX_SERIES 10

#define SERIES_DELAY \
  GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MICROSECONDS, 10)

static struct GNUNET_TIME_Relative minimum_expiration_time;

#define NS_BATCH_SIZE 1024

struct Zone {
  struct Zone *next;

  struct Zone *prev;

  char *domain;

  struct GNUNET_CRYPTO_EcdsaPrivateKey key;
};


struct Record {
  struct Record *next;

  struct Record *prev;

  struct GNUNET_GNSRECORD_Data grd;
};


struct Request {
  struct GNUNET_CONTAINER_HeapNode *hn;

  struct Request *next;

  struct Request *prev;

  struct Record *rec_head;

  struct Record *rec_tail;

  struct GNUNET_DNSSTUB_RequestSocket *rs;

  char *hostname;

  struct GNUNET_NAMESTORE_QueueEntry *qe;

  const struct Zone *zone;

  struct GNUNET_TIME_Absolute expires;

  struct GNUNET_TIME_Absolute op_start_time;

  unsigned int issue_num;

  uint16_t id;
};


static unsigned int map_size = 1024;

static struct GNUNET_IDENTITY_Handle *id;

static struct GNUNET_NAMESTORE_Handle *ns;

static struct GNUNET_STATISTICS_Handle *stats;

static struct GNUNET_DNSSTUB_Context *ctx;

static unsigned int pending;

static unsigned int pending_rs;

static unsigned int lookups;

static unsigned int cached;

static unsigned int rejects;

static unsigned int failures;

static unsigned int records;

static unsigned int record_sets;

static struct GNUNET_CONTAINER_Heap *req_heap;

static struct Request *req_head;

static struct Request *req_tail;

static struct GNUNET_SCHEDULER_Task *t;

static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;

static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;

static struct Zone *zone_head;

static struct Zone *zone_tail;

static uint64_t ns_iterator_trigger_next;

static uint64_t total_dns_latency_cnt;

static struct GNUNET_TIME_Relative total_dns_latency;

static uint64_t total_reg_proc_dns;

static uint64_t total_reg_proc_dns_ns;

static struct GNUNET_TIME_Absolute start_time_reg_proc;

static struct GNUNET_TIME_Absolute sleep_time_reg_proc;

static struct GNUNET_TIME_Relative idle_time;


typedef void (*RecordProcessor) (void *cls,
                                 const struct GNUNET_DNSPARSER_Record *rec);


static void
for_all_records(const struct GNUNET_DNSPARSER_Packet *p,
                RecordProcessor rp,
                void *rp_cls)
{
  for (unsigned int i = 0; i < p->num_answers; i++)
    {
      struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];

      rp(rp_cls, rs);
    }
  for (unsigned int i = 0; i < p->num_authority_records; i++)
    {
      struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];

      rp(rp_cls, rs);
    }
  for (unsigned int i = 0; i < p->num_additional_records; i++)
    {
      struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];

      rp(rp_cls, rs);
    }
}


static const char *
get_label(struct Request *req)
{
  static char label[64];
  const char *dot;

  dot = strchr(req->hostname, (unsigned char)'.');
  if (NULL == dot)
    {
      GNUNET_break(0);
      return NULL;
    }
  if (((size_t)(dot - req->hostname)) >= sizeof(label))
    {
      GNUNET_break(0);
      return NULL;
    }
  GNUNET_memcpy(label, req->hostname, dot - req->hostname);
  label[dot - req->hostname] = '\0';
  return label;
}


static void *
build_dns_query(struct Request *req, size_t *raw_size)
{
  static char raw[512];
  char *rawp;
  struct GNUNET_DNSPARSER_Packet p;
  struct GNUNET_DNSPARSER_Query q;
  int ret;

  q.name = (char *)req->hostname;
  q.type = GNUNET_DNSPARSER_TYPE_NS;
  q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;

  memset(&p, 0, sizeof(p));
  p.num_queries = 1;
  p.queries = &q;
  p.id = req->id;
  ret = GNUNET_DNSPARSER_pack(&p, UINT16_MAX, &rawp, raw_size);
  if (GNUNET_OK != ret)
    {
      if (GNUNET_NO == ret)
        GNUNET_free(rawp);
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Failed to pack query for hostname `%s'\n",
                 req->hostname);
      rejects++;
      return NULL;
    }
  if (*raw_size > sizeof(raw))
    {
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Failed to pack query for hostname `%s'\n",
                 req->hostname);
      rejects++;
      GNUNET_break(0);
      GNUNET_free(rawp);
      return NULL;
    }
  GNUNET_memcpy(raw, rawp, *raw_size);
  GNUNET_free(rawp);
  return raw;
}


static void
free_records(struct Request *req)
{
  struct Record *rec;

  /* Free records */
  while (NULL != (rec = req->rec_head))
    {
      GNUNET_CONTAINER_DLL_remove(req->rec_head, req->rec_tail, rec);
      GNUNET_free(rec);
    }
}


static void
free_request(struct Request *req)
{
  free_records(req);
  GNUNET_free(req);
}


static void
process_queue(void *cls);


static void
insert_sorted(struct Request *req)
{
  req->hn =
    GNUNET_CONTAINER_heap_insert(req_heap, req, req->expires.abs_value_us);
  if (req == GNUNET_CONTAINER_heap_peek(req_heap))
    {
      if (NULL != t)
        GNUNET_SCHEDULER_cancel(t);
      sleep_time_reg_proc = GNUNET_TIME_absolute_get();
      t = GNUNET_SCHEDULER_add_at(req->expires, &process_queue, NULL);
    }
}


static void
add_record(struct Request *req,
           uint32_t type,
           struct GNUNET_TIME_Absolute expiration_time,
           const void *data,
           size_t data_len)
{
  struct Record *rec;

  rec = GNUNET_malloc(sizeof(struct Record) + data_len);
  rec->grd.data = &rec[1];
  rec->grd.expiration_time = expiration_time.abs_value_us;
  rec->grd.data_size = data_len;
  rec->grd.record_type = type;
  rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
  GNUNET_memcpy(&rec[1], data, data_len);
  GNUNET_CONTAINER_DLL_insert(req->rec_head, req->rec_tail, rec);
}


struct GlueClosure {
  struct Request *req;

  const char *ns;

  int found;
};


static void
check_for_glue(void *cls, const struct GNUNET_DNSPARSER_Record *rec)
{
  struct GlueClosure *gc = cls;
  char dst[65536];
  size_t dst_len;
  size_t off;
  char ip[INET6_ADDRSTRLEN + 1];
  socklen_t ip_size = (socklen_t)sizeof(ip);
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;

  if (0 != strcasecmp(rec->name, gc->ns))
    return;
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining(expiration_time);
  if (0 == left.rel_value_us)
    return; /* ignore expired glue records */
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute(minimum_expiration_time);
  dst_len = sizeof(dst);
  off = 0;
  switch (rec->type)
    {
    case GNUNET_DNSPARSER_TYPE_A:
      if (sizeof(struct in_addr) != rec->data.raw.data_len)
        {
          GNUNET_break(0);
          return;
        }
      if (NULL == inet_ntop(AF_INET, rec->data.raw.data, ip, ip_size))
        {
          GNUNET_break(0);
          return;
        }
      if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                          dst_len,
                                                          &off,
                                                          gc->req->hostname)) &&
          (GNUNET_OK ==
           GNUNET_DNSPARSER_builder_add_name(dst, dst_len, &off, ip)))
        {
          add_record(gc->req,
                     GNUNET_GNSRECORD_TYPE_GNS2DNS,
                     expiration_time,
                     dst,
                     off);
          gc->found = GNUNET_YES;
        }
      break;

    case GNUNET_DNSPARSER_TYPE_AAAA:
      if (sizeof(struct in6_addr) != rec->data.raw.data_len)
        {
          GNUNET_break(0);
          return;
        }
      if (NULL == inet_ntop(AF_INET6, rec->data.raw.data, ip, ip_size))
        {
          GNUNET_break(0);
          return;
        }
      if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                          dst_len,
                                                          &off,
                                                          gc->req->hostname)) &&
          (GNUNET_OK ==
           GNUNET_DNSPARSER_builder_add_name(dst, dst_len, &off, ip)))
        {
          add_record(gc->req,
                     GNUNET_GNSRECORD_TYPE_GNS2DNS,
                     expiration_time,
                     dst,
                     off);
          gc->found = GNUNET_YES;
        }
      break;

    case GNUNET_DNSPARSER_TYPE_CNAME:
      if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                          dst_len,
                                                          &off,
                                                          gc->req->hostname)) &&
          (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                          dst_len,
                                                          &off,
                                                          rec->data.hostname)))
        {
          add_record(gc->req,
                     GNUNET_GNSRECORD_TYPE_GNS2DNS,
                     expiration_time,
                     dst,
                     off);
          gc->found = GNUNET_YES;
        }
      break;

    default:
      /* useless, do nothing */
      break;
    }
}


struct ProcessRecordContext {
  struct GNUNET_DNSPARSER_Packet *p;

  struct Request *req;
};


static void
process_record(void *cls, const struct GNUNET_DNSPARSER_Record *rec)
{
  struct ProcessRecordContext *prc = cls;
  struct Request *req = prc->req;
  char dst[65536];
  size_t dst_len;
  size_t off;
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;

  dst_len = sizeof(dst);
  off = 0;
  records++;
  if (0 != strcasecmp(rec->name, req->hostname))
    {
      GNUNET_log(
        GNUNET_ERROR_TYPE_DEBUG,
        "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
        rec->name,
        (unsigned int)rec->type,
        req->hostname);
      return; /* does not match hostname, might be glue, but
                 not useful for this pass! */
    }
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining(expiration_time);
  if (0 == left.rel_value_us)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "DNS returned expired record for `%s'\n",
                 req->hostname);
      GNUNET_STATISTICS_update(stats,
                               "# expired records obtained from DNS",
                               1,
                               GNUNET_NO);
      return; /* record expired */
    }

  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
             "DNS returned record that expires at %s for `%s'\n",
             GNUNET_STRINGS_absolute_time_to_string(expiration_time),
             req->hostname);
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute(minimum_expiration_time);
  switch (rec->type)
    {
    case GNUNET_DNSPARSER_TYPE_NS: {
      struct GlueClosure gc;

      /* check for glue */
      gc.req = req;
      gc.ns = rec->data.hostname;
      gc.found = GNUNET_NO;
      for_all_records(prc->p, &check_for_glue, &gc);
      if ((GNUNET_NO == gc.found) &&
          (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                          dst_len,
                                                          &off,
                                                          req->hostname)) &&
          (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                          dst_len,
                                                          &off,
                                                          rec->data.hostname)))
        {
          /* FIXME: actually check if this is out-of-bailiwick,
               and if not request explicit resolution... */
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converted OOB (`%s') NS record for `%s'\n",
                     rec->data.hostname,
                     rec->name);
          add_record(req,
                     GNUNET_GNSRECORD_TYPE_GNS2DNS,
                     expiration_time,
                     dst,
                     off);
        }
      else
        {
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converted NS record for `%s' using glue\n",
                     rec->name);
        }
      break;
    }

    case GNUNET_DNSPARSER_TYPE_CNAME:
      if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                         dst_len,
                                                         &off,
                                                         rec->data.hostname))
        {
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converting CNAME (`%s') record for `%s'\n",
                     rec->data.hostname,
                     rec->name);
          add_record(req, rec->type, expiration_time, dst, off);
        }
      break;

    case GNUNET_DNSPARSER_TYPE_DNAME:
      /* No support for DNAME in GNS yet! FIXME: support later! */
      GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
                 "FIXME: not supported: %s DNAME %s\n",
                 rec->name,
                 rec->data.hostname);
      break;

    case GNUNET_DNSPARSER_TYPE_MX:
      if (GNUNET_OK ==
          GNUNET_DNSPARSER_builder_add_mx(dst, dst_len, &off, rec->data.mx))
        {
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converting MX (`%s') record for `%s'\n",
                     rec->data.mx->mxhost,
                     rec->name);
          add_record(req, rec->type, expiration_time, dst, off);
        }
      break;

    case GNUNET_DNSPARSER_TYPE_SOA:
      if (GNUNET_OK ==
          GNUNET_DNSPARSER_builder_add_soa(dst, dst_len, &off, rec->data.soa))
        {
          /* NOTE: GNS does not really use SOAs */
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converting SOA record for `%s'\n",
                     rec->name);
          add_record(req, rec->type, expiration_time, dst, off);
        }
      break;

    case GNUNET_DNSPARSER_TYPE_SRV:
      if (GNUNET_OK ==
          GNUNET_DNSPARSER_builder_add_srv(dst, dst_len, &off, rec->data.srv))
        {
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converting SRV record for `%s'\n",
                     rec->name);
          add_record(req, rec->type, expiration_time, dst, off);
        }
      break;

    case GNUNET_DNSPARSER_TYPE_PTR:
      if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name(dst,
                                                         dst_len,
                                                         &off,
                                                         rec->data.hostname))
        {
          /* !?: what does a PTR record do in a regular TLD??? */
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converting PTR record for `%s' (weird)\n",
                     rec->name);
          add_record(req, rec->type, expiration_time, dst, off);
        }
      break;

    case GNUNET_DNSPARSER_TYPE_CERT:
      if (GNUNET_OK ==
          GNUNET_DNSPARSER_builder_add_cert(dst, dst_len, &off, rec->data.cert))
        {
          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                     "Converting CERT record for `%s'\n",
                     rec->name);
          add_record(req, rec->type, expiration_time, dst, off);
        }
      break;

    /* Rest is 'raw' encoded and just needs to be copied IF
       the hostname matches the requested name; otherwise we
       simply cannot use it. */
    case GNUNET_DNSPARSER_TYPE_A:
    case GNUNET_DNSPARSER_TYPE_AAAA:
    case GNUNET_DNSPARSER_TYPE_TXT:
    default:
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "Converting record of type %u for `%s'\n",
                 (unsigned int)rec->type,
                 rec->name);
      add_record(req,
                 rec->type,
                 expiration_time,
                 rec->data.raw.data,
                 rec->data.raw.data_len);
      break;
    }
}


static void
store_completed_cb(void *cls, int32_t success, const char *emsg)
{
  static struct GNUNET_TIME_Absolute last;
  struct Request *req = cls;

  req->qe = NULL;
  if (GNUNET_SYSERR == success)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Failed to store zone data for `%s': %s\n",
                 req->hostname,
                 emsg);
    }
  else
    {
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "Stored records under `%s' (%d)\n",
                 req->hostname,
                 success);
    }
  total_reg_proc_dns_ns++; /* finished regular processing */
  pending_rs--;
  free_records(req);
  /* compute NAMESTORE statistics */
  {
    static uint64_t total_ns_latency_cnt;
    static struct GNUNET_TIME_Relative total_ns_latency;
    struct GNUNET_TIME_Relative ns_latency;

    ns_latency = GNUNET_TIME_absolute_get_duration(req->op_start_time);
    total_ns_latency = GNUNET_TIME_relative_add(total_ns_latency, ns_latency);
    if (0 == total_ns_latency_cnt)
      last = GNUNET_TIME_absolute_get();
    total_ns_latency_cnt++;
    if (0 == (total_ns_latency_cnt % 1000))
      {
        struct GNUNET_TIME_Relative delta;

        delta = GNUNET_TIME_absolute_get_duration(last);
        last = GNUNET_TIME_absolute_get();
        fprintf(stderr,
                "Processed 1000 records in %s\n",
                GNUNET_STRINGS_relative_time_to_string(delta, GNUNET_YES));
        GNUNET_STATISTICS_set(stats,
                              "# average NAMESTORE PUT latency (μs)",
                              total_ns_latency.rel_value_us /
                              total_ns_latency_cnt,
                              GNUNET_NO);
      }
  }
  /* compute and publish overall velocity */
  if (0 == (total_reg_proc_dns_ns % 100))
    {
      struct GNUNET_TIME_Relative runtime;

      runtime = GNUNET_TIME_absolute_get_duration(start_time_reg_proc);
      runtime = GNUNET_TIME_relative_subtract(runtime, idle_time);
      runtime =
        GNUNET_TIME_relative_divide(runtime,
                                    total_reg_proc_dns + total_reg_proc_dns_ns);
      GNUNET_STATISTICS_set(stats,
                            "# Regular processing completed without NAMESTORE",
                            total_reg_proc_dns,
                            GNUNET_NO);
      GNUNET_STATISTICS_set(stats,
                            "# Regular processing completed with NAMESTORE PUT",
                            total_reg_proc_dns_ns,
                            GNUNET_NO);
      GNUNET_STATISTICS_set(stats,
                            "# average request processing latency (μs)",
                            runtime.rel_value_us,
                            GNUNET_NO);
      GNUNET_STATISTICS_set(stats,
                            "# total time spent idle (μs)",
                            idle_time.rel_value_us,
                            GNUNET_NO);
    }

  if (NULL == t)
    {
      sleep_time_reg_proc = GNUNET_TIME_absolute_get();
      t = GNUNET_SCHEDULER_add_now(&process_queue, NULL);
    }
}


static void
process_result(void *cls,
               const struct GNUNET_TUN_DnsHeader *dns,
               size_t dns_len)
{
  struct Request *req = cls;
  struct Record *rec;
  struct GNUNET_DNSPARSER_Packet *p;
  unsigned int rd_count;

  GNUNET_assert(NULL == req->hn);
  if (NULL == dns)
    {
      /* stub gave up */
      GNUNET_CONTAINER_DLL_remove(req_head, req_tail, req);
      pending--;
      if (NULL == t)
        {
          sleep_time_reg_proc = GNUNET_TIME_absolute_get();
          t = GNUNET_SCHEDULER_add_now(&process_queue, NULL);
        }
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Stub gave up on DNS reply for `%s'\n",
                 req->hostname);
      GNUNET_STATISTICS_update(stats, "# DNS lookups timed out", 1, GNUNET_NO);
      if (req->issue_num > MAX_RETRIES)
        {
          failures++;
          free_request(req);
          GNUNET_STATISTICS_update(stats, "# requests given up on", 1, GNUNET_NO);
          return;
        }
      total_reg_proc_dns++;
      req->rs = NULL;
      insert_sorted(req);
      return;
    }
  if (req->id != dns->id)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "DNS ID did not match request, ignoring reply\n");
      GNUNET_STATISTICS_update(stats, "# DNS ID mismatches", 1, GNUNET_NO);
      return;
    }
  GNUNET_CONTAINER_DLL_remove(req_head, req_tail, req);
  GNUNET_DNSSTUB_resolve_cancel(req->rs);
  req->rs = NULL;
  pending--;
  p = GNUNET_DNSPARSER_parse((const char *)dns, dns_len);
  if (NULL == p)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Failed to parse DNS reply for `%s'\n",
                 req->hostname);
      GNUNET_STATISTICS_update(stats, "# DNS parser errors", 1, GNUNET_NO);
      if (NULL == t)
        {
          sleep_time_reg_proc = GNUNET_TIME_absolute_get();
          t = GNUNET_SCHEDULER_add_now(&process_queue, NULL);
        }
      if (req->issue_num > MAX_RETRIES)
        {
          failures++;
          free_request(req);
          GNUNET_STATISTICS_update(stats, "# requests given up on", 1, GNUNET_NO);
          return;
        }
      insert_sorted(req);
      return;
    }
  /* import new records */
  req->issue_num = 0; /* success, reset counter! */
  {
    struct ProcessRecordContext prc = { .req = req, .p = p };

    for_all_records(p, &process_record, &prc);
  }
  GNUNET_DNSPARSER_free_packet(p);
  /* count records found, determine minimum expiration time */
  req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
  {
    struct GNUNET_TIME_Relative dns_latency;

    dns_latency = GNUNET_TIME_absolute_get_duration(req->op_start_time);
    total_dns_latency =
      GNUNET_TIME_relative_add(total_dns_latency, dns_latency);
    total_dns_latency_cnt++;
    if (0 == (total_dns_latency_cnt % 1000))
      {
        GNUNET_STATISTICS_set(stats,
                              "# average DNS lookup latency (μs)",
                              total_dns_latency.rel_value_us /
                              total_dns_latency_cnt,
                              GNUNET_NO);
      }
  }
  rd_count = 0;
  for (rec = req->rec_head; NULL != rec; rec = rec->next)
    {
      struct GNUNET_TIME_Absolute at;

      at.abs_value_us = rec->grd.expiration_time;
      req->expires = GNUNET_TIME_absolute_min(req->expires, at);
      rd_count++;
    }
  GNUNET_log(GNUNET_ERROR_TYPE_INFO,
             "Obtained %u records for `%s'\n",
             rd_count,
             req->hostname);
  /* Instead of going for SOA, simplified for now to look each
     day in case we got an empty response */
  if (0 == rd_count)
    {
      req->expires = GNUNET_TIME_relative_to_absolute(GNUNET_TIME_UNIT_DAYS);
      GNUNET_STATISTICS_update(stats,
                               "# empty DNS replies (usually NXDOMAIN)",
                               1,
                               GNUNET_NO);
    }
  else
    {
      record_sets++;
    }
  /* convert records to namestore import format */
  {
    struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
    unsigned int off = 0;

    /* convert linked list into array */
    for (rec = req->rec_head; NULL != rec; rec = rec->next)
      rd[off++] = rec->grd;
    pending_rs++;
    req->op_start_time = GNUNET_TIME_absolute_get();
    req->qe = GNUNET_NAMESTORE_records_store(ns,
                                             &req->zone->key,
                                             get_label(req),
                                             rd_count,
                                             rd,
                                             &store_completed_cb,
                                             req);
    GNUNET_assert(NULL != req->qe);
  }
  insert_sorted(req);
}


static void
process_queue(void *cls)
{
  struct Request *req;
  unsigned int series;
  void *raw;
  size_t raw_size;
  struct GNUNET_TIME_Relative delay;

  (void)cls;
  delay = GNUNET_TIME_absolute_get_duration(sleep_time_reg_proc);
  idle_time = GNUNET_TIME_relative_add(idle_time, delay);
  series = 0;
  t = NULL;
  while (pending + pending_rs < THRESH)
    {
      req = GNUNET_CONTAINER_heap_peek(req_heap);
      if (NULL == req)
        break;
      if (NULL != req->qe)
        return; /* namestore op still pending */
      if (NULL != req->rs)
        {
          GNUNET_break(0);
          return; /* already submitted */
        }
      if (GNUNET_TIME_absolute_get_remaining(req->expires).rel_value_us > 0)
        break;
      GNUNET_assert(req == GNUNET_CONTAINER_heap_remove_root(req_heap));
      req->hn = NULL;
      GNUNET_CONTAINER_DLL_insert(req_head, req_tail, req);
      GNUNET_assert(NULL == req->rs);
      GNUNET_log(GNUNET_ERROR_TYPE_INFO,
                 "Requesting resolution for `%s'\n",
                 req->hostname);
      raw = build_dns_query(req, &raw_size);
      if (NULL == raw)
        {
          GNUNET_break(0);
          free_request(req);
          continue;
        }
      req->op_start_time = GNUNET_TIME_absolute_get();
      req->rs = GNUNET_DNSSTUB_resolve(ctx, raw, raw_size, &process_result, req);
      GNUNET_assert(NULL != req->rs);
      req->issue_num++;
      lookups++;
      pending++;
      series++;
      if (series > MAX_SERIES)
        break;
    }
  if (pending + pending_rs >= THRESH)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "Stopped processing queue (%u+%u/%u)]\n",
                 pending,
                 pending_rs,
                 THRESH);
      return; /* wait for replies */
    }
  req = GNUNET_CONTAINER_heap_peek(req_heap);
  if (NULL == req)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "Stopped processing queue: empty queue\n");
      return;
    }
  if (GNUNET_TIME_absolute_get_remaining(req->expires).rel_value_us > 0)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_INFO,
                 "Waiting until %s for next record (`%s') to expire\n",
                 GNUNET_STRINGS_absolute_time_to_string(req->expires),
                 req->hostname);
      if (NULL != t)
        GNUNET_SCHEDULER_cancel(t);
      sleep_time_reg_proc = GNUNET_TIME_absolute_get();
      t = GNUNET_SCHEDULER_add_at(req->expires, &process_queue, NULL);
      return;
    }
  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
  if (NULL != t)
    GNUNET_SCHEDULER_cancel(t);
  sleep_time_reg_proc = GNUNET_TIME_absolute_get();
  t = GNUNET_SCHEDULER_add_delayed(SERIES_DELAY, &process_queue, NULL);
}


static int
free_request_it(void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct Request *req = value;

  (void)cls;
  (void)key;
  free_request(req);
  return GNUNET_OK;
}


static void
do_shutdown(void *cls)
{
  struct Request *req;
  struct Zone *zone;

  (void)cls;
  if (NULL != id)
    {
      GNUNET_IDENTITY_disconnect(id);
      id = NULL;
    }
  if (NULL != t)
    {
      GNUNET_SCHEDULER_cancel(t);
      t = NULL;
    }
  while (NULL != (req = req_head))
    {
      GNUNET_CONTAINER_DLL_remove(req_head, req_tail, req);
      if (NULL != req->qe)
        GNUNET_NAMESTORE_cancel(req->qe);
      free_request(req);
    }
  while (NULL != (req = GNUNET_CONTAINER_heap_remove_root(req_heap)))
    {
      req->hn = NULL;
      if (NULL != req->qe)
        GNUNET_NAMESTORE_cancel(req->qe);
      free_request(req);
    }
  if (NULL != zone_it)
    {
      GNUNET_NAMESTORE_zone_iteration_stop(zone_it);
      zone_it = NULL;
    }
  if (NULL != ns)
    {
      GNUNET_NAMESTORE_disconnect(ns);
      ns = NULL;
    }
  if (NULL != ctx)
    {
      GNUNET_DNSSTUB_stop(ctx);
      ctx = NULL;
    }
  if (NULL != req_heap)
    {
      GNUNET_CONTAINER_heap_destroy(req_heap);
      req_heap = NULL;
    }
  if (NULL != ns_pending)
    {
      GNUNET_CONTAINER_multihashmap_iterate(ns_pending, &free_request_it, NULL);
      GNUNET_CONTAINER_multihashmap_destroy(ns_pending);
      ns_pending = NULL;
    }
  while (NULL != (zone = zone_head))
    {
      GNUNET_CONTAINER_DLL_remove(zone_head, zone_tail, zone);
      GNUNET_free(zone->domain);
      GNUNET_free(zone);
    }
  if (NULL != stats)
    {
      GNUNET_STATISTICS_destroy(stats, GNUNET_NO);
      stats = NULL;
    }
}


static void
iterate_zones(void *cls);


static void
ns_lookup_error_cb(void *cls)
{
  struct Zone *zone = cls;

  GNUNET_log(GNUNET_ERROR_TYPE_INFO,
             "Failed to load data from namestore for zone `%s'\n",
             zone->domain);
  zone_it = NULL;
  ns_iterator_trigger_next = 0;
  iterate_zones(NULL);
}


static void
ns_lookup_result_cb(void *cls,
                    const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
                    const char *label,
                    unsigned int rd_count,
                    const struct GNUNET_GNSRECORD_Data *rd)
{
  struct Zone *zone = cls;
  struct Request *req;
  struct GNUNET_HashCode hc;
  char *fqdn;

  ns_iterator_trigger_next--;
  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
             "Obtained NAMESTORE reply, %llu left in round\n",
             (unsigned long long)ns_iterator_trigger_next);
  if (0 == ns_iterator_trigger_next)
    {
      ns_iterator_trigger_next = NS_BATCH_SIZE;
      GNUNET_STATISTICS_update(stats,
                               "# NAMESTORE records requested from cache",
                               ns_iterator_trigger_next,
                               GNUNET_NO);
      GNUNET_NAMESTORE_zone_iterator_next(zone_it, ns_iterator_trigger_next);
    }
  GNUNET_asprintf(&fqdn, "%s.%s", label, zone->domain);
  GNUNET_CRYPTO_hash(fqdn, strlen(fqdn) + 1, &hc);
  GNUNET_free(fqdn);
  req = GNUNET_CONTAINER_multihashmap_get(ns_pending, &hc);
  if (NULL == req)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_INFO,
                 "Ignoring record `%s' in zone `%s': not on my list!\n",
                 label,
                 zone->domain);
      return;
    }
  GNUNET_assert(GNUNET_OK ==
                GNUNET_CONTAINER_multihashmap_remove(ns_pending, &hc, req));
  GNUNET_break(0 == GNUNET_memcmp(key, &req->zone->key));
  GNUNET_break(0 == strcasecmp(label, get_label(req)));
  for (unsigned int i = 0; i < rd_count; i++)
    {
      struct GNUNET_TIME_Absolute at;

      if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
        {
          struct GNUNET_TIME_Relative rel;

          rel.rel_value_us = rd->expiration_time;
          at = GNUNET_TIME_relative_to_absolute(rel);
        }
      else
        {
          at.abs_value_us = rd->expiration_time;
        }
      add_record(req, rd->record_type, at, rd->data, rd->data_size);
    }
  if (0 == rd_count)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_INFO,
                 "Empty record set in namestore for `%s'\n",
                 req->hostname);
    }
  else
    {
      unsigned int pos = 0;

      cached++;
      req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
      for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
        {
          struct GNUNET_TIME_Absolute at;

          at.abs_value_us = rec->grd.expiration_time;
          req->expires = GNUNET_TIME_absolute_min(req->expires, at);
          pos++;
        }
      if (0 == pos)
        req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
      GNUNET_log(GNUNET_ERROR_TYPE_INFO,
                 "Hot-start with %u existing records for `%s'\n",
                 pos,
                 req->hostname);
    }
  free_records(req);

  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
             "Adding `%s' to worklist to start at %s\n",
             req->hostname,
             GNUNET_STRINGS_absolute_time_to_string(req->expires));
  insert_sorted(req);
}


static void
queue(const char *hostname)
{
  struct Request *req;
  const char *dot;
  struct Zone *zone;
  size_t hlen;
  struct GNUNET_HashCode hc;

  if (GNUNET_OK != GNUNET_DNSPARSER_check_name(hostname))
    {
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Refusing invalid hostname `%s'\n",
                 hostname);
      rejects++;
      return;
    }
  dot = strchr(hostname, (unsigned char)'.');
  if (NULL == dot)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Refusing invalid hostname `%s' (lacks '.')\n",
                 hostname);
      rejects++;
      return;
    }
  for (zone = zone_head; NULL != zone; zone = zone->next)
    if (0 == strcmp(zone->domain, dot + 1))
      break;
  if (NULL == zone)
    {
      rejects++;
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 "Domain name `%s' not in ego list!\n",
                 dot + 1);
      return;
    }

  hlen = strlen(hostname) + 1;
  req = GNUNET_malloc(sizeof(struct Request) + hlen);
  req->zone = zone;
  req->hostname = (char *)&req[1];
  GNUNET_memcpy(req->hostname, hostname, hlen);
  req->id = (uint16_t)GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_NONCE,
                                               UINT16_MAX);
  GNUNET_CRYPTO_hash(req->hostname, hlen, &hc);
  if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put(
        ns_pending,
        &hc,
        req,
        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
    {
      GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
                 "Duplicate hostname `%s' ignored\n",
                 hostname);
      GNUNET_free(req);
      return;
    }
}


static int
move_to_queue(void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct Request *req = value;

  (void)cls;
  (void)key;
  insert_sorted(req);
  return GNUNET_OK;
}


static void
iterate_zones(void *cls)
{
  static struct Zone *last;

  (void)cls;
  if (NULL != zone_it)
    {
      zone_it = NULL;
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "Finished iteration over zone `%s'!\n",
                 last->domain);
      /* subtract left-overs from previous iteration */
      GNUNET_STATISTICS_update(stats,
                               "# NAMESTORE records requested from cache",
                               (long long)(-ns_iterator_trigger_next),
                               GNUNET_NO);
      ns_iterator_trigger_next = 0;
    }
  GNUNET_assert(NULL != zone_tail);
  if (zone_tail == last)
    {
      /* Done iterating over relevant zones in NAMESTORE, move
         rest of hash map to work queue as well. */
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "Finished all NAMESTORE iterations!\n");
      GNUNET_STATISTICS_set(stats,
                            "# Domain names without cached reply",
                            GNUNET_CONTAINER_multihashmap_size(ns_pending),
                            GNUNET_NO);
      GNUNET_CONTAINER_multihashmap_iterate(ns_pending, &move_to_queue, NULL);
      GNUNET_CONTAINER_multihashmap_destroy(ns_pending);
      ns_pending = NULL;
      start_time_reg_proc = GNUNET_TIME_absolute_get();
      total_reg_proc_dns = 0;
      total_reg_proc_dns_ns = 0;
      return;
    }
  if (NULL == last)
    last = zone_head;
  else
    last = last->next;
  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
             "Starting iteration over zone `%s'!\n",
             last->domain);
  /* subtract left-overs from previous iteration */
  GNUNET_STATISTICS_update(stats,
                           "# NAMESTORE records requested from cache",
                           1,
                           GNUNET_NO);
  ns_iterator_trigger_next = 1;
  GNUNET_STATISTICS_update(stats, "# zones iterated", 1, GNUNET_NO);
  zone_it = GNUNET_NAMESTORE_zone_iteration_start(ns,
                                                  &last->key,
                                                  &ns_lookup_error_cb,
                                                  NULL,
                                                  &ns_lookup_result_cb,
                                                  last,
                                                  &iterate_zones,
                                                  NULL);
}


static void
process_stdin(void *cls)
{
  static struct GNUNET_TIME_Absolute last;
  static uint64_t idot;
  char hn[256];

  (void)cls;
  t = NULL;
  if (NULL != id)
    {
      GNUNET_IDENTITY_disconnect(id);
      id = NULL;
    }
  while (NULL != fgets(hn, sizeof(hn), stdin))
    {
      if (strlen(hn) > 0)
        hn[strlen(hn) - 1] = '\0'; /* eat newline */
      if (0 == idot)
        last = GNUNET_TIME_absolute_get();
      idot++;
      if (0 == idot % 100000)
        {
          struct GNUNET_TIME_Relative delta;

          delta = GNUNET_TIME_absolute_get_duration(last);
          last = GNUNET_TIME_absolute_get();
          fprintf(stderr,
                  "Read 100000 domain names in %s\n",
                  GNUNET_STRINGS_relative_time_to_string(delta, GNUNET_YES));
          GNUNET_STATISTICS_set(stats, "# domain names provided", idot, GNUNET_NO);
        }
      queue(hn);
    }
  fprintf(stderr,
          "Done reading %llu domain names\n",
          (unsigned long long)idot);
  GNUNET_STATISTICS_set(stats, "# domain names provided", idot, GNUNET_NO);
  iterate_zones(NULL);
}


static void
identity_cb(void *cls,
            struct GNUNET_IDENTITY_Ego *ego,
            void **ctx,
            const char *name)
{
  (void)cls;
  (void)ctx;

  if (NULL == ego)
    {
      /* end of iteration */
      if (NULL == zone_head)
        {
          GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
          GNUNET_SCHEDULER_shutdown();
          return;
        }
      /* zone_head non-null, process hostnames from stdin */
      t = GNUNET_SCHEDULER_add_now(&process_stdin, NULL);
      return;
    }
  if (NULL != name)
    {
      struct Zone *zone;

      zone = GNUNET_new(struct Zone);
      zone->key = *GNUNET_IDENTITY_ego_get_private_key(ego);
      zone->domain = GNUNET_strdup(name);
      GNUNET_CONTAINER_DLL_insert(zone_head, zone_tail, zone);
    }
}


static void
run(void *cls,
    char *const *args,
    const char *cfgfile,
    const struct GNUNET_CONFIGURATION_Handle *cfg)
{
  (void)cls;
  (void)args;
  (void)cfgfile;
  stats = GNUNET_STATISTICS_create("zoneimport", cfg);
  req_heap = GNUNET_CONTAINER_heap_create(GNUNET_CONTAINER_HEAP_ORDER_MIN);
  ns_pending = GNUNET_CONTAINER_multihashmap_create(map_size, GNUNET_NO);
  if (NULL == ns_pending)
    {
      fprintf(stderr, "Failed to allocate memory for main hash map\n");
      return;
    }
  ctx = GNUNET_DNSSTUB_start(256);
  if (NULL == ctx)
    {
      fprintf(stderr, "Failed to initialize GNUnet DNS STUB\n");
      return;
    }
  if (NULL == args[0])
    {
      fprintf(stderr,
              "You must provide a list of DNS resolvers on the command line\n");
      return;
    }
  for (unsigned int i = 0; NULL != args[i]; i++)
    {
      if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip(ctx, args[i]))
        {
          fprintf(stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
          return;
        }
    }


  GNUNET_SCHEDULER_add_shutdown(&do_shutdown, NULL);
  ns = GNUNET_NAMESTORE_connect(cfg);
  if (NULL == ns)
    {
      GNUNET_SCHEDULER_shutdown();
      return;
    }
  id = GNUNET_IDENTITY_connect(cfg, &identity_cb, NULL);
}


int
main(int argc, char *const *argv)
{
  struct GNUNET_GETOPT_CommandLineOption options[] =
  { GNUNET_GETOPT_option_uint('s',
                              "size",
                              "MAPSIZE",
                              gettext_noop(
                                "size to use for the main hash map"),
                              &map_size),
    GNUNET_GETOPT_option_relative_time(
      'm',
      "minimum-expiration",
      "RELATIVETIME",
      gettext_noop("minimum expiration time we assume for imported records"),
      &minimum_expiration_time),
    GNUNET_GETOPT_OPTION_END };
  int ret;

  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args(argc, argv, &argc, &argv))
    return 2;
  if (GNUNET_OK != (ret = GNUNET_PROGRAM_run(argc,
                                             argv,
                                             "gnunet-zoneimport",
                                             "import DNS zone into namestore",
                                             options,
                                             &run,
                                             NULL)))
    return ret;
  GNUNET_free((void *)argv);
  fprintf(stderr,
          "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
          "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
          rejects,
          cached,
          lookups,
          record_sets,
          records,
          failures,
          pending,
          pending_rs);
  return 0;
}

/* end of gnunet-zoneimport.c */