gnunet-zoneimport.c File Reference

#include "platform.h"
#include <gnunet_util_lib.h>
#include <gnunet_gnsrecord_lib.h>
#include <gnunet_namestore_service.h>
#include <gnunet_statistics_service.h>
#include <gnunet_identity_service.h>

Include dependency graph for gnunet-zoneimport.c:

Go to the source code of this file.

Data Structures
struct	Zone
	Egos / Zones. More...
struct	Record
	Record for the request to be stored by GNS. More...
struct	Request
	Request we should make. More...
struct	MissingZoneCreationCtx
	Missing identity creation context. More...
struct	GlueClosure
	Closure for check_for_glue. More...
struct	ProcessRecordContext
	Closure for process_record(). More...

Macros
#define	THRESH   100
	Maximum number of queries pending at the same time.
#define	TIME_THRESH   10
	TIME_THRESH is in usecs.
#define	MAX_RETRIES   5
	How often do we retry a query before giving up for good?
#define	MAX_SERIES   10
	How many DNS requests do we at most issue in rapid series?
#define	SERIES_DELAY    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
	How long do we wait at least between series of requests?
#define	NS_BATCH_SIZE   1024
	How many requests do we request from NAMESTORE in one batch during our initial iteration?

Typedefs
typedef void(*	RecordProcessor) (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
	Callback for for_all_records.

Functions
static void	for_all_records (const struct GNUNET_DNSPARSER_Packet *p, RecordProcessor rp, void *rp_cls)
	Call rp for each record in p, regardless of what response section it is in.
static const char *	get_label (struct Request *req)
	Return just the label of the hostname in req.
static char *	get_label_utf8 (struct Request *req)
static void *	build_dns_query (struct Request *req, size_t *raw_size)
	Build DNS query for hostname.
static void	free_records (struct Request *req)
	Free records associated with req.
static void	free_request (struct Request *req)
	Free req and data structures reachable from it.
static void	process_queue (void *cls)
	Process as many requests as possible from the queue.
static void	insert_sorted (struct Request *req)
	Insert req into DLL sorted by next fetch time.
static void	add_record (struct Request *req, uint32_t type, struct GNUNET_TIME_Absolute expiration_time, const void *data, size_t data_len)
	Add record to the GNS record set for req.
static void	check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
	Try to find glue records for a given NS record.
static void	process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
	We received rec for req.
static void	store_completed_cb (void *cls, enum GNUNET_ErrorCode ec)
static void	process_result (void *cls, const struct GNUNET_TUN_DnsHeader *dns, size_t dns_len)
	Function called with the result of a DNS resolution.
static int	free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value)
	Iterator called during do_shutdown() to free requests in the ns_pending map.
static void	do_shutdown (void *cls)
	Clean up and terminate the process.
static void	iterate_zones (void *cls)
	Iterate over all of the zones we care about and see which records we may need to re-fetch when.
static void	ns_lookup_error_cb (void *cls)
	Function called if GNUNET_NAMESTORE_records_lookup() failed.
static void	ns_lookup_result_cb (void *cls, const struct GNUNET_CRYPTO_PrivateKey *key, const char *label_utf8, unsigned int rd_count, const struct GNUNET_GNSRECORD_Data *rd)
	Process a record that was stored in the namestore.
static void	delegation_store_cont (void *cls, enum GNUNET_ErrorCode ec)
static void	missing_zone_creation_cont (void *cls, const struct GNUNET_CRYPTO_PrivateKey *sk, enum GNUNET_ErrorCode ec)
static void	queue (const char *hostname)
	Add hostname to the list of requests to be made.
static int	move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value)
	We have completed the initial iteration over the namestore's database.
static void	process_stdin (void *cls)
	Begin processing hostnames from stdin.
static void	identity_cb (void *cls, struct GNUNET_IDENTITY_Ego *ego, void **ctx, const char *name)
	Method called to inform about the egos of this peer.
static void	run (void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
	Process requests from the queue, then if the queue is not empty, try again.
int	main (int argc, char *const *argv)
	Call with IP address of resolver to query.

Variables
static struct GNUNET_TIME_Relative	minimum_expiration_time
	How long do DNS records have to last at least after being imported?
static struct MissingZoneCreationCtx *	missing_zones_head
static struct MissingZoneCreationCtx *	missing_zones_tail
static uint64_t	stdin_lines_read
static unsigned int	map_size = 1024
	Command-line argument specifying desired size of the hash map with all of our pending names.
static struct GNUNET_IDENTITY_Handle *	id
	Handle to the identity service.
static struct GNUNET_NAMESTORE_Handle *	ns
	Namestore handle.
static struct GNUNET_STATISTICS_Handle *	stats
	Handle to the statistics service.
static struct GNUNET_DNSSTUB_Context *	dns_ctx
	Context for DNS resolution.
static unsigned int	pending
	The number of DNS queries that are outstanding.
static unsigned int	pending_rs
	The number of NAMESTORE record store operations that are outstanding.
static unsigned int	lookups
	Number of lookups we performed overall.
static unsigned int	cached
	Number of records we had cached.
static unsigned int	rejects
	How many hostnames did we reject (malformed).
static unsigned int	failures
	Number of lookups that failed.
static unsigned int	records
	Number of records we found.
static unsigned int	record_sets
	Number of record sets given to namestore.
static struct GNUNET_CONTAINER_Heap *	req_heap
	Heap of all requests to perform, sorted by the time we should next do the request (i.e.
static struct Request *	req_head
	Active requests are kept in a DLL.
static struct Request *	req_tail
	Active requests are kept in a DLL.
static struct GNUNET_SCHEDULER_Task *	t
	Main task.
static struct GNUNET_CONTAINER_MultiHashMap *	ns_pending
	Hash map of requests for which we may still get a response from the namestore.
static struct GNUNET_NAMESTORE_ZoneIterator *	zone_it
	Current zone iteration handle.
static struct Zone *	zone_head
	Head of list of zones we are managing.
static struct Zone *	zone_tail
	Tail of list of zones we are managing.
static uint64_t	ns_iterator_trigger_next
	After how many more results must ns_lookup_result_cb() ask the namestore for more?
static uint64_t	total_dns_latency_cnt
	Number of DNS requests counted in latency total.
static struct GNUNET_TIME_Relative	total_dns_latency
	Sum of DNS latencies observed.
static uint64_t	total_reg_proc_dns
	Number of records processed (DNS lookup, no NAMESTORE) in total.
static uint64_t	total_reg_proc_dns_ns
	Number of records processed (DNS lookup, with NAMESTORE) in total.
static struct GNUNET_TIME_Absolute	start_time_reg_proc
	Start time of the regular processing.
static struct GNUNET_TIME_Absolute	sleep_time_reg_proc
	Last time we worked before going idle.
static struct GNUNET_TIME_Relative	idle_time
	Time we slept just waiting for work.

Macro Definition Documentation

THRESH

#define THRESH   100

Maximum number of queries pending at the same time.

Definition at line 49 of file gnunet-zoneimport.c.

TIME_THRESH

#define TIME_THRESH   10

TIME_THRESH is in usecs.

How quickly do we submit fresh queries. Used as an additional throttle.

Definition at line 55 of file gnunet-zoneimport.c.

MAX_RETRIES

#define MAX_RETRIES   5

How often do we retry a query before giving up for good?

Definition at line 60 of file gnunet-zoneimport.c.

MAX_SERIES

#define MAX_SERIES   10

How many DNS requests do we at most issue in rapid series?

Definition at line 65 of file gnunet-zoneimport.c.

SERIES_DELAY

#define SERIES_DELAY    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)

How long do we wait at least between series of requests?

Definition at line 70 of file gnunet-zoneimport.c.

{
  struct Zone *next;
 
  struct Zone *prev;
 
  char *domain;
 
  struct GNUNET_CRYPTO_PrivateKey key;
};
 
 
struct Record
{
  struct Record *next;
 
  struct Record *prev;
 
  struct GNUNET_GNSRECORD_Data grd;
};
 
 
struct Request
{
  struct GNUNET_CONTAINER_HeapNode *hn;
 
  struct Request *next;
 
  struct Request *prev;
 
  struct Record *rec_head;
 
  struct Record *rec_tail;
 
  struct GNUNET_DNSSTUB_RequestSocket *rs;
 
  char *hostname;
 
  struct GNUNET_NAMESTORE_QueueEntry *qe;
 
  const struct Zone *zone;
 
  struct GNUNET_TIME_Absolute expires;
 
  struct GNUNET_TIME_Absolute op_start_time;
 
  unsigned int issue_num;
 
  uint16_t id;
};
 
struct MissingZoneCreationCtx
{
  // DLL
  struct MissingZoneCreationCtx *next;
 
  // DLL
  struct MissingZoneCreationCtx *prev;
 
  // Operation
  struct GNUNET_IDENTITY_Operation *id_op;
 
  // NS Operation
  struct GNUNET_NAMESTORE_QueueEntry *ns_qe;
 
  // Request
  struct Request *req;
};
 
// Missing zones list
static struct MissingZoneCreationCtx *missing_zones_head;
 
// Missing zones list
static struct MissingZoneCreationCtx *missing_zones_tail;
 
// Lines provided on stdin
static uint64_t stdin_lines_read;
 
static unsigned int map_size = 1024;
 
static struct GNUNET_IDENTITY_Handle *id;
 
static struct GNUNET_NAMESTORE_Handle *ns;
 
static struct GNUNET_STATISTICS_Handle *stats;
 
static struct GNUNET_DNSSTUB_Context *dns_ctx;
 
static unsigned int pending;
 
static unsigned int pending_rs;
 
static unsigned int lookups;
 
static unsigned int cached;
 
static unsigned int rejects;
 
static unsigned int failures;
 
static unsigned int records;
 
static unsigned int record_sets;
 
static struct GNUNET_CONTAINER_Heap *req_heap;
 
static struct Request *req_head;
 
static struct Request *req_tail;
 
static struct GNUNET_SCHEDULER_Task *t;
 
static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
 
static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
 
static struct Zone *zone_head;
 
static struct Zone *zone_tail;
 
static uint64_t ns_iterator_trigger_next;
 
static uint64_t total_dns_latency_cnt;
 
static struct GNUNET_TIME_Relative total_dns_latency;
 
static uint64_t total_reg_proc_dns;
 
static uint64_t total_reg_proc_dns_ns;
 
static struct GNUNET_TIME_Absolute start_time_reg_proc;
 
static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
 
static struct GNUNET_TIME_Relative idle_time;
 
 
typedef void (*RecordProcessor) (void *cls,
                                 const struct GNUNET_DNSPARSER_Record *rec);
 
 
static void
for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
                 RecordProcessor rp,
                 void *rp_cls)
{
  for (unsigned int i = 0; i < p->num_answers; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
 
    rp (rp_cls, rs);
  }
  for (unsigned int i = 0; i < p->num_authority_records; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
 
    rp (rp_cls, rs);
  }
  for (unsigned int i = 0; i < p->num_additional_records; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
 
    rp (rp_cls, rs);
  }
}
 
 
static const char *
get_label (struct Request *req)
{
  static char label[64];
  const char *dot;
 
  dot = strchr (req->hostname, (unsigned char) '.');
  if (NULL == dot)
  {
    GNUNET_break (0);
    return NULL;
  }
  if (((size_t) (dot - req->hostname)) >= sizeof(label))
  {
    GNUNET_break (0);
    return NULL;
  }
  GNUNET_memcpy (label, req->hostname, dot - req->hostname);
  label[dot - req->hostname] = '\0';
 
  return label;
}
 
 
static char*
get_label_utf8 (struct Request *req)
{
  char *label_utf8;
  const char *label_ace;
 
  label_ace = get_label (req);
  idna_to_unicode_8z8z (label_ace,
                        &label_utf8,
                        IDNA_ALLOW_UNASSIGNED);
  return label_utf8;
}
 
 
static void *
build_dns_query (struct Request *req, size_t *raw_size)
{
  static char raw[512];
  char *name_ace;
  char *rawp;
  struct GNUNET_DNSPARSER_Packet p;
  struct GNUNET_DNSPARSER_Query q;
  int ret;
 
  if (IDNA_SUCCESS != idna_to_ascii_8z (req->hostname,
                                        &name_ace,
                                        IDNA_ALLOW_UNASSIGNED))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Name `%s' cannot be converted to IDNA."),
                req->hostname);
    return NULL;
  }
  q.name = (char *) name_ace;
  q.type = GNUNET_DNSPARSER_TYPE_NS;
  q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
 
  memset (&p, 0, sizeof(p));
  p.num_queries = 1;
  p.queries = &q;
  p.id = req->id;
  ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size);
  if (GNUNET_OK != ret)
  {
    if (GNUNET_NO == ret)
      GNUNET_free (rawp);
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to pack query for hostname `%s'\n",
                name_ace);
    rejects++;
    GNUNET_free (name_ace);
    return NULL;
  }
  if (*raw_size > sizeof(raw))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to pack query for hostname `%s'\n",
                name_ace);
    rejects++;
    GNUNET_break (0);
    GNUNET_free (name_ace);
    GNUNET_free (rawp);
    return NULL;
  }
  GNUNET_memcpy (raw, rawp, *raw_size);
  GNUNET_free (rawp);
  return raw;
}
 
 
static void
free_records (struct Request *req)
{
  struct Record *rec;
 
  /* Free records */
  while (NULL != (rec = req->rec_head))
  {
    GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec);
    GNUNET_free (rec);
  }
}
 
 
static void
free_request (struct Request *req)
{
  free_records (req);
  GNUNET_free (req);
}
 
 
static void
process_queue (void *cls);
 
 
static void
insert_sorted (struct Request *req)
{
  req->hn =
    GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us);
  if (req == GNUNET_CONTAINER_heap_peek (req_heap))
  {
    if (NULL != t)
      GNUNET_SCHEDULER_cancel (t);
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
  }
}
 
 
static void
add_record (struct Request *req,
            uint32_t type,
            struct GNUNET_TIME_Absolute expiration_time,
            const void *data,
            size_t data_len)
{
  struct Record *rec;
 
  rec = GNUNET_malloc (sizeof(struct Record) + data_len);
  rec->grd.data = &rec[1];
  rec->grd.expiration_time = expiration_time.abs_value_us;
  rec->grd.data_size = data_len;
  rec->grd.record_type = type;
  rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
  GNUNET_memcpy (&rec[1], data, data_len);
  GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec);
}
 
 
struct GlueClosure
{
  struct Request *req;
 
  const char *ns;
 
  int found;
};
 
 
static void
check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
{
  struct GlueClosure *gc = cls;
  char dst[65536];
  size_t rd_len;
  char ip[INET6_ADDRSTRLEN + 1];
  socklen_t ip_size = (socklen_t) sizeof(ip);
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;
 
  if (0 != strcasecmp (rec->name, gc->ns))
    return;
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining (expiration_time);
  if (0 == left.rel_value_us)
    return; /* ignore expired glue records */
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
  rd_len = 0;
  switch (rec->type)
  {
  case GNUNET_DNSPARSER_TYPE_A:
    if (sizeof(struct in_addr) != rec->data.raw.data_len)
    {
      GNUNET_break (0);
      return;
    }
    if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size))
    {
      GNUNET_break (0);
      return;
    }
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   ip,
                   strlen (ip) + 1);
    rd_len += strlen (ip) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;
 
  case GNUNET_DNSPARSER_TYPE_AAAA:
    if (sizeof(struct in6_addr) != rec->data.raw.data_len)
    {
      GNUNET_break (0);
      return;
    }
    if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size))
    {
      GNUNET_break (0);
      return;
    }
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   ip,
                   strlen (ip) + 1);
    rd_len += strlen (ip) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;
 
  case GNUNET_DNSPARSER_TYPE_CNAME:
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   rec->data.hostname,
                   strlen (rec->data.hostname) + 1);
    rd_len += strlen (rec->data.hostname) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;
 
  default:
    /* useless, do nothing */
    break;
  }
}
 
 
struct ProcessRecordContext
{
  struct GNUNET_DNSPARSER_Packet *p;
 
  struct Request *req;
};
 
 
static void
process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
{
  struct ProcessRecordContext *prc = cls;
  struct Request *req = prc->req;
  char dst[65536];
  size_t dst_len;
  size_t off;
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;
 
  dst_len = sizeof(dst);
  off = 0;
  records++;
  if (0 != strcasecmp (rec->name, req->hostname))
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_DEBUG,
      "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
      rec->name,
      (unsigned int) rec->type,
      req->hostname);
    return;   /* does not match hostname, might be glue, but
                 not useful for this pass! */
  }
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining (expiration_time);
  if (0 == left.rel_value_us)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "DNS returned expired record for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats,
                              "# expired records obtained from DNS",
                              1,
                              GNUNET_NO);
    return;   /* record expired */
  }
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "DNS returned record that expires at %s for `%s'\n",
              GNUNET_STRINGS_absolute_time_to_string (expiration_time),
              req->hostname);
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
  switch (rec->type)
  {
  case GNUNET_DNSPARSER_TYPE_NS: {
      struct GlueClosure gc;
 
      /* check for glue */
      gc.req = req;
      gc.ns = rec->data.hostname;
      gc.found = GNUNET_NO;
      for_all_records (prc->p, &check_for_glue, &gc);
      if (GNUNET_NO == gc.found)
      {
        // NAME
        GNUNET_memcpy (dst,
                       req->hostname,
                       strlen (req->hostname) + 1);
        off += strlen (req->hostname) + 1;
        // DNS SERVER NAME
        GNUNET_memcpy (dst + strlen (req->hostname) + 1,
                       rec->data.hostname,
                       strlen (rec->data.hostname) + 1);
        off += strlen (rec->data.hostname) + 1;
        /* FIXME: actually check if this is out-of-bailiwick,
             and if not request explicit resolution... */
        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                    "Converted OOB (`%s') NS record for `%s'\n",
                    rec->data.hostname,
                    rec->name);
        add_record (req,
                    GNUNET_GNSRECORD_TYPE_GNS2DNS,
                    expiration_time,
                    dst,
                    off);
      }
      else
      {
        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                    "Converted NS record for `%s' using glue\n",
                    rec->name);
      }
      break;
    }
 
  case GNUNET_DNSPARSER_TYPE_CNAME:
    if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
                                                        dst_len,
                                                        &off,
                                                        rec->data.hostname))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting CNAME (`%s') record for `%s'\n",
                  rec->data.hostname,
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_DNAME:
    /* No support for DNAME in GNS yet! FIXME: support later! */
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "FIXME: not supported: %s DNAME %s\n",
                rec->name,
                rec->data.hostname);
    break;
 
  case GNUNET_DNSPARSER_TYPE_MX:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting MX (`%s') record for `%s'\n",
                  rec->data.mx->mxhost,
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_SOA:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa))
    {
      /* NOTE: GNS does not really use SOAs */
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting SOA record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_SRV:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting SRV record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_URI:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_uri (dst, dst_len, &off, rec->data.uri))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting URI record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_PTR:
    if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
                                                        dst_len,
                                                        &off,
                                                        rec->data.hostname))
    {
      /* !?: what does a PTR record do in a regular TLD??? */
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting PTR record for `%s' (weird)\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_CERT:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting CERT record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  /* Rest is 'raw' encoded and just needs to be copied IF
     the hostname matches the requested name; otherwise we
     simply cannot use it. */
  case GNUNET_DNSPARSER_TYPE_A:
  case GNUNET_DNSPARSER_TYPE_AAAA:
  case GNUNET_DNSPARSER_TYPE_TXT:
  default:
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Converting record of type %u for `%s'\n",
                (unsigned int) rec->type,
                rec->name);
    add_record (req,
                rec->type,
                expiration_time,
                rec->data.raw.data,
                rec->data.raw.data_len);
    break;
  }
}
 
 
static void
store_completed_cb (void *cls, enum GNUNET_ErrorCode ec)
{
  static struct GNUNET_TIME_Absolute last;
  struct Request *req = cls;
 
  req->qe = NULL;
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to store zone data for `%s': %s\n",
                req->hostname,
                GNUNET_ErrorCode_get_hint (ec));
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stored records under `%s' (%d)\n",
                req->hostname,
                ec);
  }
  total_reg_proc_dns_ns++; /* finished regular processing */
  pending_rs--;
  free_records (req);
  /* compute NAMESTORE statistics */
  {
    static uint64_t total_ns_latency_cnt;
    static struct GNUNET_TIME_Relative total_ns_latency;
    struct GNUNET_TIME_Relative ns_latency;
 
    ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
    total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency);
    if (0 == total_ns_latency_cnt)
      last = GNUNET_TIME_absolute_get ();
    total_ns_latency_cnt++;
    if (0 == (total_ns_latency_cnt % 1000))
    {
      struct GNUNET_TIME_Relative delta;
      struct GNUNET_TIME_Relative est_remaining;
 
      delta = GNUNET_TIME_absolute_get_duration (last);
      last = GNUNET_TIME_absolute_get ();
      est_remaining = GNUNET_TIME_relative_multiply (
        delta,
        (stdin_lines_read - total_ns_latency_cnt) / 1000);
      fprintf (stderr,
               "Processed 1000 records in %s. ",
               GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
      fprintf (stderr,
               "Estimated remaining time: %s. ",
               GNUNET_STRINGS_relative_time_to_string (
                 est_remaining,
                 GNUNET_YES));
      fprintf (stderr,
               "Last domain processed: %s\n",
               req->hostname);
      GNUNET_STATISTICS_set (stats,
                             "# average NAMESTORE PUT latency (μs)",
                             total_ns_latency.rel_value_us
                             / total_ns_latency_cnt,
                             GNUNET_NO);
    }
  }
  /* compute and publish overall velocity */
  if (0 == (total_reg_proc_dns_ns % 100))
  {
    struct GNUNET_TIME_Relative runtime;
 
    runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
    runtime = GNUNET_TIME_relative_subtract (runtime, idle_time);
    runtime =
      GNUNET_TIME_relative_divide (runtime,
                                   total_reg_proc_dns + total_reg_proc_dns_ns);
    GNUNET_STATISTICS_set (stats,
                           "# Regular processing completed without NAMESTORE",
                           total_reg_proc_dns,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# Regular processing completed with NAMESTORE PUT",
                           total_reg_proc_dns_ns,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# average request processing latency (μs)",
                           runtime.rel_value_us,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# total time spent idle (μs)",
                           idle_time.rel_value_us,
                           GNUNET_NO);
  }
 
  if (NULL == t)
  {
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
  }
}
 
 
static void
process_result (void *cls,
                const struct GNUNET_TUN_DnsHeader *dns,
                size_t dns_len)
{
  struct Request *req = cls;
  struct Record *rec;
  struct GNUNET_DNSPARSER_Packet *p;
  unsigned int rd_count;
 
  GNUNET_assert (NULL == req->hn);
  if ((NULL == dns) ||
      (dns->flags.return_code != GNUNET_TUN_DNS_RETURN_CODE_NO_ERROR))
  {
    /* stub gave up */
    GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
    if (NULL != dns)
    {
      GNUNET_DNSSTUB_resolve_cancel (req->rs);
      req->rs = NULL;
    }
    pending--;
    if (NULL == t)
    {
      sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
      t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
    }
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stub gave up on DNS reply for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats, "# Unsuccessful DNS lookups", 1, GNUNET_NO)
    ;
    if (req->issue_num > MAX_RETRIES)
    {
      failures++;
      free_request (req);
      GNUNET_STATISTICS_update (stats, "# DNS requests given up on", 1,
                                GNUNET_NO);
      return;
    }
    total_reg_proc_dns++;
    req->rs = NULL;
    insert_sorted (req);
    return;
  }
  if (req->id != dns->id)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "DNS ID did not match request, ignoring reply\n");
    GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO);
    return;
  }
  GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
  GNUNET_DNSSTUB_resolve_cancel (req->rs);
  req->rs = NULL;
  pending--;
  p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len);
  if (NULL == p)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to parse DNS reply for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO);
    if (NULL == t)
    {
      sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
      t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
    }
    if (req->issue_num > MAX_RETRIES)
    {
      failures++;
      free_request (req);
      GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
      return;
    }
    insert_sorted (req);
    return;
  }
  /* import new records */
  req->issue_num = 0; /* success, reset counter! */
  {
    struct ProcessRecordContext prc = { .req = req, .p = p };
 
    for_all_records (p, &process_record, &prc);
  }
  GNUNET_DNSPARSER_free_packet (p);
  /* count records found, determine minimum expiration time */
  req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
  {
    struct GNUNET_TIME_Relative dns_latency;
 
    dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
    total_dns_latency =
      GNUNET_TIME_relative_add (total_dns_latency, dns_latency);
    total_dns_latency_cnt++;
    if (0 == (total_dns_latency_cnt % 1000))
    {
      GNUNET_STATISTICS_set (stats,
                             "# average DNS lookup latency (μs)",
                             total_dns_latency.rel_value_us
                             / total_dns_latency_cnt,
                             GNUNET_NO);
    }
  }
  rd_count = 0;
  for (rec = req->rec_head; NULL != rec; rec = rec->next)
  {
    struct GNUNET_TIME_Absolute at;
 
    at.abs_value_us = rec->grd.expiration_time;
    req->expires = GNUNET_TIME_absolute_min (req->expires, at);
    rd_count++;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Obtained %u records for `%s'\n",
              rd_count,
              req->hostname);
  /* Instead of going for SOA, simplified for now to look each
     day in case we got an empty response */
  if (0 == rd_count)
  {
    req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
    GNUNET_STATISTICS_update (stats,
                              "# empty DNS replies (usually NXDOMAIN)",
                              1,
                              GNUNET_NO);
  }
  else
  {
    record_sets++;
  }
  /* convert records to namestore import format */
  {
    struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)];
    unsigned int off = 0;
    char *label_utf8 = get_label_utf8 (req);
 
    /* convert linked list into array */
    for (rec = req->rec_head; NULL != rec; rec = rec->next)
      rd[off++] = rec->grd;
    pending_rs++;
    req->op_start_time = GNUNET_TIME_absolute_get ();
    req->qe = GNUNET_NAMESTORE_record_set_store (ns,
                                                 &req->zone->key,
                                                 label_utf8,
                                                 rd_count,
                                                 rd,
                                                 &store_completed_cb,
                                                 req);
    GNUNET_free (label_utf8);
    GNUNET_assert (NULL != req->qe);
  }
  insert_sorted (req);
  return;
}
 
 
static void
process_queue (void *cls)
{
  struct Request *req;
  unsigned int series;
  void *raw;
  size_t raw_size;
  struct GNUNET_TIME_Relative delay;
 
  (void) cls;
  delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
  idle_time = GNUNET_TIME_relative_add (idle_time, delay);
  series = 0;
  t = NULL;
  while (pending + pending_rs < THRESH)
  {
    req = GNUNET_CONTAINER_heap_peek (req_heap);
    if (NULL == req)
      break;
    if (NULL != req->qe)
      return;   /* namestore op still pending */
    if (NULL != req->rs)
    {
      GNUNET_break (0);
      return;     /* already submitted */
    }
    if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
      break;
    GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap));
    req->hn = NULL;
    GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req);
    GNUNET_assert (NULL == req->rs);
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Requesting resolution for `%s'\n",
                req->hostname);
    raw = build_dns_query (req, &raw_size);
    if (NULL == raw)
    {
      GNUNET_break (0);
      free_request (req);
      continue;
    }
    req->op_start_time = GNUNET_TIME_absolute_get ();
    req->rs = GNUNET_DNSSTUB_resolve (dns_ctx, raw, raw_size, &process_result,
                                      req);
    GNUNET_assert (NULL != req->rs);
    req->issue_num++;
    lookups++;
    pending++;
    series++;
    if (series > MAX_SERIES)
      break;
  }
  if (pending + pending_rs >= THRESH)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stopped processing queue (%u+%u/%u)]\n",
                pending,
                pending_rs,
                THRESH);
    return;   /* wait for replies */
  }
  req = GNUNET_CONTAINER_heap_peek (req_heap);
  if (NULL == req)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stopped processing queue: empty queue\n");
    return;
  }
  if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Waiting until %s for next record (`%s') to expire\n",
                GNUNET_STRINGS_absolute_time_to_string (req->expires),
                req->hostname);
    if (NULL != t)
      GNUNET_SCHEDULER_cancel (t);
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
  if (NULL != t)
    GNUNET_SCHEDULER_cancel (t);
  sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
  t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL);
}
 
 
static int
free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct Request *req = value;
 
  (void) cls;
  (void) key;
  free_request (req);
  return GNUNET_OK;
}
 
 
static void
do_shutdown (void *cls)
{
  struct Request *req;
  struct Zone *zone;
  struct MissingZoneCreationCtx *mzctx;
 
  (void) cls;
  while (NULL != (mzctx = missing_zones_head))
  {
    GNUNET_CONTAINER_DLL_remove (missing_zones_head,
                                 missing_zones_tail,
                                 mzctx);
    if (NULL != mzctx->id_op)
    {
      GNUNET_IDENTITY_cancel (mzctx->id_op);
      mzctx->id_op = NULL;
    }
    if (NULL != mzctx->ns_qe)
    {
      GNUNET_NAMESTORE_cancel (mzctx->ns_qe);
      mzctx->ns_qe = NULL;
    }
    GNUNET_free (mzctx);
  }
  if (NULL != id)
  {
    GNUNET_IDENTITY_disconnect (id);
    id = NULL;
  }
  if (NULL != t)
  {
    GNUNET_SCHEDULER_cancel (t);
    t = NULL;
  }
  while (NULL != (req = req_head))
  {
    GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
    if (NULL != req->qe)
      GNUNET_NAMESTORE_cancel (req->qe);
    free_request (req);
  }
  while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
  {
    req->hn = NULL;
    if (NULL != req->qe)
      GNUNET_NAMESTORE_cancel (req->qe);
    free_request (req);
  }
  if (NULL != zone_it)
  {
    GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
    zone_it = NULL;
  }
  if (NULL != ns)
  {
    GNUNET_NAMESTORE_disconnect (ns);
    ns = NULL;
  }
  if (NULL != dns_ctx)
  {
    GNUNET_DNSSTUB_stop (dns_ctx);
    dns_ctx = NULL;
  }
  if (NULL != req_heap)
  {
    GNUNET_CONTAINER_heap_destroy (req_heap);
    req_heap = NULL;
  }
  if (NULL != ns_pending)
  {
    GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL);
    GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
    ns_pending = NULL;
  }
  while (NULL != (zone = zone_head))
  {
    GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone);
    GNUNET_free (zone->domain);
    GNUNET_free (zone);
  }
  if (NULL != stats)
  {
    GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
    stats = NULL;
  }
}
 
 
static void
iterate_zones (void *cls);
 
 
static void
ns_lookup_error_cb (void *cls)
{
  struct Zone *zone = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Failed to load data from namestore for zone `%s'\n",
              zone->domain);
  zone_it = NULL;
  ns_iterator_trigger_next = 0;
  iterate_zones (NULL);
}
 
 
static void
ns_lookup_result_cb (void *cls,
                     const struct GNUNET_CRYPTO_PrivateKey *key,
                     const char *label_utf8,
                     unsigned int rd_count,
                     const struct GNUNET_GNSRECORD_Data *rd)
{
  struct Zone *zone = cls;
  struct Request *req;
  struct GNUNET_HashCode hc;
  char *label_ace;
  char *fqdn;
 
  ns_iterator_trigger_next--;
  if (0 == ns_iterator_trigger_next)
  {
    ns_iterator_trigger_next = NS_BATCH_SIZE;
    GNUNET_STATISTICS_update (stats,
                              "# NAMESTORE records requested from cache",
                              ns_iterator_trigger_next,
                              GNUNET_NO);
    GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next);
  }
  if (IDNA_SUCCESS != idna_to_ascii_8z (label_utf8,
                                        &label_ace,
                                        IDNA_ALLOW_UNASSIGNED))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Name `%s' cannot be converted to IDNA."),
                label_utf8);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Obtained NAMESTORE reply for %s (Punycode `%s', %llu left in round\n",
              label_utf8,
              label_ace,
              (unsigned long long) ns_iterator_trigger_next);
 
  GNUNET_asprintf (&fqdn, "%s.%s", label_ace, zone->domain);
  GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc);
  GNUNET_free (fqdn);
  req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc);
  if (NULL == req)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Ignoring record `%s' in zone `%s': not on my list!\n",
                label_ace,
                zone->domain);
    GNUNET_free (label_ace);
    return;
  }
  GNUNET_assert (GNUNET_OK ==
                 GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req));
  GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key));
  GNUNET_break (0 == strcasecmp (label_ace, get_label (req)));
  GNUNET_free (label_ace);
  for (unsigned int i = 0; i < rd_count; i++)
  {
    struct GNUNET_TIME_Absolute at;
 
    if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
    {
      struct GNUNET_TIME_Relative rel;
 
      rel.rel_value_us = rd->expiration_time;
      at = GNUNET_TIME_relative_to_absolute (rel);
    }
    else
    {
      at.abs_value_us = rd->expiration_time;
    }
    add_record (req, rd->record_type, at, rd->data, rd->data_size);
  }
  if (0 == rd_count)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Empty record set in namestore for `%s'\n",
                req->hostname);
  }
  else
  {
    unsigned int pos = 0;
 
    cached++;
    req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
    for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
    {
      struct GNUNET_TIME_Absolute at;
 
      at.abs_value_us = rec->grd.expiration_time;
      req->expires = GNUNET_TIME_absolute_min (req->expires, at);
      pos++;
    }
    if (0 == pos)
      req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Hot-start with %u existing records for `%s'\n",
                pos,
                req->hostname);
  }
  free_records (req);
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Adding `%s' to worklist to start at %s\n",
              req->hostname,
              GNUNET_STRINGS_absolute_time_to_string (req->expires));
  insert_sorted (req);
}
 
 
static void
delegation_store_cont (void *cls,
                       enum GNUNET_ErrorCode ec)
{
  struct MissingZoneCreationCtx *mzctx = cls;
 
  mzctx->ns_qe = NULL;
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to store delegation: `%s'\n",
                GNUNET_ErrorCode_get_hint (ec));
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Added delegation successfully\n");
  GNUNET_CONTAINER_DLL_remove (missing_zones_head,
                               missing_zones_tail,
                               mzctx);
  GNUNET_free (mzctx);
  if (NULL == missing_zones_head)
  {
    iterate_zones (NULL);
  }
 
}
 
 
static void
missing_zone_creation_cont (
  void *cls,
  const struct GNUNET_CRYPTO_PrivateKey *sk,
  enum GNUNET_ErrorCode ec)
{
  struct MissingZoneCreationCtx *mzctx = cls;
  struct GNUNET_CRYPTO_PublicKey pk;
  struct Zone *parent_zone;
  struct Zone *zone;
  const char *dot;
  const char *expected_parent_zone_name;
  char parent_delegation_label[GNUNET_DNSPARSER_MAX_NAME_LENGTH];
 
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to create zone for `%s': %s\n",
                mzctx->req->hostname,
                GNUNET_ErrorCode_get_hint (ec));
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  mzctx->id_op = NULL;
  dot = strchr (mzctx->req->hostname, (unsigned char) '.');
  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
              "Created missing ego `%s'\n",
              dot + 1);
  zone = GNUNET_new (struct Zone);
  zone->key = *sk;
  zone->domain = GNUNET_strdup (dot + 1);
  mzctx->req->zone = zone;
  GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
 
  expected_parent_zone_name = strchr (dot + 1,
                                      (unsigned char) '.');
  if (NULL == expected_parent_zone_name)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Created identity without parent zone!\n");
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  {
    int label_len = strlen (dot + 1) - strlen (expected_parent_zone_name) + 1;
    snprintf (parent_delegation_label,
              label_len,
              "%s",
              dot + 1);
    parent_delegation_label[label_len] = '\0';
  }
  for (parent_zone = zone_head;
       NULL != parent_zone;
       parent_zone = parent_zone->next)
  {
    if (0 != strcmp (expected_parent_zone_name + 1,
                     parent_zone->domain))
      continue;
    break;
  }
  GNUNET_assert (NULL != parent_zone);
  GNUNET_CRYPTO_key_get_public (sk, &pk);
  struct GNUNET_GNSRECORD_Data delegation_rd;
  char *data;
  size_t data_size;
  GNUNET_assert (GNUNET_OK ==
                 GNUNET_GNSRECORD_data_from_identity (&pk,
                                                      &data,
                                                      &data_size,
                                                      &delegation_rd.record_type
                                                      ));
  delegation_rd.flags = GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION;
  delegation_rd.expiration_time = GNUNET_TIME_UNIT_WEEKS.rel_value_us;
  delegation_rd.data = data;
  delegation_rd.data_size = data_size;
  mzctx->ns_qe = GNUNET_NAMESTORE_record_set_store (ns,
                                                    &parent_zone->key,
                                                    parent_delegation_label,
                                                    1,
                                                    &delegation_rd,
                                                    &delegation_store_cont,
                                                    mzctx);
  GNUNET_free (data);
}
 
 
static void
queue (const char *hostname)
{
  struct Request *req;
  const char *dot;
  struct Zone *zone;
  size_t hlen;
  struct GNUNET_HashCode hc;
 
  if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Refusing invalid hostname `%s'\n",
                hostname);
    rejects++;
    return;
  }
  dot = strchr (hostname, (unsigned char) '.');
  if (NULL == dot)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Refusing invalid hostname `%s' (lacks '.')\n",
                hostname);
    rejects++;
    return;
  }
  for (zone = zone_head; NULL != zone; zone = zone->next)
    if (0 == strcmp (zone->domain, dot + 1))
      break;
  hlen = strlen (hostname) + 1;
  req = GNUNET_malloc (sizeof(struct Request) + hlen);
  if (NULL == zone)
  {
    struct MissingZoneCreationCtx *mzctx;
 
    for (mzctx = missing_zones_head;
         NULL != mzctx;
         mzctx = mzctx->next)
    {
      const char *tmp_dot;
      tmp_dot = strchr (mzctx->req->hostname, (unsigned char) '.');
      GNUNET_assert (NULL != tmp_dot);
      if (0 != strcmp (dot + 1, tmp_dot + 1))
        continue;
      break;
    }
    if (NULL == mzctx)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  "Domain name `%s' not in ego list! Creating.\n",
                  dot + 1);
      mzctx = GNUNET_new (struct MissingZoneCreationCtx);
      mzctx->id_op = GNUNET_IDENTITY_create (id,
                                             dot + 1,
                                             NULL,
                                             GNUNET_PUBLIC_KEY_TYPE_EDDSA,
                                             &missing_zone_creation_cont,
                                             mzctx);
      mzctx->req = req;
      GNUNET_CONTAINER_DLL_insert (missing_zones_head,
                                   missing_zones_tail,
                                   mzctx);
    }
  }
  else
  {
    req->zone = zone;
  }
 
  req->hostname = (char *) &req[1];
  GNUNET_memcpy (req->hostname, hostname, hlen);
  req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
                                                 UINT16_MAX);
  GNUNET_CRYPTO_hash (req->hostname, hlen, &hc);
  if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
        ns_pending,
        &hc,
        req,
        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Duplicate hostname `%s' ignored\n",
                hostname);
    GNUNET_free (req);
    return;
  }
}
 
 
static int
move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct Request *req = value;
 
  (void) cls;
  (void) key;
  insert_sorted (req);
  return GNUNET_OK;
}
 
 
static void
iterate_zones (void *cls)
{
  static struct Zone *last;
 
  (void) cls;
  if (NULL != zone_it)
  {
    zone_it = NULL;
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Finished iteration over zone `%s'!\n",
                last->domain);
    /* subtract left-overs from previous iteration */
    GNUNET_STATISTICS_update (stats,
                              "# NAMESTORE records requested from cache",
                              (long long) (-ns_iterator_trigger_next),
                              GNUNET_NO);
    ns_iterator_trigger_next = 0;
  }
  GNUNET_assert (NULL != zone_tail);
  if (zone_tail == last)
  {
    /* Done iterating over relevant zones in NAMESTORE, move
       rest of hash map to work queue as well. */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Finished all NAMESTORE iterations!\n");
    GNUNET_STATISTICS_set (stats,
                           "# Domain names without cached reply",
                           GNUNET_CONTAINER_multihashmap_size (ns_pending),
                           GNUNET_NO);
    GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL);
    GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
    ns_pending = NULL;
    start_time_reg_proc = GNUNET_TIME_absolute_get ();
    total_reg_proc_dns = 0;
    total_reg_proc_dns_ns = 0;
    return;
  }
  if (NULL == last)
    last = zone_head;
  else
    last = last->next;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Starting iteration over zone `%s'!\n",
              last->domain);
  /* subtract left-overs from previous iteration */
  GNUNET_STATISTICS_update (stats,
                            "# NAMESTORE records requested from cache",
                            1,
                            GNUNET_NO);
  ns_iterator_trigger_next = 1;
  GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO);
  zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
                                                   &last->key,
                                                   &ns_lookup_error_cb,
                                                   NULL,
                                                   &ns_lookup_result_cb,
                                                   last,
                                                   &iterate_zones,
                                                   NULL);
}
 
 
static void
process_stdin (void *cls)
{
  static struct GNUNET_TIME_Absolute last;
  char hn[256];
  char *hn_utf8;
 
  (void) cls;
  t = NULL;
  while (NULL != fgets (hn, sizeof(hn), stdin))
  {
    if (strlen (hn) > 0)
      hn[strlen (hn) - 1] = '\0';  /* eat newline */
    if (0 == stdin_lines_read)
      last = GNUNET_TIME_absolute_get ();
    stdin_lines_read++;
    if (0 == stdin_lines_read % 100000)
    {
      struct GNUNET_TIME_Relative delta;
 
      delta = GNUNET_TIME_absolute_get_duration (last);
      last = GNUNET_TIME_absolute_get ();
      fprintf (stderr,
               "Read 100000 domain names in %s\n",
               GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
      GNUNET_STATISTICS_set (stats, "# domain names provided", stdin_lines_read,
                             GNUNET_NO);
    }
    idna_to_unicode_8z8z (hn,
                          &hn_utf8,
                          IDNA_ALLOW_UNASSIGNED);
    queue (hn_utf8);
    GNUNET_free (hn_utf8);
  }
  fprintf (stderr,
           "Done reading %llu domain names\n",
           (unsigned long long) stdin_lines_read);
  GNUNET_STATISTICS_set (stats, "# domain names provided", stdin_lines_read,
                         GNUNET_NO);
  // Only start iteration immediately if no zones are missing.
  if (NULL == missing_zones_head)
  {
    iterate_zones (NULL);
  }
}
 
 
static void
identity_cb (void *cls,
             struct GNUNET_IDENTITY_Ego *ego,
             void **ctx,
             const char *name)
{
  (void) cls;
  (void) ctx;
  static int initial_iteration = GNUNET_YES;
 
  if (GNUNET_NO == initial_iteration)
    return;
  if (NULL == ego)
  {
    /* end of iteration */
    if (NULL == zone_head)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
      GNUNET_SCHEDULER_shutdown ();
      return;
    }
    initial_iteration = GNUNET_NO;
    /* zone_head non-null, process hostnames from stdin */
    t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL);
    return;
  }
  if (NULL != name)
  {
    struct Zone *zone;
 
    zone = GNUNET_new (struct Zone);
    zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
    zone->domain = GNUNET_strdup (name);
    GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
  }
}
 
 
static void
run (void *cls,
     char *const *args,
     const char *cfgfile,
     const struct GNUNET_CONFIGURATION_Handle *cfg)
{
  (void) cls;
  (void) args;
  (void) cfgfile;
  stats = GNUNET_STATISTICS_create ("zoneimport", cfg);
  req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
  ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO);
  if (NULL == ns_pending)
  {
    fprintf (stderr, "Failed to allocate memory for main hash map\n");
    return;
  }
  dns_ctx = GNUNET_DNSSTUB_start (256);
  if (NULL == dns_ctx)
  {
    fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n");
    return;
  }
  if (NULL == args[0])
  {
    fprintf (stderr,
             "You must provide a list of DNS resolvers on the command line\n");
    return;
  }
  for (unsigned int i = 0; NULL != args[i]; i++)
  {
    if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (dns_ctx, args[i]))
    {
      fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
      return;
    }
  }
 
 
  GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
  ns = GNUNET_NAMESTORE_connect (cfg);
  if (NULL == ns)
  {
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL);
}
 
 
int
main (int argc, char *const *argv)
{
  struct GNUNET_GETOPT_CommandLineOption options[] =
  { GNUNET_GETOPT_option_uint ('s',
                               "size",
                               "MAPSIZE",
                               gettext_noop (
                                 "size to use for the main hash map"),
                               &map_size),
    GNUNET_GETOPT_option_relative_time (
      'm',
      "minimum-expiration",
      "RELATIVETIME",
      gettext_noop ("minimum expiration time we assume for imported records"),
      &minimum_expiration_time),
    GNUNET_GETOPT_OPTION_END };
  int ret;
 
  if (GNUNET_OK !=
      (ret = GNUNET_PROGRAM_run (GNUNET_OS_project_data_gnunet (),
                                 argc,
                                 argv,
                                 "gnunet-zoneimport",
                                 "import DNS zone into namestore",
                                 options,
                                 &run,
                                 NULL)))
    return 1;
  fprintf (stderr,
           "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
           "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
           rejects,
           cached,
           lookups,
           record_sets,
           records,
           failures,
           pending,
           pending_rs);
  return 0;
}
 
 
/* end of gnunet-zoneimport.c */

NS_BATCH_SIZE

#define NS_BATCH_SIZE   1024

How many requests do we request from NAMESTORE in one batch during our initial iteration?

Definition at line 82 of file gnunet-zoneimport.c.

Typedef Documentation

RecordProcessor

typedef void(* RecordProcessor) (void *cls, const struct GNUNET_DNSPARSER_Record *rec)

Callback for for_all_records.

Parameters

cls	closure
rec	a DNS record

Definition at line 408 of file gnunet-zoneimport.c.

Function Documentation

for_all_records()

static void for_all_records ( const struct GNUNET_DNSPARSER_Packet *  p, RecordProcessor  rp, void *  rp_cls  )

static

Call rp for each record in p, regardless of what response section it is in.

Parameters

p	packet from DNS
rp	function to call
rp_cls	closure for rp

Definition at line 421 of file gnunet-zoneimport.c.

{
  for (unsigned int i = 0; i < p->num_answers; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
 
    rp (rp_cls, rs);
  }
  for (unsigned int i = 0; i < p->num_authority_records; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
 
    rp (rp_cls, rs);
  }
  for (unsigned int i = 0; i < p->num_additional_records; i++)
  {
    struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
 
    rp (rp_cls, rs);
  }
}

References p, and rp.

Referenced by process_record(), and process_result().

Here is the caller graph for this function:

get_label()

static const char * get_label ( struct Request *  req )

static

Return just the label of the hostname in req.

Parameters

req	request to process hostname of

Returns

statically allocated pointer to the label, overwritten upon the next request!

Definition at line 454 of file gnunet-zoneimport.c.

{
  static char label[64];
  const char *dot;
 
  dot = strchr (req->hostname, (unsigned char) '.');
  if (NULL == dot)
  {
    GNUNET_break (0);
    return NULL;
  }
  if (((size_t) (dot - req->hostname)) >= sizeof(label))
  {
    GNUNET_break (0);
    return NULL;
  }
  GNUNET_memcpy (label, req->hostname, dot - req->hostname);
  label[dot - req->hostname] = '\0';
 
  return label;
}

References GNUNET_break, GNUNET_memcpy, and Request::hostname.

Referenced by get_label_utf8(), and ns_lookup_result_cb().

Here is the caller graph for this function:

get_label_utf8()

static char * get_label_utf8 ( struct Request *  req )

static

Definition at line 478 of file gnunet-zoneimport.c.

{
  char *label_utf8;
  const char *label_ace;
 
  label_ace = get_label (req);
  idna_to_unicode_8z8z (label_ace,
                        &label_utf8,
                        IDNA_ALLOW_UNASSIGNED);
  return label_utf8;
}

References get_label().

Referenced by process_result().

Here is the call graph for this function:

Here is the caller graph for this function:

build_dns_query()

static void * build_dns_query ( struct Request *  req, size_t *  raw_size  )

static

Build DNS query for hostname.

Parameters

	hostname	host to build query for
[out]	raw_size	number of bytes in the query

Returns

NULL on error, otherwise pointer to statically (!) allocated query buffer

Definition at line 500 of file gnunet-zoneimport.c.

{
  static char raw[512];
  char *name_ace;
  char *rawp;
  struct GNUNET_DNSPARSER_Packet p;
  struct GNUNET_DNSPARSER_Query q;
  int ret;
 
  if (IDNA_SUCCESS != idna_to_ascii_8z (req->hostname,
                                        &name_ace,
                                        IDNA_ALLOW_UNASSIGNED))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Name `%s' cannot be converted to IDNA."),
                req->hostname);
    return NULL;
  }
  q.name = (char *) name_ace;
  q.type = GNUNET_DNSPARSER_TYPE_NS;
  q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
 
  memset (&p, 0, sizeof(p));
  p.num_queries = 1;
  p.queries = &q;
  p.id = req->id;
  ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size);
  if (GNUNET_OK != ret)
  {
    if (GNUNET_NO == ret)
      GNUNET_free (rawp);
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to pack query for hostname `%s'\n",
                name_ace);
    rejects++;
    GNUNET_free (name_ace);
    return NULL;
  }
  if (*raw_size > sizeof(raw))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to pack query for hostname `%s'\n",
                name_ace);
    rejects++;
    GNUNET_break (0);
    GNUNET_free (name_ace);
    GNUNET_free (rawp);
    return NULL;
  }
  GNUNET_memcpy (raw, rawp, *raw_size);
  GNUNET_free (rawp);
  return raw;
}

References _, GNUNET_break, GNUNET_DNSPARSER_pack(), GNUNET_DNSPARSER_TYPE_NS, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_WARNING, GNUNET_free, GNUNET_log, GNUNET_memcpy, GNUNET_NO, GNUNET_OK, GNUNET_TUN_DNS_CLASS_INTERNET, Request::hostname, Request::id, p, q, raw, rejects, and ret.

Referenced by process_queue().

Here is the call graph for this function:

Here is the caller graph for this function:

free_records()

static void free_records ( struct Request *  req )

static

Free records associated with req.

Parameters

req	request to free records of

Definition at line 561 of file gnunet-zoneimport.c.

{
  struct Record *rec;
 
  /* Free records */
  while (NULL != (rec = req->rec_head))
  {
    GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec);
    GNUNET_free (rec);
  }
}

References GNUNET_CONTAINER_DLL_remove, GNUNET_free, Request::rec_head, and Request::rec_tail.

Referenced by free_request(), ns_lookup_result_cb(), and store_completed_cb().

Here is the caller graph for this function:

free_request()

static void free_request ( struct Request *  req )

static

Free req and data structures reachable from it.

Parameters

req	request to free

Definition at line 580 of file gnunet-zoneimport.c.

{
  free_records (req);
  GNUNET_free (req);
}

References free_records(), and GNUNET_free.

Referenced by do_shutdown(), free_request_it(), process_queue(), and process_result().

Here is the call graph for this function:

Here is the caller graph for this function:

process_queue()

static void process_queue ( void *  cls )

static

Process as many requests as possible from the queue.

Parameters

cls

NULL

Definition at line 1278 of file gnunet-zoneimport.c.

{
  struct Request *req;
  unsigned int series;
  void *raw;
  size_t raw_size;
  struct GNUNET_TIME_Relative delay;
 
  (void) cls;
  delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
  idle_time = GNUNET_TIME_relative_add (idle_time, delay);
  series = 0;
  t = NULL;
  while (pending + pending_rs < THRESH)
  {
    req = GNUNET_CONTAINER_heap_peek (req_heap);
    if (NULL == req)
      break;
    if (NULL != req->qe)
      return;   /* namestore op still pending */
    if (NULL != req->rs)
    {
      GNUNET_break (0);
      return;     /* already submitted */
    }
    if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
      break;
    GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap));
    req->hn = NULL;
    GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req);
    GNUNET_assert (NULL == req->rs);
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Requesting resolution for `%s'\n",
                req->hostname);
    raw = build_dns_query (req, &raw_size);
    if (NULL == raw)
    {
      GNUNET_break (0);
      free_request (req);
      continue;
    }
    req->op_start_time = GNUNET_TIME_absolute_get ();
    req->rs = GNUNET_DNSSTUB_resolve (dns_ctx, raw, raw_size, &process_result,
                                      req);
    GNUNET_assert (NULL != req->rs);
    req->issue_num++;
    lookups++;
    pending++;
    series++;
    if (series > MAX_SERIES)
      break;
  }
  if (pending + pending_rs >= THRESH)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stopped processing queue (%u+%u/%u)]\n",
                pending,
                pending_rs,
                THRESH);
    return;   /* wait for replies */
  }
  req = GNUNET_CONTAINER_heap_peek (req_heap);
  if (NULL == req)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stopped processing queue: empty queue\n");
    return;
  }
  if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Waiting until %s for next record (`%s') to expire\n",
                GNUNET_STRINGS_absolute_time_to_string (req->expires),
                req->hostname);
    if (NULL != t)
      GNUNET_SCHEDULER_cancel (t);
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
  if (NULL != t)
    GNUNET_SCHEDULER_cancel (t);
  sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
  t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL);
}

References build_dns_query(), dns_ctx, Request::expires, free_request(), GNUNET_assert, GNUNET_break, GNUNET_CONTAINER_DLL_insert, GNUNET_CONTAINER_heap_peek(), GNUNET_CONTAINER_heap_remove_root(), GNUNET_DNSSTUB_resolve(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_INFO, GNUNET_log, GNUNET_SCHEDULER_add_at(), GNUNET_SCHEDULER_add_delayed(), GNUNET_SCHEDULER_cancel(), GNUNET_STRINGS_absolute_time_to_string(), GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_duration(), GNUNET_TIME_absolute_get_remaining(), GNUNET_TIME_relative_add(), Request::hn, Request::hostname, idle_time, Request::issue_num, lookups, MAX_SERIES, Request::op_start_time, pending, pending_rs, process_queue(), process_result(), Request::qe, raw, GNUNET_TIME_Relative::rel_value_us, req_head, req_heap, req_tail, Request::rs, SERIES_DELAY, sleep_time_reg_proc, t, and THRESH.

Referenced by insert_sorted(), process_queue(), process_result(), and store_completed_cb().

Here is the call graph for this function:

Here is the caller graph for this function:

insert_sorted()

static void insert_sorted ( struct Request *  req )

static

Insert req into DLL sorted by next fetch time.

Parameters

req	request to insert into req_heap

Definition at line 602 of file gnunet-zoneimport.c.

{
  req->hn =
    GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us);
  if (req == GNUNET_CONTAINER_heap_peek (req_heap))
  {
    if (NULL != t)
      GNUNET_SCHEDULER_cancel (t);
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
  }
}

References GNUNET_TIME_Absolute::abs_value_us, Request::expires, GNUNET_CONTAINER_heap_insert(), GNUNET_CONTAINER_heap_peek(), GNUNET_SCHEDULER_add_at(), GNUNET_SCHEDULER_cancel(), GNUNET_TIME_absolute_get(), Request::hn, process_queue(), req_heap, sleep_time_reg_proc, and t.

Referenced by move_to_queue(), ns_lookup_result_cb(), and process_result().

Here is the call graph for this function:

Here is the caller graph for this function:

add_record()

static void add_record ( struct Request *  req, uint32_t  type, struct GNUNET_TIME_Absolute  expiration_time, const void *  data, size_t  data_len  )

static

Add record to the GNS record set for req.

Parameters

req	the request to expand GNS record set for
type	type to use
expiration_time	when should rec expire
data	raw data to store
data_len	number of bytes in data

Definition at line 626 of file gnunet-zoneimport.c.

{
  struct Record *rec;
 
  rec = GNUNET_malloc (sizeof(struct Record) + data_len);
  rec->grd.data = &rec[1];
  rec->grd.expiration_time = expiration_time.abs_value_us;
  rec->grd.data_size = data_len;
  rec->grd.record_type = type;
  rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
  GNUNET_memcpy (&rec[1], data, data_len);
  GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec);
}

References GNUNET_TIME_Absolute::abs_value_us, data, GNUNET_GNSRECORD_Data::data, GNUNET_GNSRECORD_Data::data_size, GNUNET_GNSRECORD_Data::expiration_time, GNUNET_GNSRECORD_Data::flags, GNUNET_CONTAINER_DLL_insert, GNUNET_GNSRECORD_RF_NONE, GNUNET_malloc, GNUNET_memcpy, Record::grd, Request::rec_head, Request::rec_tail, GNUNET_GNSRECORD_Data::record_type, and type.

Referenced by check_for_glue(), ns_lookup_result_cb(), and process_record().

Here is the caller graph for this function:

check_for_glue()

static void check_for_glue ( void *  cls, const struct GNUNET_DNSPARSER_Record *  rec  )

static

Try to find glue records for a given NS record.

Parameters

cls	a struct GlueClosure *
rec	record that may contain glue information

Definition at line 674 of file gnunet-zoneimport.c.

{
  struct GlueClosure *gc = cls;
  char dst[65536];
  size_t rd_len;
  char ip[INET6_ADDRSTRLEN + 1];
  socklen_t ip_size = (socklen_t) sizeof(ip);
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;
 
  if (0 != strcasecmp (rec->name, gc->ns))
    return;
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining (expiration_time);
  if (0 == left.rel_value_us)
    return; /* ignore expired glue records */
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
  rd_len = 0;
  switch (rec->type)
  {
  case GNUNET_DNSPARSER_TYPE_A:
    if (sizeof(struct in_addr) != rec->data.raw.data_len)
    {
      GNUNET_break (0);
      return;
    }
    if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size))
    {
      GNUNET_break (0);
      return;
    }
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   ip,
                   strlen (ip) + 1);
    rd_len += strlen (ip) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;
 
  case GNUNET_DNSPARSER_TYPE_AAAA:
    if (sizeof(struct in6_addr) != rec->data.raw.data_len)
    {
      GNUNET_break (0);
      return;
    }
    if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size))
    {
      GNUNET_break (0);
      return;
    }
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   ip,
                   strlen (ip) + 1);
    rd_len += strlen (ip) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;
 
  case GNUNET_DNSPARSER_TYPE_CNAME:
    // NAME
    GNUNET_memcpy (dst,
                   gc->req->hostname,
                   strlen (gc->req->hostname) + 1);
    rd_len += strlen (gc->req->hostname) + 1;
    // DNS SERVER NAME
    GNUNET_memcpy (dst + strlen (gc->req->hostname) + 1,
                   rec->data.hostname,
                   strlen (rec->data.hostname) + 1);
    rd_len += strlen (rec->data.hostname) + 1;
    add_record (gc->req,
                GNUNET_GNSRECORD_TYPE_GNS2DNS,
                expiration_time,
                dst,
                rd_len);
    gc->found = GNUNET_YES;
    break;
 
  default:
    /* useless, do nothing */
    break;
  }
}

References add_record(), GNUNET_DNSPARSER_RawRecord::data, GNUNET_DNSPARSER_Record::data, GNUNET_DNSPARSER_RawRecord::data_len, GNUNET_DNSPARSER_Record::expiration_time, GlueClosure::found, GNUNET_break, GNUNET_DNSPARSER_TYPE_A, GNUNET_DNSPARSER_TYPE_AAAA, GNUNET_DNSPARSER_TYPE_CNAME, GNUNET_GNSRECORD_TYPE_GNS2DNS, GNUNET_memcpy, GNUNET_TIME_absolute_get_remaining(), GNUNET_TIME_relative_to_absolute(), GNUNET_YES, Request::hostname, GNUNET_DNSPARSER_Record::hostname, minimum_expiration_time, GNUNET_DNSPARSER_Record::name, GlueClosure::ns, GNUNET_DNSPARSER_Record::raw, GNUNET_TIME_Relative::rel_value_us, GlueClosure::req, and GNUNET_DNSPARSER_Record::type.

Referenced by process_record().

Here is the call graph for this function:

Here is the caller graph for this function:

process_record()

static void process_record ( void *  cls, const struct GNUNET_DNSPARSER_Record *  rec  )

static

We received rec for req.

Remember the answer.

Parameters

cls	a struct ProcessRecordContext
rec	response

Definition at line 806 of file gnunet-zoneimport.c.

{
  struct ProcessRecordContext *prc = cls;
  struct Request *req = prc->req;
  char dst[65536];
  size_t dst_len;
  size_t off;
  struct GNUNET_TIME_Absolute expiration_time;
  struct GNUNET_TIME_Relative left;
 
  dst_len = sizeof(dst);
  off = 0;
  records++;
  if (0 != strcasecmp (rec->name, req->hostname))
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_DEBUG,
      "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
      rec->name,
      (unsigned int) rec->type,
      req->hostname);
    return;   /* does not match hostname, might be glue, but
                 not useful for this pass! */
  }
  expiration_time = rec->expiration_time;
  left = GNUNET_TIME_absolute_get_remaining (expiration_time);
  if (0 == left.rel_value_us)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "DNS returned expired record for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats,
                              "# expired records obtained from DNS",
                              1,
                              GNUNET_NO);
    return;   /* record expired */
  }
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "DNS returned record that expires at %s for `%s'\n",
              GNUNET_STRINGS_absolute_time_to_string (expiration_time),
              req->hostname);
  /* if expiration window is too short, bump it to configured minimum */
  if (left.rel_value_us < minimum_expiration_time.rel_value_us)
    expiration_time =
      GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
  switch (rec->type)
  {
  case GNUNET_DNSPARSER_TYPE_NS: {
      struct GlueClosure gc;
 
      /* check for glue */
      gc.req = req;
      gc.ns = rec->data.hostname;
      gc.found = GNUNET_NO;
      for_all_records (prc->p, &check_for_glue, &gc);
      if (GNUNET_NO == gc.found)
      {
        // NAME
        GNUNET_memcpy (dst,
                       req->hostname,
                       strlen (req->hostname) + 1);
        off += strlen (req->hostname) + 1;
        // DNS SERVER NAME
        GNUNET_memcpy (dst + strlen (req->hostname) + 1,
                       rec->data.hostname,
                       strlen (rec->data.hostname) + 1);
        off += strlen (rec->data.hostname) + 1;
        /* FIXME: actually check if this is out-of-bailiwick,
             and if not request explicit resolution... */
        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                    "Converted OOB (`%s') NS record for `%s'\n",
                    rec->data.hostname,
                    rec->name);
        add_record (req,
                    GNUNET_GNSRECORD_TYPE_GNS2DNS,
                    expiration_time,
                    dst,
                    off);
      }
      else
      {
        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                    "Converted NS record for `%s' using glue\n",
                    rec->name);
      }
      break;
    }
 
  case GNUNET_DNSPARSER_TYPE_CNAME:
    if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
                                                        dst_len,
                                                        &off,
                                                        rec->data.hostname))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting CNAME (`%s') record for `%s'\n",
                  rec->data.hostname,
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_DNAME:
    /* No support for DNAME in GNS yet! FIXME: support later! */
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "FIXME: not supported: %s DNAME %s\n",
                rec->name,
                rec->data.hostname);
    break;
 
  case GNUNET_DNSPARSER_TYPE_MX:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting MX (`%s') record for `%s'\n",
                  rec->data.mx->mxhost,
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_SOA:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa))
    {
      /* NOTE: GNS does not really use SOAs */
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting SOA record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_SRV:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting SRV record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_URI:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_uri (dst, dst_len, &off, rec->data.uri))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting URI record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_PTR:
    if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
                                                        dst_len,
                                                        &off,
                                                        rec->data.hostname))
    {
      /* !?: what does a PTR record do in a regular TLD??? */
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting PTR record for `%s' (weird)\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  case GNUNET_DNSPARSER_TYPE_CERT:
    if (GNUNET_OK ==
        GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Converting CERT record for `%s'\n",
                  rec->name);
      add_record (req, rec->type, expiration_time, dst, off);
    }
    break;
 
  /* Rest is 'raw' encoded and just needs to be copied IF
     the hostname matches the requested name; otherwise we
     simply cannot use it. */
  case GNUNET_DNSPARSER_TYPE_A:
  case GNUNET_DNSPARSER_TYPE_AAAA:
  case GNUNET_DNSPARSER_TYPE_TXT:
  default:
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Converting record of type %u for `%s'\n",
                (unsigned int) rec->type,
                rec->name);
    add_record (req,
                rec->type,
                expiration_time,
                rec->data.raw.data,
                rec->data.raw.data_len);
    break;
  }
}

References add_record(), GNUNET_DNSPARSER_Record::cert, check_for_glue(), GNUNET_DNSPARSER_RawRecord::data, GNUNET_DNSPARSER_Record::data, GNUNET_DNSPARSER_RawRecord::data_len, GNUNET_DNSPARSER_Record::expiration_time, for_all_records(), GlueClosure::found, GNUNET_DNSPARSER_builder_add_cert(), GNUNET_DNSPARSER_builder_add_mx(), GNUNET_DNSPARSER_builder_add_name(), GNUNET_DNSPARSER_builder_add_soa(), GNUNET_DNSPARSER_builder_add_srv(), GNUNET_DNSPARSER_builder_add_uri(), GNUNET_DNSPARSER_TYPE_A, GNUNET_DNSPARSER_TYPE_AAAA, GNUNET_DNSPARSER_TYPE_CERT, GNUNET_DNSPARSER_TYPE_CNAME, GNUNET_DNSPARSER_TYPE_DNAME, GNUNET_DNSPARSER_TYPE_MX, GNUNET_DNSPARSER_TYPE_NS, GNUNET_DNSPARSER_TYPE_PTR, GNUNET_DNSPARSER_TYPE_SOA, GNUNET_DNSPARSER_TYPE_SRV, GNUNET_DNSPARSER_TYPE_TXT, GNUNET_DNSPARSER_TYPE_URI, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_WARNING, GNUNET_GNSRECORD_TYPE_GNS2DNS, GNUNET_log, GNUNET_memcpy, GNUNET_NO, GNUNET_OK, GNUNET_STATISTICS_update(), GNUNET_STRINGS_absolute_time_to_string(), GNUNET_TIME_absolute_get_remaining(), GNUNET_TIME_relative_to_absolute(), Request::hostname, GNUNET_DNSPARSER_Record::hostname, minimum_expiration_time, GNUNET_DNSPARSER_Record::mx, GNUNET_DNSPARSER_MxRecord::mxhost, GNUNET_DNSPARSER_Record::name, GlueClosure::ns, ProcessRecordContext::p, GNUNET_DNSPARSER_Record::raw, records, GNUNET_TIME_Relative::rel_value_us, GlueClosure::req, ProcessRecordContext::req, GNUNET_DNSPARSER_Record::soa, GNUNET_DNSPARSER_Record::srv, stats, GNUNET_DNSPARSER_Record::type, and GNUNET_DNSPARSER_Record::uri.

Referenced by process_result().

Here is the call graph for this function:

Here is the caller graph for this function:

store_completed_cb()

static void store_completed_cb ( void *  cls, enum GNUNET_ErrorCode  ec  )

static

Definition at line 1010 of file gnunet-zoneimport.c.

{
  static struct GNUNET_TIME_Absolute last;
  struct Request *req = cls;
 
  req->qe = NULL;
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to store zone data for `%s': %s\n",
                req->hostname,
                GNUNET_ErrorCode_get_hint (ec));
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stored records under `%s' (%d)\n",
                req->hostname,
                ec);
  }
  total_reg_proc_dns_ns++; /* finished regular processing */
  pending_rs--;
  free_records (req);
  /* compute NAMESTORE statistics */
  {
    static uint64_t total_ns_latency_cnt;
    static struct GNUNET_TIME_Relative total_ns_latency;
    struct GNUNET_TIME_Relative ns_latency;
 
    ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
    total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency);
    if (0 == total_ns_latency_cnt)
      last = GNUNET_TIME_absolute_get ();
    total_ns_latency_cnt++;
    if (0 == (total_ns_latency_cnt % 1000))
    {
      struct GNUNET_TIME_Relative delta;
      struct GNUNET_TIME_Relative est_remaining;
 
      delta = GNUNET_TIME_absolute_get_duration (last);
      last = GNUNET_TIME_absolute_get ();
      est_remaining = GNUNET_TIME_relative_multiply (
        delta,
        (stdin_lines_read - total_ns_latency_cnt) / 1000);
      fprintf (stderr,
               "Processed 1000 records in %s. ",
               GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
      fprintf (stderr,
               "Estimated remaining time: %s. ",
               GNUNET_STRINGS_relative_time_to_string (
                 est_remaining,
                 GNUNET_YES));
      fprintf (stderr,
               "Last domain processed: %s\n",
               req->hostname);
      GNUNET_STATISTICS_set (stats,
                             "# average NAMESTORE PUT latency (μs)",
                             total_ns_latency.rel_value_us
                             / total_ns_latency_cnt,
                             GNUNET_NO);
    }
  }
  /* compute and publish overall velocity */
  if (0 == (total_reg_proc_dns_ns % 100))
  {
    struct GNUNET_TIME_Relative runtime;
 
    runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
    runtime = GNUNET_TIME_relative_subtract (runtime, idle_time);
    runtime =
      GNUNET_TIME_relative_divide (runtime,
                                   total_reg_proc_dns + total_reg_proc_dns_ns);
    GNUNET_STATISTICS_set (stats,
                           "# Regular processing completed without NAMESTORE",
                           total_reg_proc_dns,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# Regular processing completed with NAMESTORE PUT",
                           total_reg_proc_dns_ns,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# average request processing latency (μs)",
                           runtime.rel_value_us,
                           GNUNET_NO);
    GNUNET_STATISTICS_set (stats,
                           "# total time spent idle (μs)",
                           idle_time.rel_value_us,
                           GNUNET_NO);
  }
 
  if (NULL == t)
  {
    sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
    t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
  }
}

References delta, free_records(), GNUNET_EC_NONE, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ErrorCode_get_hint(), GNUNET_log, GNUNET_NO, GNUNET_SCHEDULER_add_now(), GNUNET_STATISTICS_set(), GNUNET_STRINGS_relative_time_to_string(), GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_duration(), GNUNET_TIME_relative_add(), GNUNET_TIME_relative_divide(), GNUNET_TIME_relative_multiply(), GNUNET_TIME_relative_subtract(), GNUNET_YES, Request::hostname, idle_time, Request::op_start_time, pending_rs, process_queue(), Request::qe, GNUNET_TIME_Relative::rel_value_us, sleep_time_reg_proc, start_time_reg_proc, stats, stdin_lines_read, t, total_reg_proc_dns, and total_reg_proc_dns_ns.

Referenced by process_result().

Here is the call graph for this function:

Here is the caller graph for this function:

process_result()

static void process_result ( void *  cls, const struct GNUNET_TUN_DnsHeader *  dns, size_t  dns_len  )

static

Function called with the result of a DNS resolution.

Parameters

cls	closure with the struct Request
dns	dns response, never NULL
dns_len	number of bytes in dns

Definition at line 1116 of file gnunet-zoneimport.c.

{
  struct Request *req = cls;
  struct Record *rec;
  struct GNUNET_DNSPARSER_Packet *p;
  unsigned int rd_count;
 
  GNUNET_assert (NULL == req->hn);
  if ((NULL == dns) ||
      (dns->flags.return_code != GNUNET_TUN_DNS_RETURN_CODE_NO_ERROR))
  {
    /* stub gave up */
    GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
    if (NULL != dns)
    {
      GNUNET_DNSSTUB_resolve_cancel (req->rs);
      req->rs = NULL;
    }
    pending--;
    if (NULL == t)
    {
      sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
      t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
    }
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Stub gave up on DNS reply for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats, "# Unsuccessful DNS lookups", 1, GNUNET_NO)
    ;
    if (req->issue_num > MAX_RETRIES)
    {
      failures++;
      free_request (req);
      GNUNET_STATISTICS_update (stats, "# DNS requests given up on", 1,
                                GNUNET_NO);
      return;
    }
    total_reg_proc_dns++;
    req->rs = NULL;
    insert_sorted (req);
    return;
  }
  if (req->id != dns->id)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "DNS ID did not match request, ignoring reply\n");
    GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO);
    return;
  }
  GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
  GNUNET_DNSSTUB_resolve_cancel (req->rs);
  req->rs = NULL;
  pending--;
  p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len);
  if (NULL == p)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to parse DNS reply for `%s'\n",
                req->hostname);
    GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO);
    if (NULL == t)
    {
      sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
      t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
    }
    if (req->issue_num > MAX_RETRIES)
    {
      failures++;
      free_request (req);
      GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
      return;
    }
    insert_sorted (req);
    return;
  }
  /* import new records */
  req->issue_num = 0; /* success, reset counter! */
  {
    struct ProcessRecordContext prc = { .req = req, .p = p };
 
    for_all_records (p, &process_record, &prc);
  }
  GNUNET_DNSPARSER_free_packet (p);
  /* count records found, determine minimum expiration time */
  req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
  {
    struct GNUNET_TIME_Relative dns_latency;
 
    dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
    total_dns_latency =
      GNUNET_TIME_relative_add (total_dns_latency, dns_latency);
    total_dns_latency_cnt++;
    if (0 == (total_dns_latency_cnt % 1000))
    {
      GNUNET_STATISTICS_set (stats,
                             "# average DNS lookup latency (μs)",
                             total_dns_latency.rel_value_us
                             / total_dns_latency_cnt,
                             GNUNET_NO);
    }
  }
  rd_count = 0;
  for (rec = req->rec_head; NULL != rec; rec = rec->next)
  {
    struct GNUNET_TIME_Absolute at;
 
    at.abs_value_us = rec->grd.expiration_time;
    req->expires = GNUNET_TIME_absolute_min (req->expires, at);
    rd_count++;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Obtained %u records for `%s'\n",
              rd_count,
              req->hostname);
  /* Instead of going for SOA, simplified for now to look each
     day in case we got an empty response */
  if (0 == rd_count)
  {
    req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
    GNUNET_STATISTICS_update (stats,
                              "# empty DNS replies (usually NXDOMAIN)",
                              1,
                              GNUNET_NO);
  }
  else
  {
    record_sets++;
  }
  /* convert records to namestore import format */
  {
    struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)];
    unsigned int off = 0;
    char *label_utf8 = get_label_utf8 (req);
 
    /* convert linked list into array */
    for (rec = req->rec_head; NULL != rec; rec = rec->next)
      rd[off++] = rec->grd;
    pending_rs++;
    req->op_start_time = GNUNET_TIME_absolute_get ();
    req->qe = GNUNET_NAMESTORE_record_set_store (ns,
                                                 &req->zone->key,
                                                 label_utf8,
                                                 rd_count,
                                                 rd,
                                                 &store_completed_cb,
                                                 req);
    GNUNET_free (label_utf8);
    GNUNET_assert (NULL != req->qe);
  }
  insert_sorted (req);
  return;
}

References GNUNET_TIME_Absolute::abs_value_us, GNUNET_GNSRECORD_Data::expiration_time, Request::expires, failures, GNUNET_TUN_DnsHeader::flags, for_all_records(), free_request(), get_label_utf8(), GNUNET_assert, GNUNET_CONTAINER_DLL_remove, GNUNET_DNSPARSER_free_packet(), GNUNET_DNSPARSER_parse(), GNUNET_DNSSTUB_resolve_cancel(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_INFO, GNUNET_free, GNUNET_log, GNUNET_NAMESTORE_record_set_store(), GNUNET_NO, GNUNET_NZL, GNUNET_SCHEDULER_add_now(), GNUNET_STATISTICS_set(), GNUNET_STATISTICS_update(), GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_duration(), GNUNET_TIME_absolute_min(), GNUNET_TIME_relative_add(), GNUNET_TIME_relative_to_absolute(), GNUNET_TIME_UNIT_DAYS, GNUNET_TIME_UNIT_FOREVER_ABS, GNUNET_TUN_DNS_RETURN_CODE_NO_ERROR, Record::grd, Request::hn, Request::hostname, Request::id, GNUNET_TUN_DnsHeader::id, insert_sorted(), Request::issue_num, Zone::key, MAX_RETRIES, Record::next, ns, Request::op_start_time, p, pending, pending_rs, process_queue(), process_record(), Request::qe, rd, rd_count, Request::rec_head, record_sets, GNUNET_TIME_Relative::rel_value_us, ProcessRecordContext::req, req_head, req_tail, GNUNET_TUN_DnsFlags::return_code, Request::rs, sleep_time_reg_proc, stats, store_completed_cb(), t, total_dns_latency, total_dns_latency_cnt, total_reg_proc_dns, and Request::zone.

Referenced by process_queue().

Here is the call graph for this function:

Here is the caller graph for this function:

free_request_it()

static int free_request_it ( void *  cls, const struct GNUNET_HashCode *  key, void *  value  )

static

Iterator called during do_shutdown() to free requests in the ns_pending map.

Parameters

cls	NULL
key	unused
value	the struct Request to free

Returns

GNUNET_OK

Definition at line 1376 of file gnunet-zoneimport.c.

{
  struct Request *req = value;
 
  (void) cls;
  (void) key;
  free_request (req);
  return GNUNET_OK;
}

References free_request(), GNUNET_OK, key, and value.

Referenced by do_shutdown().

Here is the call graph for this function:

Here is the caller graph for this function:

do_shutdown()

static void do_shutdown ( void *  cls )

static

Clean up and terminate the process.

Parameters

cls

NULL

Definition at line 1393 of file gnunet-zoneimport.c.

{
  struct Request *req;
  struct Zone *zone;
  struct MissingZoneCreationCtx *mzctx;
 
  (void) cls;
  while (NULL != (mzctx = missing_zones_head))
  {
    GNUNET_CONTAINER_DLL_remove (missing_zones_head,
                                 missing_zones_tail,
                                 mzctx);
    if (NULL != mzctx->id_op)
    {
      GNUNET_IDENTITY_cancel (mzctx->id_op);
      mzctx->id_op = NULL;
    }
    if (NULL != mzctx->ns_qe)
    {
      GNUNET_NAMESTORE_cancel (mzctx->ns_qe);
      mzctx->ns_qe = NULL;
    }
    GNUNET_free (mzctx);
  }
  if (NULL != id)
  {
    GNUNET_IDENTITY_disconnect (id);
    id = NULL;
  }
  if (NULL != t)
  {
    GNUNET_SCHEDULER_cancel (t);
    t = NULL;
  }
  while (NULL != (req = req_head))
  {
    GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
    if (NULL != req->qe)
      GNUNET_NAMESTORE_cancel (req->qe);
    free_request (req);
  }
  while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
  {
    req->hn = NULL;
    if (NULL != req->qe)
      GNUNET_NAMESTORE_cancel (req->qe);
    free_request (req);
  }
  if (NULL != zone_it)
  {
    GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
    zone_it = NULL;
  }
  if (NULL != ns)
  {
    GNUNET_NAMESTORE_disconnect (ns);
    ns = NULL;
  }
  if (NULL != dns_ctx)
  {
    GNUNET_DNSSTUB_stop (dns_ctx);
    dns_ctx = NULL;
  }
  if (NULL != req_heap)
  {
    GNUNET_CONTAINER_heap_destroy (req_heap);
    req_heap = NULL;
  }
  if (NULL != ns_pending)
  {
    GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL);
    GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
    ns_pending = NULL;
  }
  while (NULL != (zone = zone_head))
  {
    GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone);
    GNUNET_free (zone->domain);
    GNUNET_free (zone);
  }
  if (NULL != stats)
  {
    GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
    stats = NULL;
  }
}

References dns_ctx, Zone::domain, free_request(), free_request_it(), GNUNET_CONTAINER_DLL_remove, GNUNET_CONTAINER_heap_destroy(), GNUNET_CONTAINER_heap_remove_root(), GNUNET_CONTAINER_multihashmap_destroy(), GNUNET_CONTAINER_multihashmap_iterate(), GNUNET_DNSSTUB_stop(), GNUNET_free, GNUNET_IDENTITY_cancel(), GNUNET_IDENTITY_disconnect(), GNUNET_NAMESTORE_cancel(), GNUNET_NAMESTORE_disconnect(), GNUNET_NAMESTORE_zone_iteration_stop(), GNUNET_NO, GNUNET_SCHEDULER_cancel(), GNUNET_STATISTICS_destroy(), Request::hn, MissingZoneCreationCtx::id_op, missing_zones_head, missing_zones_tail, ns, ns_pending, MissingZoneCreationCtx::ns_qe, Request::qe, MissingZoneCreationCtx::req, req_head, req_heap, req_tail, stats, t, zone_head, zone_it, and zone_tail.

Here is the call graph for this function:

iterate_zones()

static void iterate_zones ( void *  cls )

static

Iterate over all of the zones we care about and see which records we may need to re-fetch when.

Parameters

cls

NULL

Definition at line 1869 of file gnunet-zoneimport.c.

{
  static struct Zone *last;
 
  (void) cls;
  if (NULL != zone_it)
  {
    zone_it = NULL;
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Finished iteration over zone `%s'!\n",
                last->domain);
    /* subtract left-overs from previous iteration */
    GNUNET_STATISTICS_update (stats,
                              "# NAMESTORE records requested from cache",
                              (long long) (-ns_iterator_trigger_next),
                              GNUNET_NO);
    ns_iterator_trigger_next = 0;
  }
  GNUNET_assert (NULL != zone_tail);
  if (zone_tail == last)
  {
    /* Done iterating over relevant zones in NAMESTORE, move
       rest of hash map to work queue as well. */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Finished all NAMESTORE iterations!\n");
    GNUNET_STATISTICS_set (stats,
                           "# Domain names without cached reply",
                           GNUNET_CONTAINER_multihashmap_size (ns_pending),
                           GNUNET_NO);
    GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL);
    GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
    ns_pending = NULL;
    start_time_reg_proc = GNUNET_TIME_absolute_get ();
    total_reg_proc_dns = 0;
    total_reg_proc_dns_ns = 0;
    return;
  }
  if (NULL == last)
    last = zone_head;
  else
    last = last->next;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Starting iteration over zone `%s'!\n",
              last->domain);
  /* subtract left-overs from previous iteration */
  GNUNET_STATISTICS_update (stats,
                            "# NAMESTORE records requested from cache",
                            1,
                            GNUNET_NO);
  ns_iterator_trigger_next = 1;
  GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO);
  zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
                                                   &last->key,
                                                   &ns_lookup_error_cb,
                                                   NULL,
                                                   &ns_lookup_result_cb,
                                                   last,
                                                   &iterate_zones,
                                                   NULL);
}

References Zone::domain, GNUNET_assert, GNUNET_CONTAINER_multihashmap_destroy(), GNUNET_CONTAINER_multihashmap_iterate(), GNUNET_CONTAINER_multihashmap_size(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_NAMESTORE_zone_iteration_start(), GNUNET_NO, GNUNET_STATISTICS_set(), GNUNET_STATISTICS_update(), GNUNET_TIME_absolute_get(), iterate_zones(), Zone::key, move_to_queue(), Zone::next, ns, ns_iterator_trigger_next, ns_lookup_error_cb(), ns_lookup_result_cb(), ns_pending, start_time_reg_proc, stats, total_reg_proc_dns, total_reg_proc_dns_ns, zone_head, zone_it, and zone_tail.

Referenced by delegation_store_cont(), iterate_zones(), ns_lookup_error_cb(), and process_stdin().

Here is the call graph for this function:

Here is the caller graph for this function:

ns_lookup_error_cb()

static void ns_lookup_error_cb ( void *  cls )

static

Function called if GNUNET_NAMESTORE_records_lookup() failed.

Just logs an error.

Parameters

cls	a struct Zone

Definition at line 1498 of file gnunet-zoneimport.c.

{
  struct Zone *zone = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Failed to load data from namestore for zone `%s'\n",
              zone->domain);
  zone_it = NULL;
  ns_iterator_trigger_next = 0;
  iterate_zones (NULL);
}

References Zone::domain, GNUNET_ERROR_TYPE_INFO, GNUNET_log, iterate_zones(), ns_iterator_trigger_next, and zone_it.

Referenced by iterate_zones().

Here is the call graph for this function:

Here is the caller graph for this function:

ns_lookup_result_cb()

static void ns_lookup_result_cb ( void *  cls, const struct GNUNET_CRYPTO_PrivateKey *  key, const char *  label_utf8, unsigned int  rd_count, const struct GNUNET_GNSRECORD_Data *  rd  )

static

Process a record that was stored in the namestore.

Parameters

cls	a struct Zone *
key	private key of the zone
label	label of the records
rd_count	number of entries in rd array, 0 if label was deleted
rd	array of records with data to store

Definition at line 1521 of file gnunet-zoneimport.c.

{
  struct Zone *zone = cls;
  struct Request *req;
  struct GNUNET_HashCode hc;
  char *label_ace;
  char *fqdn;
 
  ns_iterator_trigger_next--;
  if (0 == ns_iterator_trigger_next)
  {
    ns_iterator_trigger_next = NS_BATCH_SIZE;
    GNUNET_STATISTICS_update (stats,
                              "# NAMESTORE records requested from cache",
                              ns_iterator_trigger_next,
                              GNUNET_NO);
    GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next);
  }
  if (IDNA_SUCCESS != idna_to_ascii_8z (label_utf8,
                                        &label_ace,
                                        IDNA_ALLOW_UNASSIGNED))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Name `%s' cannot be converted to IDNA."),
                label_utf8);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Obtained NAMESTORE reply for %s (Punycode `%s', %llu left in round\n",
              label_utf8,
              label_ace,
              (unsigned long long) ns_iterator_trigger_next);
 
  GNUNET_asprintf (&fqdn, "%s.%s", label_ace, zone->domain);
  GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc);
  GNUNET_free (fqdn);
  req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc);
  if (NULL == req)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Ignoring record `%s' in zone `%s': not on my list!\n",
                label_ace,
                zone->domain);
    GNUNET_free (label_ace);
    return;
  }
  GNUNET_assert (GNUNET_OK ==
                 GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req));
  GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key));
  GNUNET_break (0 == strcasecmp (label_ace, get_label (req)));
  GNUNET_free (label_ace);
  for (unsigned int i = 0; i < rd_count; i++)
  {
    struct GNUNET_TIME_Absolute at;
 
    if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
    {
      struct GNUNET_TIME_Relative rel;
 
      rel.rel_value_us = rd->expiration_time;
      at = GNUNET_TIME_relative_to_absolute (rel);
    }
    else
    {
      at.abs_value_us = rd->expiration_time;
    }
    add_record (req, rd->record_type, at, rd->data, rd->data_size);
  }
  if (0 == rd_count)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Empty record set in namestore for `%s'\n",
                req->hostname);
  }
  else
  {
    unsigned int pos = 0;
 
    cached++;
    req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
    for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
    {
      struct GNUNET_TIME_Absolute at;
 
      at.abs_value_us = rec->grd.expiration_time;
      req->expires = GNUNET_TIME_absolute_min (req->expires, at);
      pos++;
    }
    if (0 == pos)
      req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Hot-start with %u existing records for `%s'\n",
                pos,
                req->hostname);
  }
  free_records (req);
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Adding `%s' to worklist to start at %s\n",
              req->hostname,
              GNUNET_STRINGS_absolute_time_to_string (req->expires));
  insert_sorted (req);
}

References _, GNUNET_TIME_Absolute::abs_value_us, add_record(), cached, GNUNET_GNSRECORD_Data::data, GNUNET_GNSRECORD_Data::data_size, Zone::domain, GNUNET_GNSRECORD_Data::expiration_time, Request::expires, GNUNET_GNSRECORD_Data::flags, free_records(), get_label(), GNUNET_asprintf(), GNUNET_assert, GNUNET_break, GNUNET_CONTAINER_multihashmap_get(), GNUNET_CONTAINER_multihashmap_remove(), GNUNET_CRYPTO_hash(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_INFO, GNUNET_ERROR_TYPE_WARNING, GNUNET_free, GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION, GNUNET_log, GNUNET_memcmp, GNUNET_NAMESTORE_zone_iterator_next(), GNUNET_NO, GNUNET_OK, GNUNET_STATISTICS_update(), GNUNET_STRINGS_absolute_time_to_string(), GNUNET_TIME_absolute_min(), GNUNET_TIME_relative_to_absolute(), GNUNET_TIME_UNIT_FOREVER_ABS, GNUNET_TIME_UNIT_ZERO_ABS, Request::hostname, insert_sorted(), key, Zone::key, NS_BATCH_SIZE, ns_iterator_trigger_next, ns_pending, rd, rd_count, Request::rec_head, GNUNET_GNSRECORD_Data::record_type, GNUNET_TIME_Relative::rel_value_us, stats, Request::zone, and zone_it.

Referenced by iterate_zones().

Here is the call graph for this function:

Here is the caller graph for this function:

delegation_store_cont()

static void delegation_store_cont ( void *  cls, enum GNUNET_ErrorCode  ec  )

static

Definition at line 1631 of file gnunet-zoneimport.c.

{
  struct MissingZoneCreationCtx *mzctx = cls;
 
  mzctx->ns_qe = NULL;
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to store delegation: `%s'\n",
                GNUNET_ErrorCode_get_hint (ec));
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Added delegation successfully\n");
  GNUNET_CONTAINER_DLL_remove (missing_zones_head,
                               missing_zones_tail,
                               mzctx);
  GNUNET_free (mzctx);
  if (NULL == missing_zones_head)
  {
    iterate_zones (NULL);
  }
 
}

References GNUNET_CONTAINER_DLL_remove, GNUNET_EC_NONE, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ErrorCode_get_hint(), GNUNET_free, GNUNET_log, GNUNET_SCHEDULER_shutdown(), iterate_zones(), missing_zones_head, missing_zones_tail, and MissingZoneCreationCtx::ns_qe.

Referenced by missing_zone_creation_cont().

Here is the call graph for this function:

Here is the caller graph for this function:

missing_zone_creation_cont()

static void missing_zone_creation_cont ( void *  cls, const struct GNUNET_CRYPTO_PrivateKey *  sk, enum GNUNET_ErrorCode  ec  )

static

Definition at line 1660 of file gnunet-zoneimport.c.

{
  struct MissingZoneCreationCtx *mzctx = cls;
  struct GNUNET_CRYPTO_PublicKey pk;
  struct Zone *parent_zone;
  struct Zone *zone;
  const char *dot;
  const char *expected_parent_zone_name;
  char parent_delegation_label[GNUNET_DNSPARSER_MAX_NAME_LENGTH];
 
  if (GNUNET_EC_NONE != ec)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to create zone for `%s': %s\n",
                mzctx->req->hostname,
                GNUNET_ErrorCode_get_hint (ec));
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  mzctx->id_op = NULL;
  dot = strchr (mzctx->req->hostname, (unsigned char) '.');
  GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
              "Created missing ego `%s'\n",
              dot + 1);
  zone = GNUNET_new (struct Zone);
  zone->key = *sk;
  zone->domain = GNUNET_strdup (dot + 1);
  mzctx->req->zone = zone;
  GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
 
  expected_parent_zone_name = strchr (dot + 1,
                                      (unsigned char) '.');
  if (NULL == expected_parent_zone_name)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Created identity without parent zone!\n");
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  {
    int label_len = strlen (dot + 1) - strlen (expected_parent_zone_name) + 1;
    snprintf (parent_delegation_label,
              label_len,
              "%s",
              dot + 1);
    parent_delegation_label[label_len] = '\0';
  }
  for (parent_zone = zone_head;
       NULL != parent_zone;
       parent_zone = parent_zone->next)
  {
    if (0 != strcmp (expected_parent_zone_name + 1,
                     parent_zone->domain))
      continue;
    break;
  }
  GNUNET_assert (NULL != parent_zone);
  GNUNET_CRYPTO_key_get_public (sk, &pk);
  struct GNUNET_GNSRECORD_Data delegation_rd;
  char *data;
  size_t data_size;
  GNUNET_assert (GNUNET_OK ==
                 GNUNET_GNSRECORD_data_from_identity (&pk,
                                                      &data,
                                                      &data_size,
                                                      &delegation_rd.record_type
                                                      ));
  delegation_rd.flags = GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION;
  delegation_rd.expiration_time = GNUNET_TIME_UNIT_WEEKS.rel_value_us;
  delegation_rd.data = data;
  delegation_rd.data_size = data_size;
  mzctx->ns_qe = GNUNET_NAMESTORE_record_set_store (ns,
                                                    &parent_zone->key,
                                                    parent_delegation_label,
                                                    1,
                                                    &delegation_rd,
                                                    &delegation_store_cont,
                                                    mzctx);
  GNUNET_free (data);
}

References data, GNUNET_GNSRECORD_Data::data, data_size, GNUNET_GNSRECORD_Data::data_size, delegation_store_cont(), Zone::domain, GNUNET_GNSRECORD_Data::expiration_time, GNUNET_GNSRECORD_Data::flags, GNUNET_assert, GNUNET_CONTAINER_DLL_insert, GNUNET_CRYPTO_key_get_public(), GNUNET_DNSPARSER_MAX_NAME_LENGTH, GNUNET_EC_NONE, GNUNET_ERROR_TYPE_ERROR, GNUNET_ErrorCode_get_hint(), GNUNET_free, GNUNET_GNSRECORD_data_from_identity(), GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION, GNUNET_log, GNUNET_NAMESTORE_record_set_store(), GNUNET_new, GNUNET_OK, GNUNET_SCHEDULER_shutdown(), GNUNET_strdup, GNUNET_TIME_UNIT_WEEKS, Request::hostname, MissingZoneCreationCtx::id_op, Zone::key, Zone::next, ns, MissingZoneCreationCtx::ns_qe, pk, GNUNET_GNSRECORD_Data::record_type, MissingZoneCreationCtx::req, Request::zone, zone_head, and zone_tail.

Referenced by queue().

Here is the call graph for this function:

Here is the caller graph for this function:

queue()

static void queue ( const char *  hostname )

static

Add hostname to the list of requests to be made.

Parameters

hostname

name to resolve

Definition at line 1751 of file gnunet-zoneimport.c.

{
  struct Request *req;
  const char *dot;
  struct Zone *zone;
  size_t hlen;
  struct GNUNET_HashCode hc;
 
  if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Refusing invalid hostname `%s'\n",
                hostname);
    rejects++;
    return;
  }
  dot = strchr (hostname, (unsigned char) '.');
  if (NULL == dot)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Refusing invalid hostname `%s' (lacks '.')\n",
                hostname);
    rejects++;
    return;
  }
  for (zone = zone_head; NULL != zone; zone = zone->next)
    if (0 == strcmp (zone->domain, dot + 1))
      break;
  hlen = strlen (hostname) + 1;
  req = GNUNET_malloc (sizeof(struct Request) + hlen);
  if (NULL == zone)
  {
    struct MissingZoneCreationCtx *mzctx;
 
    for (mzctx = missing_zones_head;
         NULL != mzctx;
         mzctx = mzctx->next)
    {
      const char *tmp_dot;
      tmp_dot = strchr (mzctx->req->hostname, (unsigned char) '.');
      GNUNET_assert (NULL != tmp_dot);
      if (0 != strcmp (dot + 1, tmp_dot + 1))
        continue;
      break;
    }
    if (NULL == mzctx)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  "Domain name `%s' not in ego list! Creating.\n",
                  dot + 1);
      mzctx = GNUNET_new (struct MissingZoneCreationCtx);
      mzctx->id_op = GNUNET_IDENTITY_create (id,
                                             dot + 1,
                                             NULL,
                                             GNUNET_PUBLIC_KEY_TYPE_EDDSA,
                                             &missing_zone_creation_cont,
                                             mzctx);
      mzctx->req = req;
      GNUNET_CONTAINER_DLL_insert (missing_zones_head,
                                   missing_zones_tail,
                                   mzctx);
    }
  }
  else
  {
    req->zone = zone;
  }
 
  req->hostname = (char *) &req[1];
  GNUNET_memcpy (req->hostname, hostname, hlen);
  req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
                                                 UINT16_MAX);
  GNUNET_CRYPTO_hash (req->hostname, hlen, &hc);
  if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
        ns_pending,
        &hc,
        req,
        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Duplicate hostname `%s' ignored\n",
                hostname);
    GNUNET_free (req);
    return;
  }
}

References Zone::domain, GNUNET_assert, GNUNET_CONTAINER_DLL_insert, GNUNET_CONTAINER_multihashmap_put(), GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY, GNUNET_CRYPTO_hash(), GNUNET_CRYPTO_QUALITY_NONCE, GNUNET_CRYPTO_random_u32(), GNUNET_DNSPARSER_check_name(), GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_INFO, GNUNET_ERROR_TYPE_WARNING, GNUNET_free, GNUNET_IDENTITY_create(), GNUNET_log, GNUNET_malloc, GNUNET_memcpy, GNUNET_new, GNUNET_OK, GNUNET_PUBLIC_KEY_TYPE_EDDSA, Request::hostname, Request::id, MissingZoneCreationCtx::id_op, missing_zone_creation_cont(), missing_zones_head, missing_zones_tail, Zone::next, MissingZoneCreationCtx::next, ns_pending, rejects, MissingZoneCreationCtx::req, Request::zone, and zone_head.

Referenced by process_stdin().

Here is the call graph for this function:

Here is the caller graph for this function:

move_to_queue()

static int move_to_queue ( void *  cls, const struct GNUNET_HashCode *  key, void *  value  )

static

We have completed the initial iteration over the namestore's database.

This function is called on each of the remaining records in move_to_queue to queue() them, as we will simply not find existing records for them any longer.

Parameters

cls	NULL
key	unused
value	a struct Request

Returns

GNUNET_OK (continue to iterate)

Definition at line 1851 of file gnunet-zoneimport.c.

{
  struct Request *req = value;
 
  (void) cls;
  (void) key;
  insert_sorted (req);
  return GNUNET_OK;
}

References GNUNET_OK, insert_sorted(), key, and value.

Referenced by iterate_zones().

Here is the call graph for this function:

Here is the caller graph for this function:

process_stdin()

static void process_stdin ( void *  cls )

static

Begin processing hostnames from stdin.

Parameters

cls

NULL

Definition at line 1937 of file gnunet-zoneimport.c.

{
  static struct GNUNET_TIME_Absolute last;
  char hn[256];
  char *hn_utf8;
 
  (void) cls;
  t = NULL;
  while (NULL != fgets (hn, sizeof(hn), stdin))
  {
    if (strlen (hn) > 0)
      hn[strlen (hn) - 1] = '\0';  /* eat newline */
    if (0 == stdin_lines_read)
      last = GNUNET_TIME_absolute_get ();
    stdin_lines_read++;
    if (0 == stdin_lines_read % 100000)
    {
      struct GNUNET_TIME_Relative delta;
 
      delta = GNUNET_TIME_absolute_get_duration (last);
      last = GNUNET_TIME_absolute_get ();
      fprintf (stderr,
               "Read 100000 domain names in %s\n",
               GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
      GNUNET_STATISTICS_set (stats, "# domain names provided", stdin_lines_read,
                             GNUNET_NO);
    }
    idna_to_unicode_8z8z (hn,
                          &hn_utf8,
                          IDNA_ALLOW_UNASSIGNED);
    queue (hn_utf8);
    GNUNET_free (hn_utf8);
  }
  fprintf (stderr,
           "Done reading %llu domain names\n",
           (unsigned long long) stdin_lines_read);
  GNUNET_STATISTICS_set (stats, "# domain names provided", stdin_lines_read,
                         GNUNET_NO);
  // Only start iteration immediately if no zones are missing.
  if (NULL == missing_zones_head)
  {
    iterate_zones (NULL);
  }
}

References delta, GNUNET_free, GNUNET_NO, GNUNET_STATISTICS_set(), GNUNET_STRINGS_relative_time_to_string(), GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_duration(), GNUNET_YES, iterate_zones(), missing_zones_head, queue(), stats, stdin_lines_read, and t.

Referenced by identity_cb().

Here is the call graph for this function:

Here is the caller graph for this function:

identity_cb()

static void identity_cb ( void *  cls, struct GNUNET_IDENTITY_Ego *  ego, void **  ctx, const char *  name  )

static

Method called to inform about the egos of this peer.

When used with GNUNET_IDENTITY_connect, this function is initially called for all egos and then again whenever a ego's name changes or if it is deleted. At the end of the initial pass over all egos, the function is once called with 'NULL' for ego. That does NOT mean that the callback won't be invoked in the future or that there was an error.

When used with GNUNET_IDENTITY_create, this function is only called ONCE, and 'NULL' being passed in ego does indicate an error (for example because name is taken or no default value is known). If ego is non-NULL and if '*ctx' is set in those callbacks, the value WILL be passed to a subsequent call to the identity callback of GNUNET_IDENTITY_connect (if that one was not NULL).

When an identity is renamed, this function is called with the (known) ego but the NEW name.

When an identity is deleted, this function is called with the (known) ego and "NULL" for the name. In this case, the ego is henceforth invalid (and the ctx should also be cleaned up).

Parameters

cls	closure
ego	ego handle, NULL for end of list
ctx	context for application to store data for this ego (during the lifetime of this process, initially NULL)
name	name assigned by the user for this ego, NULL if the user just deleted the ego and it must thus no longer be used

Definition at line 2017 of file gnunet-zoneimport.c.

{
  (void) cls;
  (void) ctx;
  static int initial_iteration = GNUNET_YES;
 
  if (GNUNET_NO == initial_iteration)
    return;
  if (NULL == ego)
  {
    /* end of iteration */
    if (NULL == zone_head)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
      GNUNET_SCHEDULER_shutdown ();
      return;
    }
    initial_iteration = GNUNET_NO;
    /* zone_head non-null, process hostnames from stdin */
    t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL);
    return;
  }
  if (NULL != name)
  {
    struct Zone *zone;
 
    zone = GNUNET_new (struct Zone);
    zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
    zone->domain = GNUNET_strdup (name);
    GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
  }
}

References ctx, Zone::domain, GNUNET_CONTAINER_DLL_insert, GNUNET_ERROR_TYPE_ERROR, GNUNET_IDENTITY_ego_get_private_key(), GNUNET_log, GNUNET_new, GNUNET_NO, GNUNET_SCHEDULER_add_now(), GNUNET_SCHEDULER_shutdown(), GNUNET_strdup, GNUNET_YES, Zone::key, name, process_stdin(), t, zone_head, and zone_tail.

Referenced by run().

Here is the call graph for this function:

Here is the caller graph for this function:

run()

static void run ( void *  cls, char *const *  args, const char *  cfgfile, const struct GNUNET_CONFIGURATION_Handle *  cfg  )

static

Process requests from the queue, then if the queue is not empty, try again.

Parameters

cls	NULL
args	remaining command-line arguments
cfgfile	name of the configuration file used (for saving, can be NULL!)
cfg	configuration

Definition at line 2064 of file gnunet-zoneimport.c.

{
  (void) cls;
  (void) args;
  (void) cfgfile;
  stats = GNUNET_STATISTICS_create ("zoneimport", cfg);
  req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
  ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO);
  if (NULL == ns_pending)
  {
    fprintf (stderr, "Failed to allocate memory for main hash map\n");
    return;
  }
  dns_ctx = GNUNET_DNSSTUB_start (256);
  if (NULL == dns_ctx)
  {
    fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n");
    return;
  }
  if (NULL == args[0])
  {
    fprintf (stderr,
             "You must provide a list of DNS resolvers on the command line\n");
    return;
  }
  for (unsigned int i = 0; NULL != args[i]; i++)
  {
    if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (dns_ctx, args[i]))
    {
      fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
      return;
    }
  }
 
 
  GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
  ns = GNUNET_NAMESTORE_connect (cfg);
  if (NULL == ns)
  {
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL);
}

References cfg, dns_ctx, do_shutdown, GNUNET_CONTAINER_heap_create(), GNUNET_CONTAINER_HEAP_ORDER_MIN, GNUNET_CONTAINER_multihashmap_create(), GNUNET_DNSSTUB_add_dns_ip(), GNUNET_DNSSTUB_start(), GNUNET_IDENTITY_connect(), GNUNET_NAMESTORE_connect(), GNUNET_NO, GNUNET_OK, GNUNET_SCHEDULER_add_shutdown(), GNUNET_SCHEDULER_shutdown(), GNUNET_STATISTICS_create(), identity_cb(), map_size, ns, ns_pending, req_heap, and stats.

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

main()

int main	(	int	argc,
		char *const *	argv
	)

Call with IP address of resolver to query.

Parameters

argc	should be 2
argv[1]	should contain IP address

Returns

0 on success

Definition at line 2121 of file gnunet-zoneimport.c.

{
  struct GNUNET_GETOPT_CommandLineOption options[] =
  { GNUNET_GETOPT_option_uint ('s',
                               "size",
                               "MAPSIZE",
                               gettext_noop (
                                 "size to use for the main hash map"),
                               &map_size),
    GNUNET_GETOPT_option_relative_time (
      'm',
      "minimum-expiration",
      "RELATIVETIME",
      gettext_noop ("minimum expiration time we assume for imported records"),
      &minimum_expiration_time),
    GNUNET_GETOPT_OPTION_END };
  int ret;
 
  if (GNUNET_OK !=
      (ret = GNUNET_PROGRAM_run (GNUNET_OS_project_data_gnunet (),
                                 argc,
                                 argv,
                                 "gnunet-zoneimport",
                                 "import DNS zone into namestore",
                                 options,
                                 &run,
                                 NULL)))
    return 1;
  fprintf (stderr,
           "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
           "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
           rejects,
           cached,
           lookups,
           record_sets,
           records,
           failures,
           pending,
           pending_rs);
  return 0;
}

References cached, failures, gettext_noop, GNUNET_GETOPT_OPTION_END, GNUNET_GETOPT_option_relative_time(), GNUNET_GETOPT_option_uint(), GNUNET_OK, GNUNET_OS_project_data_gnunet(), GNUNET_PROGRAM_run(), lookups, map_size, minimum_expiration_time, options, pending, pending_rs, record_sets, records, rejects, ret, and run().

Here is the call graph for this function:

Variable Documentation

minimum_expiration_time

struct GNUNET_TIME_Relative minimum_expiration_time

static

How long do DNS records have to last at least after being imported?

Definition at line 76 of file gnunet-zoneimport.c.

Referenced by check_for_glue(), main(), and process_record().

missing_zones_head

struct MissingZoneCreationCtx* missing_zones_head

static

Definition at line 241 of file gnunet-zoneimport.c.

Referenced by delegation_store_cont(), do_shutdown(), process_stdin(), and queue().

missing_zones_tail

struct MissingZoneCreationCtx* missing_zones_tail

static

Definition at line 244 of file gnunet-zoneimport.c.

Referenced by delegation_store_cont(), do_shutdown(), and queue().

stdin_lines_read

uint64_t stdin_lines_read

static

Definition at line 247 of file gnunet-zoneimport.c.

Referenced by process_stdin(), and store_completed_cb().

map_size

unsigned int map_size = 1024

static

Command-line argument specifying desired size of the hash map with all of our pending names.

Usually, we use an automatically growing map, but this is only OK up to about a million entries. Above that number, the user must explicitly specify the size at startup.

Definition at line 255 of file gnunet-zoneimport.c.

Referenced by main(), process_notify(), and run().

id

struct GNUNET_IDENTITY_Handle* id

static

Handle to the identity service.

Definition at line 260 of file gnunet-zoneimport.c.

ns

struct GNUNET_NAMESTORE_Handle* ns

static

Namestore handle.

Definition at line 265 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), iterate_zones(), missing_zone_creation_cont(), process_result(), and run().

stats

struct GNUNET_STATISTICS_Handle* stats

static

Handle to the statistics service.

Definition at line 270 of file gnunet-zoneimport.c.

Referenced by announce_id(), destroy_direction(), destroy_route(), dir_ready_cb(), discard_all_from_rung_tail(), discard_buffer(), do_shutdown(), GCC_latency_observed(), GCCH_channel_incoming_new(), GCCH_channel_local_new(), GCCH_handle_channel_open_ack(), GCCH_handle_channel_plaintext_data(), GCCH_handle_channel_plaintext_data_ack(), GCD_search(), GCT_handle_encrypted(), GCT_handle_kx(), GCT_handle_kx_auth(), GCT_send(), handle_connection_create(), handle_plaintext_keepalive(), iterate_zones(), ns_lookup_result_cb(), process_record(), process_result(), process_stdin(), REGEX_INTERNAL_announce(), REGEX_INTERNAL_search(), route_message(), run(), send_keepalive(), send_kx(), send_kx_auth(), store_completed_cb(), and update_ax_by_kx().

dns_ctx

struct GNUNET_DNSSTUB_Context* dns_ctx

static

Context for DNS resolution.

Definition at line 275 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), process_queue(), and run().

pending

unsigned int pending

static

The number of DNS queries that are outstanding.

Definition at line 280 of file gnunet-zoneimport.c.

Referenced by main(), process_queue(), and process_result().

pending_rs

unsigned int pending_rs

static

The number of NAMESTORE record store operations that are outstanding.

Definition at line 285 of file gnunet-zoneimport.c.

Referenced by main(), process_queue(), process_result(), and store_completed_cb().

lookups

unsigned int lookups

static

Number of lookups we performed overall.

Definition at line 290 of file gnunet-zoneimport.c.

Referenced by main(), and process_queue().

cached

unsigned int cached

static

Number of records we had cached.

Definition at line 295 of file gnunet-zoneimport.c.

Referenced by main(), and ns_lookup_result_cb().

rejects

unsigned int rejects

static

How many hostnames did we reject (malformed).

Definition at line 300 of file gnunet-zoneimport.c.

Referenced by build_dns_query(), main(), and queue().

failures

unsigned int failures

static

Number of lookups that failed.

Definition at line 305 of file gnunet-zoneimport.c.

Referenced by main(), and process_result().

records

unsigned int records

static

Number of records we found.

Definition at line 310 of file gnunet-zoneimport.c.

Referenced by cache_answers(), GNUNET_GNSRECORD_JSON_from_gnsrecord(), main(), and process_record().

record_sets

unsigned int record_sets

static

Number of record sets given to namestore.

Definition at line 315 of file gnunet-zoneimport.c.

Referenced by main(), and process_result().

req_heap

struct GNUNET_CONTAINER_Heap* req_heap

static

Heap of all requests to perform, sorted by the time we should next do the request (i.e.

by expires).

Definition at line 321 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), insert_sorted(), process_queue(), and run().

req_head

struct Request* req_head

static

Active requests are kept in a DLL.

Definition at line 326 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), process_queue(), and process_result().

req_tail

struct Request* req_tail

static

Active requests are kept in a DLL.

Definition at line 331 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), process_queue(), and process_result().

t

struct GNUNET_SCHEDULER_Task* t

static

Main task.

Definition at line 336 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), identity_cb(), insert_sorted(), process_queue(), process_result(), process_stdin(), and store_completed_cb().

ns_pending

struct GNUNET_CONTAINER_MultiHashMap* ns_pending

static

Hash map of requests for which we may still get a response from the namestore.

Set to NULL once the initial namestore iteration is done.

Definition at line 343 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), iterate_zones(), ns_lookup_result_cb(), queue(), and run().

zone_it

struct GNUNET_NAMESTORE_ZoneIterator* zone_it

static

Current zone iteration handle.

Definition at line 348 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), iterate_zones(), ns_lookup_error_cb(), and ns_lookup_result_cb().

zone_head

struct Zone* zone_head

static

Head of list of zones we are managing.

Definition at line 353 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), identity_cb(), iterate_zones(), missing_zone_creation_cont(), and queue().

zone_tail

struct Zone* zone_tail

static

Tail of list of zones we are managing.

Definition at line 358 of file gnunet-zoneimport.c.

Referenced by do_shutdown(), identity_cb(), iterate_zones(), and missing_zone_creation_cont().

ns_iterator_trigger_next

uint64_t ns_iterator_trigger_next

static

After how many more results must ns_lookup_result_cb() ask the namestore for more?

Definition at line 364 of file gnunet-zoneimport.c.

Referenced by iterate_zones(), ns_lookup_error_cb(), and ns_lookup_result_cb().

total_dns_latency_cnt

uint64_t total_dns_latency_cnt

static

Number of DNS requests counted in latency total.

Definition at line 369 of file gnunet-zoneimport.c.

Referenced by process_result().

total_dns_latency

struct GNUNET_TIME_Relative total_dns_latency

static

Sum of DNS latencies observed.

Definition at line 374 of file gnunet-zoneimport.c.

Referenced by process_result().

total_reg_proc_dns

uint64_t total_reg_proc_dns

static

Number of records processed (DNS lookup, no NAMESTORE) in total.

Definition at line 379 of file gnunet-zoneimport.c.

Referenced by iterate_zones(), process_result(), and store_completed_cb().

total_reg_proc_dns_ns

uint64_t total_reg_proc_dns_ns

static

Number of records processed (DNS lookup, with NAMESTORE) in total.

Definition at line 384 of file gnunet-zoneimport.c.

Referenced by iterate_zones(), and store_completed_cb().

start_time_reg_proc

struct GNUNET_TIME_Absolute start_time_reg_proc

static

Start time of the regular processing.

Definition at line 389 of file gnunet-zoneimport.c.

Referenced by iterate_zones(), and store_completed_cb().

sleep_time_reg_proc

struct GNUNET_TIME_Absolute sleep_time_reg_proc

static

Last time we worked before going idle.

Definition at line 394 of file gnunet-zoneimport.c.

Referenced by insert_sorted(), process_queue(), process_result(), and store_completed_cb().

idle_time

struct GNUNET_TIME_Relative idle_time

static

Time we slept just waiting for work.

Definition at line 399 of file gnunet-zoneimport.c.

Referenced by process_queue(), and store_completed_cb().