GNUnet  0.19.2
gnunet-service-zonemaster.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
26 #include "platform.h"
27 #include <pthread.h>
28 #include "gnunet_util_lib.h"
29 #include "gnunet_dht_service.h"
33 
34 #define LOG_STRERROR_FILE(kind, syscall, \
35  filename) GNUNET_log_from_strerror_file (kind, "util", \
36  syscall, \
37  filename)
38 
39 
44 #define PUBLISH_OPS_PER_EXPIRATION 4
45 
51 #define DELTA_INTERVAL 100
52 
56 #define NS_BLOCK_SIZE 1000
57 
61 #define JOB_QUEUE_LIMIT 5000
62 
67 #define NAMESTORE_MONITOR_QUEUE_LIMIT 5
68 
73 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
74 
79 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply ( \
80  GNUNET_TIME_UNIT_MINUTES, 15)
81 
86 #define LATE_ITERATION_SPEEDUP_FACTOR 2
87 
91 #define DHT_GNS_REPLICATION_LEVEL 5
92 
96 static pthread_t * worker;
97 
101 static pthread_mutex_t sign_jobs_lock;
102 
106 static pthread_mutex_t sign_results_lock;
107 
111 static pthread_cond_t sign_jobs_cond;
112 
116 static int in_shutdown = GNUNET_NO;
117 
122 
127 
132 
134 {
135 
140 
145 
150 
155 
161 
165  size_t block_size;
166 
171 
175  char *label;
176 
181 
186 };
187 
188 
194 
199 
204 
209 
210 
215 
220 
221 
226 {
231 
236 
241 
242 };
243 
244 
249 
254 
259 
264 
269 
274 static int disable_namecache;
275 
280 
284 static unsigned int job_queue_length;
285 
289 static unsigned long long num_public_records;
290 
294 static unsigned long long last_num_public_records;
295 
300 static unsigned long long put_cnt;
301 
310 
316 
322 
327 
333 
338 static struct GNUNET_TIME_Absolute last_put_100;
339 
346 static struct GNUNET_TIME_Relative sub_delta;
347 
352 
357 static unsigned int ns_iteration_left;
358 
363 
368 static int cache_keys;
369 
373 static struct CacheOperation *cop_head;
374 
378 static struct CacheOperation *cop_tail;
379 
380 
381 static void
383 {
384  if (job->block != job->block_priv)
385  GNUNET_free (job->block_priv);
386  GNUNET_free (job->block);
387  if (NULL != job->label)
388  GNUNET_free (job->label);
389  GNUNET_free (job);
390 }
391 
398 static void
399 shutdown_task (void *cls)
400 {
401  struct CacheOperation *cop;
402  struct RecordPublicationJob *job;
403 
404  (void) cls;
407  "Shutting down!\n");
408  if (NULL != notification_pipe)
410  if (NULL != pipe_read_task)
412  while (NULL != (cop = cop_head))
413  {
415  "Aborting incomplete namecache operation\n");
418  GNUNET_free (cop);
419  }
420  GNUNET_assert (0 == pthread_mutex_lock (&sign_jobs_lock));
421  while (NULL != (job = sign_jobs_head))
422  {
424  "Removing incomplete jobs\n");
427  free_job (job);
428  }
429  GNUNET_assert (0 == pthread_mutex_unlock (&sign_jobs_lock));
430  GNUNET_assert (0 == pthread_mutex_lock (&sign_results_lock));
431  while (NULL != (job = sign_results_head))
432  {
434  "Removing incomplete jobs\n");
436  free_job (job);
437  }
438  GNUNET_assert (0 == pthread_mutex_unlock (&sign_results_lock));
439  while (NULL != (job = dht_jobs_head))
440  {
442  "Removing incomplete jobs\n");
444  if (NULL != job->ph)
446  free_job (job);
447  }
448  if (NULL != statistics)
449  {
451  GNUNET_NO);
452  statistics = NULL;
453  }
454  if (NULL != zone_publish_task)
455  {
457  zone_publish_task = NULL;
458  }
459  if (NULL != namestore_iter)
460  {
462  namestore_iter = NULL;
463  }
464  if (NULL != zmon)
465  {
467  zmon = NULL;
468  }
469  if (NULL != namestore_handle)
470  {
472  namestore_handle = NULL;
473  }
474  if (NULL != namecache)
475  {
477  namecache = NULL;
478  }
479  if (NULL != dht_handle)
480  {
482  dht_handle = NULL;
483  }
484 }
485 
486 
494 static void
495 finish_cache_operation (void *cls, int32_t success, const char *emsg)
496 {
497  struct CacheOperation *cop = cls;
498 
499  if (NULL != emsg)
501  _ ("Failed to replicate block in namecache: %s\n"),
502  emsg);
503  else
504  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CACHE operation completed\n");
506  GNUNET_free (cop);
507 }
508 
509 
518 static void
520 {
521  struct CacheOperation *cop;
522 
524  {
526  "Namecache updates skipped (NC disabled)",
527  1,
528  GNUNET_NO);
529  return;
530  }
531  GNUNET_assert (NULL != block);
532  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Caching block in namecache\n");
534  "Namecache updates pushed",
535  1,
536  GNUNET_NO);
537  cop = GNUNET_new (struct CacheOperation);
540  block,
542  cop);
543 }
544 
545 
551 static void
553 {
554  (void) cls;
555  zone_publish_task = NULL;
556  GNUNET_assert (NULL != namestore_iter);
560  NS_BLOCK_SIZE);
561 }
562 
563 
569 static void
570 publish_zone_dht_start (void *cls);
571 
572 
576 static void
578 {
579  if (0 == num_public_records)
587  "No records in namestore database.\n");
588  }
589  else
590  {
602  }
607  "Minimum relative record expiration (in μs)",
609  GNUNET_NO);
611  "Zone publication time window (in μs)",
613  GNUNET_NO);
615  "Target zone iteration velocity (μs)",
617  GNUNET_NO);
618 }
619 
620 
628 static void
629 update_velocity (unsigned int cnt)
630 {
632  unsigned long long pct = 0;
633 
634  if (0 == cnt)
635  return;
636  /* How fast were we really? */
638  delta.rel_value_us /= cnt;
640 
641  /* calculate expected frequency */
644  {
646  "Last record count was lower than current record count. Reducing interval.\n");
650  }
652  "Desired global zone iteration interval is %s/record!\n",
655  GNUNET_YES));
656 
657  /* Tell statistics actual vs. desired speed */
659  "Current zone iteration velocity (μs/record)",
661  GNUNET_NO);
662  /* update "sub_delta" based on difference, taking
663  previous sub_delta into account! */
665  {
666  /* We were too fast, reduce sub_delta! */
667  struct GNUNET_TIME_Relative corr;
668 
670  delta);
672  {
673  /* Reduce sub_delta by corr */
675  corr);
676  }
677  else
678  {
679  /* We're doing fine with waiting the full time, this
680  should theoretically only happen if we run at
681  infinite speed. */
683  }
684  }
687  {
688  /* We were too slow, increase sub_delta! */
689  struct GNUNET_TIME_Relative corr;
690 
694  corr);
695  if (sub_delta.rel_value_us >
697  {
698  /* CPU overload detected, we cannot go at desired speed,
699  as this would mean using a negative delay. */
700  /* compute how much faster we would want to be for
701  the desired velocity */
703  pct = UINT64_MAX; /* desired speed is infinity ... */
704  else
705  pct = (sub_delta.rel_value_us
709  }
710  }
712  "# dispatched jobs",
714  GNUNET_NO);
716  "% speed increase needed for target velocity",
717  pct,
718  GNUNET_NO);
720  "# records processed in current iteration",
722  GNUNET_NO);
723 }
724 
725 
730 static void
732 {
734 
735  if (0 != ns_iteration_left)
736  return; /* current NAMESTORE iteration not yet done */
738  {
740  "Job queue length exceeded (%u/%u). Pausing namestore iteration.\n",
743  return;
744  }
746  put_cnt = 0;
748  sub_delta);
749  /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
750  per-record delay calculated so far with the #NS_BLOCK_SIZE */
752  "Current artificial NAMESTORE delay (μs/record)",
754  GNUNET_NO);
756  NS_BLOCK_SIZE);
757  /* make sure we do not overshoot because of the #NS_BLOCK_SIZE factor */
759  delay);
760  /* no delays on first iteration */
766  NULL);
767 }
768 
769 
774 static void
776 {
777  struct RecordPublicationJob *job = cls;
778 
780  "PUT complete; Pending jobs: %u\n", job_queue_length - 1);
781  /* When we just fall under the limit, trigger monitor/iterator again
782  * if halted. We can only safely trigger one, prefer iterator. */
783  if (NULL == zone_publish_task)
786  {
787  if (GNUNET_YES == monitor_halted)
788  {
791  }
792  }
796  job);
797  free_job (job);
798 }
799 
800 
811 static void
813  const char *label,
814  const struct GNUNET_GNSRECORD_Data *rd,
815  unsigned int rd_count,
816  const struct GNUNET_TIME_Absolute expire)
817 {
818  struct GNUNET_GNSRECORD_Data rd_public[rd_count];
819  struct GNUNET_GNSRECORD_Block *block;
820  struct GNUNET_GNSRECORD_Block *block_priv;
821  struct GNUNET_TIME_Absolute expire_pub;
822  size_t block_size;
823  unsigned int rd_public_count = 0;
824  char *emsg;
825 
826  if (GNUNET_OK !=
828  rd,
829  rd_count,
830  rd_public,
831  &rd_public_count,
832  &expire_pub,
834  &emsg))
835  {
837  "%s\n", emsg);
838  GNUNET_free (emsg);
839  }
840 
842  expire_pub,
843  label,
844  rd_public,
845  rd_public_count,
846  &block));
847  if (NULL == block)
848  {
849  GNUNET_break (0);
850  return; /* whoops */
851  }
852  if (rd_count != rd_public_count)
854  expire,
855  label,
856  rd,
857  rd_count,
858  &
859  block_priv));
860  else
861  block_priv = block;
862  block_size = GNUNET_GNSRECORD_block_get_size (block);
863  GNUNET_assert (0 == pthread_mutex_lock (&sign_jobs_lock));
865  job->block = block;
866  job->block_size = block_size;
867  job->block_priv = block_priv;
868  job->zone = *key;
869  job->label = GNUNET_strdup (label);
870  job->expire_pub = expire_pub;
872  GNUNET_assert (0 == pthread_cond_signal (&sign_jobs_cond));
873  GNUNET_assert (0 == pthread_mutex_unlock (&sign_jobs_lock));
875  "Creating job with %u record(s) for label `%s', expiration `%s'\n",
876  rd_public_count,
877  label,
880  return;
881 }
882 
883 
884 static void
885 notification_pipe_cb (void *cls);
886 
887 static void
889 {
890  struct GNUNET_HashCode query;
891  struct RecordPublicationJob *job;
892  const struct GNUNET_DISK_FileHandle *np_fh;
893  char buf[100];
894  ssize_t nf_count;
895 
896  pipe_read_task = NULL;
902  np_fh,
904  NULL);
905  /* empty queue */
906  nf_count = GNUNET_DISK_file_read (np_fh, buf, sizeof (buf));
907  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Read %lld notifications from pipe\n",
908  (long long) nf_count);
909  while (true)
910  {
911  GNUNET_assert (0 == pthread_mutex_lock (&sign_results_lock));
912  if (NULL == sign_results_head)
913  {
915  "No more results. Back to sleep.\n");
916  GNUNET_assert (0 == pthread_mutex_unlock (&sign_results_lock));
917  return;
918  }
921  GNUNET_assert (0 == pthread_mutex_unlock (&sign_results_lock));
923  job->label,
924  &query);
926  &query,
930  job->block_size,
931  job->block,
932  job->expire_pub,
934  job);
935  if (NULL == job->ph)
936  {
938  "Could not perform DHT PUT, is the DHT running?\n");
939  free_job (job);
940  return;
941  }
943  "DHT put operations initiated",
944  1,
945  GNUNET_NO);
947  "Storing record(s) for label `%s' in DHT under key %s\n",
948  job->label,
949  GNUNET_h2s (&query));
950  refresh_block (job->block_priv);
952  }
953 }
954 
955 
961 static void
963 {
964  (void) cls;
966  "Got disconnected from namestore database, retrying.\n");
967  namestore_iter = NULL;
968  /* We end up here on error/disconnect/shutdown, so potentially
969  while a zone publish task or a DHT put is still running; hence
970  we need to cancel those. */
971  if (NULL != zone_publish_task)
972  {
974  zone_publish_task = NULL;
975  }
977  NULL);
978 }
979 
980 
986 static void
988 {
989  (void) cls;
990  /* we're done with one iteration, calculate when to do the next one */
991  namestore_iter = NULL;
996  /* reset for next iteration */
1000  "Zone iteration finished. Adjusted zone iteration interval to %s\n",
1003  GNUNET_YES));
1005  "Target zone iteration velocity (μs)",
1007  GNUNET_NO);
1009  "Number of public records in DHT",
1011  GNUNET_NO);
1012  GNUNET_assert (NULL == zone_publish_task);
1013  if (0 == last_num_public_records)
1014  {
1018  NULL);
1019  }
1020  else
1021  {
1023  NULL);
1024  }
1025 }
1026 
1027 
1037 static void
1038 handle_record (void *cls,
1039  const struct GNUNET_IDENTITY_PrivateKey *key,
1040  const char *label,
1041  unsigned int rd_count,
1042  const struct GNUNET_GNSRECORD_Data *rd,
1044 {
1045  (void) cls;
1047  if (0 == rd_count)
1048  {
1050  "Record set empty, moving to next record set\n");
1052  return;
1053  }
1054  for (unsigned int i = 0; i < rd_count; i++)
1055  {
1056  if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1057  {
1058  /* GNUNET_GNSRECORD_block_create will convert to absolute time;
1059  we just need to adjust our iteration frequency */
1061  GNUNET_MIN (rd[i].expiration_time,
1063  }
1064  }
1065 
1066 
1067  /* We got a set of records to publish */
1069  "Starting DHT PUT\n");
1070  put_cnt++;
1071  if (0 == put_cnt % DELTA_INTERVAL)
1073  dispatch_job (key,
1074  label,
1075  rd,
1076  rd_count,
1077  expire);
1078  job_queue_length++;
1080 }
1081 
1082 
1088 static void
1090 {
1091  (void) cls;
1092  zone_publish_task = NULL;
1094  "Full zone iterations launched",
1095  1,
1096  GNUNET_NO);
1098  "Starting DHT zone update!\n");
1099  /* start counting again */
1100  num_public_records = 0;
1101  GNUNET_assert (NULL == namestore_iter);
1102  ns_iteration_left = 1;
1105  NULL, /* All zones */
1107  NULL,
1108  &handle_record,
1109  NULL,
1111  NULL,
1113  GNUNET_assert (NULL != namestore_iter);
1114 }
1115 
1116 
1125 static void
1127  const char *label,
1128  const struct GNUNET_GNSRECORD_Data *rd,
1129  unsigned int rd_count,
1131 {
1132  struct GNUNET_GNSRECORD_Data rd_public[rd_count];
1133  struct GNUNET_GNSRECORD_Block *block;
1134  struct GNUNET_GNSRECORD_Block *block_priv;
1135  struct GNUNET_TIME_Absolute expire_pub;
1136  size_t block_size;
1137  unsigned int rd_public_count = 0;
1138  char *emsg;
1139 
1140  if (GNUNET_OK !=
1142  rd,
1143  rd_count,
1144  rd_public,
1145  &rd_public_count,
1146  &expire_pub,
1148  &emsg))
1149  {
1151  "%s\n", emsg);
1152  GNUNET_free (emsg);
1153  }
1154 
1156  expire_pub,
1157  label,
1158  rd_public,
1159  rd_public_count,
1160  &block));
1161  if (NULL == block)
1162  {
1163  GNUNET_break (0);
1164  return; /* whoops */
1165  }
1166  if (rd_count != rd_public_count)
1168  expire,
1169  label,
1170  rd,
1171  rd_count,
1172  &
1173  block_priv));
1174  else
1175  block_priv = block;
1176  block_size = GNUNET_GNSRECORD_block_get_size (block);
1177  GNUNET_assert (0 == pthread_mutex_lock (&sign_jobs_lock));
1179  job->block = block;
1180  job->block_size = block_size;
1181  job->block_priv = block_priv;
1182  job->zone = *key;
1183  job->label = GNUNET_strdup (label);
1184  job->expire_pub = expire_pub;
1186  GNUNET_assert (0 == pthread_cond_signal (&sign_jobs_cond));
1187  GNUNET_assert (0 == pthread_mutex_unlock (&sign_jobs_lock));
1188 }
1189 
1190 
1202 static void
1204  const struct GNUNET_IDENTITY_PrivateKey *zone,
1205  const char *label,
1206  unsigned int rd_count,
1207  const struct GNUNET_GNSRECORD_Data *rd,
1209 {
1210  (void) cls;
1212  "Namestore monitor events received",
1213  1,
1214  GNUNET_NO);
1216  "Received %u records for label `%s' via namestore monitor\n",
1217  rd_count,
1218  label);
1219  if (0 == rd_count)
1220  {
1222  1);
1223  return; /* nothing to do */
1224  }
1226  label,
1227  rd,
1228  rd_count,
1229  expire);
1230  job_queue_length++;
1232  {
1234  "Job queue length exceeded (%u/%u). Halting monitor.\n",
1236  JOB_QUEUE_LIMIT);
1238  return;
1239  }
1241  1);
1242 }
1243 
1244 
1251 static void
1253 {
1254  (void) cls;
1256  "Namestore monitor errors encountered",
1257  1,
1258  GNUNET_NO);
1259 }
1260 
1261 
1262 static void*
1263 sign_worker (void *cls)
1264 {
1265  struct RecordPublicationJob *job;
1266  const struct GNUNET_DISK_FileHandle *fh;
1267 
1269  while (GNUNET_YES != in_shutdown)
1270  {
1271  GNUNET_assert (0 == pthread_mutex_lock (&sign_jobs_lock));
1272  while (NULL == sign_jobs_head)
1273  GNUNET_assert (0 == pthread_cond_wait (&sign_jobs_cond, &sign_jobs_lock));
1274  if (GNUNET_YES == in_shutdown)
1275  {
1276  GNUNET_assert (0 == pthread_mutex_unlock (&sign_jobs_lock));
1277  return NULL;
1278  }
1280  "Taking on Job for %s\n", sign_jobs_head->label);
1281  job = sign_jobs_head;
1283  GNUNET_assert (0 == pthread_mutex_unlock (&sign_jobs_lock));
1284  GNUNET_GNSRECORD_block_sign (&job->zone, job->label, job->block);
1285  if (job->block != job->block_priv)
1286  GNUNET_GNSRECORD_block_sign (&job->zone, job->label, job->block_priv);
1287  GNUNET_assert (0 == pthread_mutex_lock (&sign_results_lock));
1289  GNUNET_assert (0 == pthread_mutex_unlock (&sign_results_lock));
1290  job = NULL;
1292  "Done, notifying main thread through pipe!\n");
1293  GNUNET_DISK_file_write (fh, "!", 1);
1294  }
1295  return NULL;
1296 }
1297 
1298 
1299 static void
1301 {
1303  "Received wake up notification through pipe, checking results\n");
1305 }
1306 
1307 
1315 static void
1316 run (void *cls,
1317  const struct GNUNET_CONFIGURATION_Handle *c,
1319 {
1320  unsigned long long max_parallel_bg_queries = 128;
1321 
1322  (void) cls;
1323  (void) service;
1324  last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
1329  if (NULL == namestore_handle)
1330  {
1332  _ ("Failed to connect to the namestore!\n"));
1334  return;
1335  }
1337  "namecache",
1338  "DISABLE");
1340  {
1342  if (NULL == namecache)
1343  {
1345  _ ("Failed to connect to the namecache!\n"));
1347  return;
1348  }
1349  }
1351  "namestore",
1352  "CACHE_KEYS");
1354  if (GNUNET_OK ==
1356  "zonemaster",
1357  "ZONE_PUBLISH_TIME_WINDOW",
1359  {
1361  "Time window for zone iteration: %s\n",
1364  GNUNET_YES));
1365  }
1367  if (GNUNET_OK ==
1369  "zonemaster",
1370  "MAX_PARALLEL_BACKGROUND_QUERIES",
1371  &max_parallel_bg_queries))
1372  {
1374  "Number of allowed parallel background queries: %llu\n",
1375  max_parallel_bg_queries);
1376  }
1377  if (0 == max_parallel_bg_queries)
1378  max_parallel_bg_queries = 1;
1380  (unsigned int) max_parallel_bg_queries);
1381  if (NULL == dht_handle)
1382  {
1384  _ ("Could not connect to DHT!\n"));
1386  NULL);
1387  return;
1388  }
1389 
1390  /* Schedule periodic put for our records. */
1392  statistics = GNUNET_STATISTICS_create ("zonemaster",
1393  c);
1395  "Target zone iteration velocity (μs)",
1397  GNUNET_NO);
1399  NULL);
1401  NULL,
1402  GNUNET_NO,
1404  NULL,
1406  NULL,
1407  NULL /* sync_cb */,
1408  NULL,
1412  GNUNET_break (NULL != zmon);
1413 
1415  NULL);
1416 
1418  const struct GNUNET_DISK_FileHandle *np_fh = GNUNET_DISK_pipe_handle (
1422  np_fh,
1423  notification_pipe_cb, NULL);
1424 
1425  long long unsigned int worker_count = 1;
1426  if (GNUNET_OK !=
1428  "zonemaster",
1429  "WORKER_COUNT",
1430  &worker_count))
1431  {
1433  "Number of workers not defined falling back to 1\n");
1434  }
1435  worker = GNUNET_malloc (sizeof (pthread_t) * worker_count);
1437  for (int i = 0; i < worker_count; i++)
1438  {
1439  if (0 !=
1440  pthread_create (&worker[i],
1441  NULL,
1442  &sign_worker,
1443  NULL))
1444  {
1446  "pthread_create");
1448  }
1449  }
1450 }
1451 
1452 
1457  ("zonemaster",
1459  &run,
1460  NULL,
1461  NULL,
1462  NULL,
1464 
1465 
1466 /* end of gnunet-service-zonemaster.c */
@ GNUNET_BLOCK_TYPE_GNS_NAMERECORD
Block for storing GNS record data.
static struct GNUNET_SCHEDULER_Task * job
Task for main job.
Definition: gnunet-cadet.c:137
struct GNUNET_HashCode key
The key used in the DHT.
static char * expire
DID Document expiration Date Attribut String.
Definition: gnunet-did.c:101
static char * zone
Name of the zone being managed.
static unsigned int rd_count
Number of records for currently parsed set.
static struct GNUNET_GNSRECORD_Data rd[50]
The record data under a single label.
static struct GNUNET_TIME_Relative delay
When should dkg communication start?
static struct GNUNET_SERVICE_Handle * service
Handle to our service instance.
static char buf[2048]
static pthread_mutex_t sign_results_lock
Lock for the DHT put jobs queue.
#define PUBLISH_OPS_PER_EXPIRATION
How often should we (re)publish each record before it expires?
static int disable_namecache
Use the namecache? Doing so creates additional cryptographic operations whenever we touch a record.
static struct CacheOperation * cop_head
Head of cop DLL.
static struct RecordPublicationJob * sign_results_tail
See above.
static void zone_iteration_finished(void *cls)
Zone iteration is completed.
static unsigned int ns_iteration_left
How many more values are left for the current query before we need to explicitly ask the namestore fo...
static void notification_pipe_cb(void *cls)
static void refresh_block(const struct GNUNET_GNSRECORD_Block *block)
Refresh the (encrypted) block in the namecache.
static struct GNUNET_TIME_Relative target_iteration_velocity_per_record
What is the frequency at which we currently would like to perform DHT puts (per record)?...
static unsigned long long put_cnt
Number of successful put operations performed in the current measurement cycle (as measured in check_...
static struct GNUNET_NAMECACHE_Handle * namecache
Our handle to the namecache service.
static unsigned long long last_num_public_records
Last seen record count.
static void dispatch_job_monitor(const struct GNUNET_IDENTITY_PrivateKey *key, const char *label, const struct GNUNET_GNSRECORD_Data *rd, unsigned int rd_count, struct GNUNET_TIME_Absolute expire)
Store GNS records in the DHT.
static unsigned int job_queue_length
Number of entries in the job queue #jobs_head.
static void check_zone_namestore_next()
Check if the current zone iteration needs to be continued by calling publish_zone_namestore_next(),...
static struct RecordPublicationJob * sign_results_head
The DLL for workers to place jobs that are signed.
static struct GNUNET_TIME_Relative sub_delta
By how much should we try to increase our per-record iteration speed (over the desired speed calculat...
static struct GNUNET_TIME_Relative zone_publish_time_window
Time window for zone iteration, adjusted based on relative record expiration times in our zone.
#define DELTA_INTERVAL
How often do we measure the delta between desired zone iteration speed and actual speed,...
static void initiate_put_from_pipe_trigger(void *cls)
static struct RecordPublicationJob * dht_jobs_tail
See above.
static void handle_monitor_error(void *cls)
The zone monitor encountered an IPC error trying to to get in sync.
GNUNET_SERVICE_MAIN("zonemaster", GNUNET_SERVICE_OPTION_NONE, &run, NULL, NULL, NULL, GNUNET_MQ_handler_end())
Define "main" method using service macro.
static void calculate_put_interval()
Calculate target_iteration_velocity_per_record.
#define NAMESTORE_MONITOR_QUEUE_LIMIT
How many events may the namestore give us before it has to wait for us to keep up?
static void publish_zone_namestore_next(void *cls)
Method called periodically that triggers iteration over authoritative records.
static struct CacheOperation * cop_tail
Tail of cop DLL.
static void finish_cache_operation(void *cls, int32_t success, const char *emsg)
Cache operation complete, clean up.
static void dispatch_job(const struct GNUNET_IDENTITY_PrivateKey *key, const char *label, const struct GNUNET_GNSRECORD_Data *rd, unsigned int rd_count, const struct GNUNET_TIME_Absolute expire)
Store GNS records in the DHT.
static struct GNUNET_NAMESTORE_Handle * namestore_handle
Our handle to the namestore service.
static int in_shutdown
For threads to know we are shutting down.
static void handle_monitor_event(void *cls, const struct GNUNET_IDENTITY_PrivateKey *zone, const char *label, unsigned int rd_count, const struct GNUNET_GNSRECORD_Data *rd, struct GNUNET_TIME_Absolute expire)
Process a record that was stored in the namestore (invoked by the monitor).
static void shutdown_task(void *cls)
Task run during shutdown.
#define NS_BLOCK_SIZE
How many records do we fetch in one shot from the namestore?
static struct GNUNET_TIME_Relative min_relative_record_time
Minimum relative expiration time of records seem during the current zone iteration.
static struct GNUNET_STATISTICS_Handle * statistics
Handle to the statistics service.
static struct GNUNET_DISK_PipeHandle * notification_pipe
Our notification pipe.
static struct GNUNET_DHT_Handle * dht_handle
Our handle to the DHT.
#define LATE_ITERATION_SPEEDUP_FACTOR
The factor the current zone iteration interval is divided by for each additional new record.
static int monitor_halted
Monitor halted?
static void publish_zone_dht_start(void *cls)
Periodically iterate over our zone and store everything in dht.
static void run(void *cls, const struct GNUNET_CONFIGURATION_Handle *c, struct GNUNET_SERVICE_Handle *service)
Perform zonemaster duties: watch namestore, publish records.
#define JOB_QUEUE_LIMIT
How many open jobs (and with it maximum amount of pending DHT operations) do we allow at most.
static void * sign_worker(void *cls)
static unsigned long long num_public_records
Useful for zone update for DHT put.
static struct RecordPublicationJob * dht_jobs_head
The DLL for jobs currently in the process of being dispatched into the DHT.
#define INITIAL_ZONE_ITERATION_INTERVAL
The initial interval in milliseconds btween puts in a zone iteration.
static int first_zone_iteration
GNUNET_YES if zone has never been published before
static int cache_keys
Optimize block insertion by caching map of private keys to public keys in memory?
static struct GNUNET_TIME_Absolute last_put_100
When did we last start measuring the DELTA_INTERVAL successful DHT puts? Used for velocity calculatio...
static struct GNUNET_SCHEDULER_Task * zone_publish_task
zone publish task
static struct GNUNET_SCHEDULER_Task * pipe_read_task
Pipe read task.
static struct GNUNET_NAMESTORE_ZoneIterator * namestore_iter
Handle to iterate over our authoritative zone in namestore.
static void free_job(struct RecordPublicationJob *job)
static struct GNUNET_TIME_Relative zone_publish_time_window_default
Default time window for zone iteration.
#define DHT_GNS_REPLICATION_LEVEL
What replication level do we use for DHT PUT operations?
static pthread_t * worker
Our workers.
static void handle_record(void *cls, const struct GNUNET_IDENTITY_PrivateKey *key, const char *label, unsigned int rd_count, const struct GNUNET_GNSRECORD_Data *rd, struct GNUNET_TIME_Absolute expire)
Function used to put all records successively into the DHT.
static struct RecordPublicationJob * sign_jobs_head
The DLL for workers to retrieve open jobs that require signing of blocks.
static struct GNUNET_TIME_Relative last_min_relative_record_time
Minimum relative expiration time of records seem during the last zone iteration.
static void update_velocity(unsigned int cnt)
Re-calculate our velocity and the desired velocity.
static pthread_cond_t sign_jobs_cond
Wait condition on new sign jobs.
static void dht_put_continuation(void *cls)
Continuation called from DHT once the PUT operation is done.
static struct GNUNET_NAMESTORE_ZoneMonitor * zmon
Handle to monitor namestore changes to instant propagation.
static struct RecordPublicationJob * sign_jobs_tail
See above.
#define MAXIMUM_ZONE_ITERATION_INTERVAL
The upper bound for the zone iteration interval (per record).
static pthread_mutex_t sign_jobs_lock
Lock for the sign jobs queue.
static void zone_iteration_error(void *cls)
We encountered an error in our zone iteration.
static struct GNUNET_DISK_FileHandle * fh
File handle to STDIN, for reading restart/quit commands.
API to the DHT service.
API that can be used to store naming information on a GNUnet node.
API that can be used to store naming information on a GNUnet node;.
API to create, modify and access statistics.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
void GNUNET_DHT_put_cancel(struct GNUNET_DHT_PutHandle *ph)
Cancels a DHT PUT operation.
Definition: dht_api.c:1148
struct GNUNET_DHT_Handle * GNUNET_DHT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int ht_len)
Initialize the connection with the DHT service.
Definition: dht_api.c:1039
#define GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY
Default republication frequency for stored data in the DHT.
struct GNUNET_DHT_PutHandle * GNUNET_DHT_put(struct GNUNET_DHT_Handle *handle, const struct GNUNET_HashCode *key, uint32_t desired_replication_level, enum GNUNET_DHT_RouteOption options, enum GNUNET_BLOCK_Type type, size_t size, const void *data, struct GNUNET_TIME_Absolute exp, GNUNET_SCHEDULER_TaskCallback cont, void *cont_cls)
Perform a PUT operation storing data in the DHT.
Definition: dht_api.c:1090
void GNUNET_DHT_disconnect(struct GNUNET_DHT_Handle *handle)
Shutdown connection with the DHT service.
Definition: dht_api.c:1062
@ GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE
Each peer along the way should process the request (otherwise only peers locally closest to the key w...
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:685
enum GNUNET_GenericReturnValue GNUNET_DISK_pipe_close(struct GNUNET_DISK_PipeHandle *p)
Closes an interprocess channel.
Definition: disk.c:1584
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:621
const struct GNUNET_DISK_FileHandle * GNUNET_DISK_pipe_handle(const struct GNUNET_DISK_PipeHandle *p, enum GNUNET_DISK_PipeEnd n)
Get the handle to a particular pipe end.
Definition: disk.c:1615
struct GNUNET_DISK_PipeHandle * GNUNET_DISK_pipe(enum GNUNET_DISK_PipeFlags pf)
Creates an interprocess channel.
Definition: disk.c:1441
@ GNUNET_DISK_PF_NONE
No special options, use non-blocking read/write operations.
@ GNUNET_DISK_PIPE_END_WRITE
The writing-end of a pipe.
@ GNUNET_DISK_PIPE_END_READ
The reading-end of a pipe.
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
void GNUNET_GNSRECORD_query_from_private_key(const struct GNUNET_IDENTITY_PrivateKey *zone, const char *label, struct GNUNET_HashCode *query)
Calculate the DHT query for a given label in a given zone.
@ GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION
This expiration time of the record is a relative time (not an absolute time).
enum GNUNET_GenericReturnValue GNUNET_GNSRECORD_block_sign(const struct GNUNET_IDENTITY_PrivateKey *key, const char *label, struct GNUNET_GNSRECORD_Block *block)
Sign a block create with GNUNET_GNSRECORD_block_create_unsigned.
size_t GNUNET_GNSRECORD_block_get_size(const struct GNUNET_GNSRECORD_Block *block)
Returns the length of this block in bytes.
enum GNUNET_GenericReturnValue GNUNET_GNSRECORD_block_create_unsigned(const struct GNUNET_IDENTITY_PrivateKey *pkey, struct GNUNET_TIME_Absolute expire, const char *label, const struct GNUNET_GNSRECORD_Data *rd, unsigned int rd_count, struct GNUNET_GNSRECORD_Block **result)
Create name and records but do not sign! Sign later with GNUNET_GNSRECORD_block_sign().
enum GNUNET_GenericReturnValue GNUNET_GNSRECORD_normalize_record_set(const char *label, const struct GNUNET_GNSRECORD_Data *rd, unsigned int rd_count, struct GNUNET_GNSRECORD_Data *rd_public, unsigned int *rd_count_public, struct GNUNET_TIME_Absolute *expiry, enum GNUNET_GNSRECORD_Filter filter, char **emsg)
Normalize namestore records: Check for consistency and expirations.
@ GNUNET_GNSRECORD_FILTER_NONE
No filter flags set.
@ GNUNET_GNSRECORD_FILTER_OMIT_PRIVATE
Filter private records.
#define GNUNET_log(kind,...)
#define GNUNET_MIN(a, b)
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
#define GNUNET_log_strerror(level, cmd)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_BULK
@ GNUNET_ERROR_TYPE_DEBUG
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
#define GNUNET_MQ_handler_end()
End-marker for the handlers array.
void GNUNET_NAMECACHE_disconnect(struct GNUNET_NAMECACHE_Handle *h)
Disconnect from the namecache service (and free associated resources).
void GNUNET_NAMECACHE_cancel(struct GNUNET_NAMECACHE_QueueEntry *qe)
Cancel a namecache operation.
struct GNUNET_NAMECACHE_Handle * GNUNET_NAMECACHE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the namecache service.
struct GNUNET_NAMECACHE_QueueEntry * GNUNET_NAMECACHE_block_cache(struct GNUNET_NAMECACHE_Handle *h, const struct GNUNET_GNSRECORD_Block *block, GNUNET_NAMECACHE_ContinuationWithStatus cont, void *cont_cls)
Store an item in the namecache.
void GNUNET_NAMESTORE_disconnect(struct GNUNET_NAMESTORE_Handle *h)
Disconnect from the namestore service (and free associated resources).
void GNUNET_NAMESTORE_zone_iterator_next(struct GNUNET_NAMESTORE_ZoneIterator *it, uint64_t limit)
Calls the record processor specified in GNUNET_NAMESTORE_zone_iteration_start for the next record.
void GNUNET_NAMESTORE_zone_monitor_stop(struct GNUNET_NAMESTORE_ZoneMonitor *zm)
Stop monitoring a zone for changes.
struct GNUNET_NAMESTORE_ZoneIterator * GNUNET_NAMESTORE_zone_iteration_start2(struct GNUNET_NAMESTORE_Handle *h, const struct GNUNET_IDENTITY_PrivateKey *zone, GNUNET_SCHEDULER_TaskCallback error_cb, void *error_cb_cls, GNUNET_NAMESTORE_RecordSetMonitor proc, void *proc_cls, GNUNET_SCHEDULER_TaskCallback finish_cb, void *finish_cb_cls, enum GNUNET_GNSRECORD_Filter filter)
Starts a new zone iteration (used to periodically PUT all of our records into our DHT).
struct GNUNET_NAMESTORE_ZoneMonitor * GNUNET_NAMESTORE_zone_monitor_start2(const struct GNUNET_CONFIGURATION_Handle *cfg, const struct GNUNET_IDENTITY_PrivateKey *zone, int iterate_first, GNUNET_SCHEDULER_TaskCallback error_cb, void *error_cb_cls, GNUNET_NAMESTORE_RecordSetMonitor monitor, void *monitor_cls, GNUNET_SCHEDULER_TaskCallback sync_cb, void *sync_cb_cls, enum GNUNET_GNSRECORD_Filter filter)
Begin monitoring a zone for changes.
struct GNUNET_NAMESTORE_Handle * GNUNET_NAMESTORE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the namestore service.
void GNUNET_NAMESTORE_zone_monitor_next(struct GNUNET_NAMESTORE_ZoneMonitor *zm, uint64_t limit)
Calls the monitor processor specified in GNUNET_NAMESTORE_zone_monitor_start for the next record(s).
void GNUNET_NAMESTORE_zone_iteration_stop(struct GNUNET_NAMESTORE_ZoneIterator *it)
Stops iteration and releases the namestore handle for further calls.
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:534
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1268
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_read_file(struct GNUNET_TIME_Relative delay, const struct GNUNET_DISK_FileHandle *rfd, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay or when the specified file descriptor is ready f...
Definition: scheduler.c:1625
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received,...
Definition: scheduler.c:1303
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:944
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1241
@ GNUNET_SERVICE_OPTION_NONE
Use defaults.
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_set(struct GNUNET_STATISTICS_Handle *handle, const char *name, uint64_t value, int make_persistent)
Set statistic value for the peer.
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
struct GNUNET_TIME_Relative GNUNET_TIME_relative_min(struct GNUNET_TIME_Relative t1, struct GNUNET_TIME_Relative t2)
Return the minimum of two relative time values.
Definition: time.c:343
#define GNUNET_TIME_UNIT_FOREVER_REL
Constant used to specify "forever".
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:436
struct GNUNET_TIME_Relative GNUNET_TIME_relative_subtract(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Subtract relative timestamp from the other.
Definition: time.c:603
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:111
const char * GNUNET_STRINGS_absolute_time_to_string(struct GNUNET_TIME_Absolute t)
Like asctime, except for GNUnet time.
Definition: strings.c:616
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:585
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:484
#define GNUNET_TIME_UNIT_ZERO
Relative time zero.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_divide(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Divide relative time by a given factor.
Definition: time.c:550
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:569
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
static struct GNUNET_TIME_Relative delta
Definition: speedup.c:36
Pending operation on the namecache.
struct CacheOperation * next
Kept in a DLL.
struct GNUNET_NAMECACHE_QueueEntry * qe
Handle to namecache queue.
struct CacheOperation * prev
Kept in a DLL.
Connection to the DHT service.
Definition: dht_api.c:237
Handle to a PUT request.
Definition: dht_api.c:45
Handle used to access files (and pipes).
Handle used to manage a pipe.
Definition: disk.c:68
A 512-bit hashcode.
A private key for an identity as per LSD0001.
Connection to the NAMECACHE service.
Definition: namecache_api.c:93
An QueueEntry used to store information for a pending NAMECACHE record operation.
Definition: namecache_api.c:46
Connection to the NAMESTORE service.
Handle for a zone iterator operation.
Handle for a monitoring activity.
Entry in list of pending tasks.
Definition: scheduler.c:136
Handle to a service.
Definition: service.c:118
Handle for the service.
Time for absolute times used by GNUnet, in microseconds.
Time for relative time used by GNUnet, in microseconds.
uint64_t rel_value_us
The actual value.
struct RecordPublicationJob * next
DLL.
struct GNUNET_DHT_PutHandle * ph
Handle for the DHT PUT operation.
struct GNUNET_IDENTITY_PrivateKey zone
The zone key to sign the block with.
struct GNUNET_TIME_Absolute start_date
When was this PUT initiated?
struct RecordPublicationJob * prev
DLL.
struct GNUNET_GNSRECORD_Block * block
The block to sign.
char * label
The label of the block needed for signing.
size_t block_size
The size of the public block for the DHT put.
struct GNUNET_TIME_Absolute expire_pub
The expiration time of the public block for the DHT put.
struct GNUNET_GNSRECORD_Block * block_priv
The private block to sign, may point to block in case the public and private blocks are the same.