GNUnet  0.11.x
gnunet-regex-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011 - 2017 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
27 #include "platform.h"
28 #include "gnunet_applications.h"
29 #include "gnunet_util_lib.h"
30 #include "regex_internal_lib.h"
31 #include "gnunet_arm_service.h"
32 #include "gnunet_dht_service.h"
33 #include "gnunet_testbed_service.h"
34 
35 #define FIND_TIMEOUT \
36  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90)
37 
42 {
47 
51  void *cls;
52 
56  struct DLLOperation *next;
57 
61  struct DLLOperation *prev;
62 };
63 
64 
68 enum State
69 {
74 
79 
84 
89 
94 
99 
105 };
106 
107 
111 struct RegexPeer
112 {
116  unsigned int id;
117 
122 
127 
131  const char *search_str;
132 
140 
145 
150 
155 
160 
165 
170 
175 };
176 
180 static int in_shutdown;
181 
185 static struct RegexPeer *peers;
186 
191 
196 
201 
206 
211 
216 
220 static uint64_t event_mask;
221 
226 
230 static struct GNUNET_TIME_Relative prof_time;
231 
235 static unsigned int num_peers;
236 
240 static int result;
241 
245 enum State state;
246 
250 static char *policy_dir;
251 
255 static char *hosts_file;
256 
260 static char *strings_file;
261 
265 static char **search_strings;
266 
270 static long long unsigned int init_parallel_searches;
271 
275 static unsigned int parallel_searches;
276 
280 static unsigned int strings_found;
281 
285 static unsigned int next_search;
286 
291 
295 static struct GNUNET_TIME_Relative search_timeout_time = { 60000 };
296 
301 
305 static char *data_filename;
306 
311 static char *regex_prefix;
312 
317 
318 
319 /******************************************************************************/
320 /****************************** DECLARATIONS ********************************/
321 /******************************************************************************/
322 
331 static void
332 dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
333  void *ca_result, const char *emsg);
334 
343 static void *
344 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg);
345 
346 
354 static void
355 dht_da (void *cls, void *op_result);
356 
357 
367 static void
368 stats_connect_cb (void *cls,
370  void *ca_result,
371  const char *emsg);
372 
373 
379 static void
380 announce_next_regex (void *cls);
381 
382 
383 /******************************************************************************/
384 /******************************** SHUTDOWN **********************************/
385 /******************************************************************************/
386 
387 
393 static void
394 do_shutdown (void *cls)
395 {
396  struct RegexPeer *peer;
397  unsigned int peer_cnt;
398  unsigned int search_str_cnt;
399  char output_buffer[512];
400  size_t size;
401 
402  if (NULL != abort_task)
403  {
405  abort_task = NULL;
406  }
407  if (NULL != register_hosts_task)
408  {
410  register_hosts_task = NULL;
411  }
412  for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++)
413  {
414  peer = &peers[peer_cnt];
415 
416  if ((GNUNET_YES != peer->search_str_matched) && (NULL != data_file) )
417  {
418  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
419  size =
420  GNUNET_snprintf (output_buffer,
421  sizeof(output_buffer),
422  "%p Search string not found: %s (%d)\n"
423  "%p On peer: %u (%p)\n"
424  "%p After: %s\n",
425  peer, peer->search_str, peer->search_str_matched,
426  peer, peer->id, peer,
427  peer,
429  GNUNET_NO));
430  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
431  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
432  }
433 
434  if (NULL != peers[peer_cnt].op_handle)
436  }
437 
438  if (NULL != data_file)
439  {
441  data_file = NULL;
442  }
443  for (search_str_cnt = 0;
444  search_str_cnt < num_peers && NULL != search_strings;
445  search_str_cnt++)
446  {
447  GNUNET_free (search_strings[search_str_cnt]);
448  }
450  search_strings = NULL;
451 
452  if (NULL != reg_handle)
453  {
455  reg_handle = NULL;
456  }
457  if (NULL != mc)
458  {
460  mc = NULL;
461  }
462  if (NULL != mc_proc)
463  {
465  mc_proc = NULL;
466  }
467  if (NULL != cfg)
468  {
470  cfg = NULL;
471  }
472 }
473 
474 
480 static void
481 do_abort (void *cls)
482 {
483  unsigned long i = (unsigned long) cls;
484 
486  "Aborting from line %lu...\n", i);
487  abort_task = NULL;
490 }
491 
492 
493 /******************************************************************************/
494 /********************* STATISTICS SERVICE CONNECTIONS ***********************/
495 /******************************************************************************/
496 
507 static void *
508 stats_ca (void *cls,
509  const struct GNUNET_CONFIGURATION_Handle *cfg)
510 {
511  return GNUNET_STATISTICS_create ("<driver>", cfg);
512 }
513 
514 
522 static void
523 stats_da (void *cls, void *op_result)
524 {
525  struct RegexPeer *peer = cls;
526 
527  GNUNET_assert (op_result == peer->stats_handle);
528 
529  GNUNET_STATISTICS_destroy (peer->stats_handle, GNUNET_NO);
530  peer->stats_handle = NULL;
531 }
532 
533 
544 static int
545 stats_iterator (void *cls,
546  const char *subsystem,
547  const char *name,
548  uint64_t value, int is_persistent)
549 {
550  struct RegexPeer *peer = cls;
551  char output_buffer[512];
552  size_t size;
553 
554  if (NULL == data_file)
555  {
557  "%p -> %s [%s]: %llu\n",
558  peer,
559  subsystem,
560  name,
561  (unsigned long long) value);
562  return GNUNET_OK;
563  }
564  size =
565  GNUNET_snprintf (output_buffer,
566  sizeof(output_buffer),
567  "%p [%s] %llu %s\n",
568  peer,
569  subsystem,
570  (unsigned long long) value,
571  name);
572  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
574  "Unable to write to file!\n");
575 
576  return GNUNET_OK;
577 }
578 
579 
588 static void
589 stats_cb (void *cls,
590  int success)
591 {
592  static unsigned int peer_cnt;
593  struct RegexPeer *peer = cls;
594 
595  if (GNUNET_OK != success)
596  {
598  "Getting statistics for peer %u failed!\n",
599  peer->id);
600  return;
601  }
602 
603  GNUNET_assert (NULL != peer->op_handle);
604 
605  GNUNET_TESTBED_operation_done (peer->op_handle);
606  peer->op_handle = NULL;
607 
608  peer_cnt++;
609  peer = &peers[peer_cnt];
610 
611  fprintf (stderr, "s");
612  if (peer_cnt == num_peers)
613  {
615  "\nCollecting stats finished. Shutting down.\n");
617  result = GNUNET_OK;
618  }
619  else
620  {
621  peer->op_handle =
623  peer->peer_handle,
624  "statistics",
626  peer,
627  &stats_ca,
628  &stats_da,
629  peer);
630  }
631 }
632 
633 
643 static void
644 stats_connect_cb (void *cls,
646  void *ca_result,
647  const char *emsg)
648 {
649  struct RegexPeer *peer = cls;
650 
651  if ((NULL == ca_result) || (NULL != emsg))
652  {
654  "Failed to connect to statistics service on peer %u: %s\n",
655  peer->id, emsg);
656 
657  peer->stats_handle = NULL;
658  return;
659  }
660 
661  peer->stats_handle = ca_result;
662 
663  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
664  &stats_cb,
665  &stats_iterator, peer))
666  {
668  "Could not get statistics of peer %u!\n", peer->id);
669  }
670 }
671 
672 
679 static void
680 do_collect_stats (void *cls)
681 {
682  struct RegexPeer *peer = &peers[0];
683 
684  GNUNET_assert (NULL != peer->peer_handle);
685 
686  peer->op_handle =
688  peer->peer_handle,
689  "statistics",
691  peer,
692  &stats_ca,
693  &stats_da,
694  peer);
695 }
696 
697 
698 /******************************************************************************/
699 /************************ REGEX FIND CONNECTIONS **************************/
700 /******************************************************************************/
701 
702 
708 static void
709 find_string (void *cls);
710 
711 
723 static void
725  const struct GNUNET_PeerIdentity *id,
726  const struct GNUNET_DHT_PathElement *get_path,
727  unsigned int get_path_length,
728  const struct GNUNET_DHT_PathElement *put_path,
729  unsigned int put_path_length)
730 {
731  struct RegexPeer *peer = cls;
732  char output_buffer[512];
733  size_t size;
734 
735  if (GNUNET_YES == peer->search_str_matched)
736  {
738  "String %s on peer %u already matched!\n",
739  peer->search_str, peer->id);
740  return;
741  }
742 
743  strings_found++;
745 
746  if (NULL != peer->timeout)
747  {
748  GNUNET_SCHEDULER_cancel (peer->timeout);
749  peer->timeout = NULL;
750  if (GNUNET_NO == in_shutdown)
752  }
753 
754  if (NULL == id)
755  {
756  // FIXME not possible right now
758  "String matching timed out for string %s on peer %u (%i/%i)\n",
759  peer->search_str, peer->id, strings_found, num_peers);
760  peer->search_str_matched = GNUNET_SYSERR;
761  }
762  else
763  {
764  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
765 
767  "String %s found on peer %u after %s (%i/%i) (%u||)\n",
768  peer->search_str, peer->id,
771 
772  peer->search_str_matched = GNUNET_YES;
773 
774  if (NULL != data_file)
775  {
776  size =
777  GNUNET_snprintf (output_buffer,
778  sizeof(output_buffer),
779  "%p Peer: %u\n"
780  "%p Search string: %s\n"
781  "%p Search duration: %s\n\n",
782  peer, peer->id,
783  peer, peer->search_str,
784  peer,
786  GNUNET_NO));
787 
788  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
789  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
790  }
791  }
792 
793  GNUNET_TESTBED_operation_done (peer->op_handle);
794  peer->op_handle = NULL;
795 
796  if (strings_found == num_peers)
797  {
800  "All strings successfully matched in %s\n",
802 
803  if (NULL != search_timeout_task)
804  {
806  search_timeout_task = NULL;
807  }
808 
809  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n");
811  }
812 }
813 
814 
821 static void
822 search_timed_out (void *cls)
823 {
824  unsigned int i;
825 
827  "Finding matches to all strings did not succeed after %s.\n",
829  GNUNET_NO));
831  "Found %i of %i strings\n", strings_found, num_peers);
832 
834  "Search timed out after %s."
835  "Collecting stats and shutting down.\n",
837  GNUNET_NO));
838 
840  for (i = 0; i < num_peers; i++)
841  {
842  if (NULL != peers[i].op_handle)
843  {
845  peers[i].op_handle = NULL;
846  }
847  }
849 }
850 
851 
858 static void
859 find_timed_out (void *cls)
860 {
861  struct RegexPeer *p = cls;
862 
863  p->timeout = NULL;
865  "Searching for string \"%s\" on peer %d timed out.\n",
866  p->search_str,
867  p->id);
868  if (GNUNET_NO == in_shutdown)
870 }
871 
872 
878 static void
879 find_string (void *cls)
880 {
881  unsigned int search_peer = (unsigned int) (long) cls;
882 
883  if ((search_peer >= num_peers) ||
884  (GNUNET_YES == in_shutdown))
885  return;
886 
888  "Searching for string \"%s\" on peer %d (%u||)\n",
889  peers[search_peer].search_str,
890  search_peer,
892 
893  peers[search_peer].op_handle =
895  peers[search_peer].peer_handle,
896  "dht",
898  &peers[search_peer],
899  &dht_ca,
900  &dht_da,
901  &peers[search_peer]);
902  GNUNET_assert (NULL != peers[search_peer].op_handle);
903  peers[search_peer].timeout
906  &peers[search_peer]);
907 }
908 
909 
917 static void
918 daemon_started (void *cls,
920  const char *emsg)
921 {
922  struct RegexPeer *peer = (struct RegexPeer *) cls;
923  unsigned long search_peer;
924  unsigned int i;
925 
926  GNUNET_TESTBED_operation_done (peer->daemon_op);
927  peer->daemon_op = NULL;
928  if (NULL != emsg)
929  {
931  "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg);
932  GNUNET_assert (0);
933  }
934  else
935  {
937  "Daemon %u started successfully\n", peer->id);
938  }
939 
940  /* Find a peer to look for a string matching the regex announced */
942  num_peers);
943  for (i = 0; peers[search_peer].search_str != NULL; i++)
944  {
945  search_peer = (search_peer + 1) % num_peers;
946  if (i > num_peers)
947  GNUNET_assert (0); /* we ran out of peers, must be a bug */
948  }
949  peers[search_peer].search_str = search_strings[peer->id];
950  peers[search_peer].search_str_matched = GNUNET_NO;
953  2),
954  &find_string,
955  (void *) search_peer);
956 }
957 
958 
966 static void
967 do_announce (void *cls)
968 {
969  unsigned int i;
970 
971  if (GNUNET_YES == in_shutdown)
972  return;
974  "Starting announce.\n");
975  for (i = 0; i < init_parallel_searches; i++)
976  {
978  " scheduling announce %u\n",
979  i);
981  }
982 }
983 
984 
990 static void
992 {
993  struct RegexPeer *peer;
994 
995  if (GNUNET_YES == in_shutdown)
996  return;
997  if (next_search >= num_peers)
998  {
999  if (strings_found != num_peers)
1000  {
1001  struct GNUNET_TIME_Relative new_delay;
1002  if (NULL != search_timeout_task)
1007  NULL);
1008  }
1009  return;
1010  }
1011 
1012  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search);
1013  peer = &peers[next_search];
1014  peer->daemon_op =
1016  peer->peer_handle,
1017  "regexprofiler",
1018  &daemon_started,
1019  peer,
1020  1);
1021  next_search++;
1023 }
1024 
1025 
1036 static void
1037 dht_connect_cb (void *cls,
1038  struct GNUNET_TESTBED_Operation *op,
1039  void *ca_result,
1040  const char *emsg)
1041 {
1042  struct RegexPeer *peer = (struct RegexPeer *) cls;
1043 
1044  if ((NULL != emsg) || (NULL == op) || (NULL == ca_result))
1045  {
1046  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg);
1047  GNUNET_assert (0);
1048  }
1049 
1050  GNUNET_assert (NULL != peer->dht_handle);
1051  GNUNET_assert (peer->op_handle == op);
1052  GNUNET_assert (peer->dht_handle == ca_result);
1053 
1054  peer->search_str_matched = GNUNET_NO;
1055  peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle,
1056  peer->search_str,
1058  NULL);
1059  peer->prof_start_time = GNUNET_TIME_absolute_get ();
1060 }
1061 
1062 
1071 static void *
1072 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
1073 {
1074  struct RegexPeer *peer = cls;
1075 
1076  peer->dht_handle = GNUNET_DHT_connect (cfg, 32);
1077 
1078  return peer->dht_handle;
1079 }
1080 
1081 
1088 static void
1089 dht_da (void *cls, void *op_result)
1090 {
1091  struct RegexPeer *peer = (struct RegexPeer *) cls;
1092 
1093  GNUNET_assert (peer->dht_handle == op_result);
1094 
1095  if (NULL != peer->search_handle)
1096  {
1097  REGEX_INTERNAL_search_cancel (peer->search_handle);
1098  peer->search_handle = NULL;
1099  }
1100 
1101  if (NULL != peer->dht_handle)
1102  {
1103  GNUNET_DHT_disconnect (peer->dht_handle);
1104  peer->dht_handle = NULL;
1105  }
1106 }
1107 
1108 
1122 static void
1123 test_master (void *cls,
1124  struct GNUNET_TESTBED_RunHandle *h,
1125  unsigned int num_peers_,
1127  unsigned int links_succeeded,
1128  unsigned int links_failed)
1129 {
1130  unsigned int i;
1131 
1132  GNUNET_assert (num_peers_ == num_peers);
1133 
1136  "Testbed started in %s\n",
1138 
1139  if (NULL != abort_task)
1140  {
1142  abort_task = NULL;
1143  }
1144 
1145  for (i = 0; i < num_peers; i++)
1146  {
1148  }
1149  if (GNUNET_NO ==
1150  GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT"))
1151  {
1152  struct GNUNET_TIME_Relative settle_time;
1153 
1154  settle_time =
1156  10 * num_peers);
1158  "Waiting for DHT for %s to settle new connections.\n\n",
1160  GNUNET_NO));
1161  GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL);
1162  }
1163  else
1164  {
1166  }
1169 }
1170 
1171 
1178 static void
1180  const struct GNUNET_TESTBED_EventInformation *event)
1181 {
1182  switch (event->type)
1183  {
1185  printf (".");
1186  break;
1187 
1189  printf ("#");
1190  break;
1191 
1192  default:
1193  break;
1194  }
1195  fflush (stdout);
1196 }
1197 
1198 
1199 /******************************************************************************/
1200 /*************************** TESTBED PEER SETUP *****************************/
1201 /******************************************************************************/
1202 
1213 static int
1215  uint64_t data_size,
1216  unsigned int str_max)
1217 {
1218  char *buf; // Keep track of last string to skip blank lines
1219  unsigned int offset;
1220  unsigned int str_cnt;
1221 
1222  buf = data;
1223  offset = 0;
1224  str_cnt = 0;
1225  while ((offset < (data_size - 1)) && (str_cnt < str_max))
1226  {
1227  offset++;
1228  if (((data[offset] == '\n')) &&
1229  (buf != &data[offset]))
1230  {
1231  data[offset] = '\0';
1232  str_cnt++;
1233  buf = &data[offset + 1];
1234  }
1235  else if ((data[offset] == '\n') ||
1236  (data[offset] == '\0'))
1237  buf = &data[offset + 1];
1238  }
1239  return str_cnt;
1240 }
1241 
1242 
1256 static int
1258  char ***strings, unsigned int str_cnt)
1259 {
1260  uint64_t offset;
1261  uint64_t len;
1262  unsigned int i;
1263 
1264  *strings = GNUNET_malloc (sizeof(char *) * str_cnt);
1265  offset = 0;
1266  for (i = 0; i < str_cnt; i++)
1267  {
1268  len = strlen (&data[offset]);
1269  if (offset + len >= data_size)
1270  {
1271  GNUNET_free (*strings);
1272  *strings = NULL;
1273  return GNUNET_SYSERR;
1274  }
1275  if (0 == len) // empty line
1276  {
1277  offset++;
1278  i--;
1279  continue;
1280  }
1281 
1282  GNUNET_asprintf (&(*strings)[i],
1283  "%s%s",
1284  regex_prefix,
1285  &data[offset]);
1286  offset += len + 1;
1287  }
1288  return GNUNET_OK;
1289 }
1290 
1291 
1301 static int
1303  char ***strings,
1304  unsigned int limit)
1305 {
1306  char *data;
1307  uint64_t filesize;
1308  int str_cnt;
1309 
1310  /* Sanity checks */
1311  if (NULL == filename)
1312  {
1313  return GNUNET_SYSERR;
1314  }
1316  {
1318  "Could not find search strings file %s\n", filename);
1319  return GNUNET_SYSERR;
1320  }
1321  if (GNUNET_OK !=
1323  &filesize,
1324  GNUNET_YES,
1325  GNUNET_YES))
1326  {
1328  "Search strings file %s cannot be read.\n",
1329  filename);
1330  return GNUNET_SYSERR;
1331  }
1332  if (0 == filesize)
1333  {
1335  "Search strings file %s is empty.\n",
1336  filename);
1337  return GNUNET_SYSERR;
1338  }
1339 
1340  /* Read data into memory */
1341  data = GNUNET_malloc (filesize + 1);
1342  if (filesize != GNUNET_DISK_fn_read (filename,
1343  data,
1344  filesize))
1345  {
1346  GNUNET_free (data);
1348  "Could not read search strings file %s.\n",
1349  filename);
1350  return GNUNET_SYSERR;
1351  }
1352 
1353  /* Process buffer and build array */
1354  str_cnt = count_and_separate_strings (data, filesize, limit);
1355  if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt))
1356  {
1357  str_cnt = GNUNET_SYSERR;
1358  }
1359  GNUNET_free (data);
1360  return str_cnt;
1361 }
1362 
1363 
1372 static void
1373 run (void *cls,
1374  char *const *args,
1375  const char *cfgfile,
1376  const struct GNUNET_CONFIGURATION_Handle *config)
1377 {
1378  unsigned int nsearchstrs;
1379  unsigned int i;
1380  struct GNUNET_TIME_Relative abort_time;
1381 
1383 
1384  /* Check config */
1385  if (NULL == config)
1386  {
1388  _ ("No configuration file given. Exiting\n"));
1390  return;
1391  }
1393  if (GNUNET_OK !=
1394  GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER",
1395  "REGEX_PREFIX",
1396  &regex_prefix))
1397  {
1399  "regexprofiler",
1400  "regex_prefix");
1402  return;
1403  }
1404  if (GNUNET_OK !=
1405  GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER",
1406  "PARALLEL_SEARCHES",
1408  {
1410  "Configuration option \"PARALLEL_SEARCHES\" missing."
1411  " Using default (%d)\n", 10);
1413  }
1414  if (GNUNET_OK !=
1415  GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER",
1416  "REANNOUNCE_PERIOD_MAX",
1418  {
1420  "reannounce_period_max not given. Using 10 minutes.\n");
1423  }
1424 
1425  /* Check arguments */
1426  if (NULL == policy_dir)
1427  {
1429  _ (
1430  "No policy directory specified on command line. Exiting.\n"));
1431  return;
1432  }
1434  {
1436  _ ("Specified policies directory does not exist. Exiting.\n"));
1438  return;
1439  }
1440  if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL,
1441  NULL)))
1442  {
1444  _ ("No files found in `%s'\n"),
1445  policy_dir);
1446  return;
1447  }
1448  GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER",
1449  "POLICY_DIR", policy_dir);
1451  {
1453  _ ("No search strings file given. Exiting.\n"));
1455  return;
1456  }
1457  nsearchstrs = load_search_strings (strings_file,
1458  &search_strings,
1459  num_peers);
1460  if (num_peers != nsearchstrs)
1461  {
1463  "Error loading search strings.\n");
1465  "File (%s) does not contain enough strings (%u/%u).\n",
1466  strings_file, nsearchstrs, num_peers);
1468  return;
1469  }
1470  if ((0 == num_peers) || (NULL == search_strings))
1471  {
1473  _ ("Error loading search strings. Exiting.\n"));
1475  return;
1476  }
1477  for (i = 0; i < num_peers; i++)
1479  "search string: %s\n",
1480  search_strings[i]);
1481 
1482  /* Check logfile */
1483  if ((NULL != data_filename) &&
1484  (NULL == (data_file =
1491  {
1493  "open",
1494  data_filename);
1495  return;
1496  }
1497 
1498  /* Initialize peers */
1499  peers = GNUNET_malloc (sizeof(struct RegexPeer) * num_peers);
1500  for (i = 0; i < num_peers; i++)
1501  peers[i].id = i;
1502 
1504  "TESTBED", "OVERLAY_RANDOM_LINKS",
1505  num_peers * 20);
1507  "DHT", "FORCE_NSE",
1508  (long long unsigned)
1509  (log (num_peers) / log (2.0)));
1510  event_mask = 0LL;
1511 /* For feedback about the start process activate these and pass master_cb */
1513 // event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
1515 // event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);
1518  cfg,
1519  num_peers,
1520  event_mask,
1522  NULL, /* master_controller_cb cls */
1523  &test_master,
1524  NULL); /* test_master cls */
1525  if (GNUNET_OK !=
1527  "SETUP_TIMEOUT",
1528  &abort_time))
1529  {
1531  "SETUP_TIMEOUT not given. Using 15 minutes.\n");
1532  abort_time =
1534  }
1535  abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES);
1536  abort_task =
1537  GNUNET_SCHEDULER_add_delayed (abort_time,
1538  &do_abort,
1539  (void *) __LINE__);
1541  "setup_timeout: %s\n",
1543 }
1544 
1545 
1553 int
1554 main (int argc, char *const *argv)
1555 {
1558  'o',
1559  "output-file",
1560  "FILENAME",
1561  gettext_noop (
1562  "name of the file for writing statistics"),
1563  &data_filename),
1565  't',
1566  "matching-timeout",
1567  "TIMEOUT",
1568  gettext_noop (
1569  "wait TIMEOUT before ending the experiment"),
1572  'p',
1573  "policy-dir",
1574  "DIRECTORY",
1575  gettext_noop ("directory with policy files"),
1576  &policy_dir),
1578  's',
1579  "strings-file",
1580  "FILENAME",
1581  gettext_noop (
1582  "name of file with input strings"),
1583  &strings_file),
1585  'H',
1586  "hosts-file",
1587  "FILENAME",
1588  gettext_noop (
1589  "name of file with hosts' names"),
1590  &hosts_file),
1591 
1593  };
1594  int ret;
1595 
1596  if (GNUNET_OK !=
1597  GNUNET_STRINGS_get_utf8_args (argc, argv,
1598  &argc, &argv))
1599  return 2;
1601  ret =
1602  GNUNET_PROGRAM_run (argc, argv,
1603  "gnunet-regex-profiler",
1604  _ ("Profiler for regex"),
1605  options,
1606  &run, NULL);
1607  if (GNUNET_OK != ret)
1608  return ret;
1609  if (GNUNET_OK != result)
1610  return 1;
1611  return 0;
1612 }
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
#define gettext_noop(String)
Definition: gettext.h:69
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:144
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
static char * filename
uint32_t data
The data value.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
static char * value
Value of the record to add/remove.
enum State state
current state of profiling
static unsigned int next_search
Index of peer to start next announce/search.
static char * data_filename
Filename to log statistics to.
static struct GNUNET_SCHEDULER_Task * abort_task
Abort task identifier.
static long long unsigned int init_parallel_searches
How many searches are we going to start in parallel.
static void stats_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
Function called by testbed once we are connected to stats service.
static int stats_iterator(void *cls, const char *subsystem, const char *name, uint64_t value, int is_persistent)
Process statistic values.
static int count_and_separate_strings(char *data, uint64_t data_size, unsigned int str_max)
Process the text buffer counting the non-empty lines and separating them with NULL characters,...
static void stats_cb(void *cls, int success)
Stats callback.
static struct GNUNET_TESTBED_Controller * mc
Handle to the master controller.
static struct GNUNET_TIME_Relative reannounce_period_max
What's the maximum regex reannounce period.
static char * policy_dir
Folder where policy files are stored.
static unsigned int parallel_searches
How many searches are running in parallel.
static struct GNUNET_TIME_Relative prof_time
Duration profiling step has taken.
static void stats_da(void *cls, void *op_result)
Adapter function called to destroy a connection to statistics service.
static void test_master(void *cls, struct GNUNET_TESTBED_RunHandle *h, unsigned int num_peers_, struct GNUNET_TESTBED_Peer **testbed_peers, unsigned int links_succeeded, unsigned int links_failed)
Signature of a main function for a testcase.
State
Available states during profiling.
@ STATE_PEERS_CREATING
Creating peers.
@ STATE_SLAVES_STARTING
Starting slaves.
@ STATE_INIT
Initial state.
@ STATE_PEERS_LINKING
Linking peers.
@ STATE_SEARCH_REGEX
Matching strings against announced regexes.
@ STATE_PEERS_STARTING
Starting peers.
@ STATE_PEERS_DESTROYING
Destroying peers; we can do this as the controller takes care of stopping a peer if it is running.
static void do_abort(void *cls)
abort task to run on test timed out
static int in_shutdown
Set when shutting down to avoid making more queries.
static struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
static void do_announce(void *cls)
Task to start the daemons on each peer so that the regexes are announced into the DHT.
static struct GNUNET_SCHEDULER_Task * register_hosts_task
Host registration task identifier.
static struct GNUNET_CONFIGURATION_Handle * cfg
Handle to global configuration.
static void do_shutdown(void *cls)
Shutdown nicely.
static uint64_t event_mask
Global event mask for all testbed events.
static char * hosts_file
File with hostnames where to execute the test.
static struct GNUNET_SCHEDULER_Task * search_timeout_task
Search timeout task identifier.
static unsigned int strings_found
Number of strings found in the published regexes.
static int result
Global testing status.
static void announce_next_regex(void *cls)
Start announcing the next regex in the DHT.
static void find_string(void *cls)
Start searching for the next string in the DHT.
static struct GNUNET_TIME_Relative search_timeout_time
Search timeout in seconds.
static char * strings_file
File with the strings to look for.
static void dht_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
DHT connect callback.
static int load_search_strings(const char *filename, char ***strings, unsigned int limit)
Load search strings from given filename.
static struct GNUNET_TESTBED_ControllerProc * mc_proc
Handle to the master controller process.
static int create_string_array(char *data, uint64_t data_size, char ***strings, unsigned int str_cnt)
Allocate a string array and fill it with the prefixed strings from a pre-processed,...
static void dht_da(void *cls, void *op_result)
Adapter function called to destroy a connection to the DHT service.
static void daemon_started(void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
Callback called when testbed has started the daemon we asked for.
static void * dht_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
DHT connect adapter.
static char ** search_strings
Search strings (num_peers of them).
#define FIND_TIMEOUT
static void * stats_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
Adapter function called to establish a connection to statistics service.
static void do_collect_stats(void *cls)
Task to collect all statistics from all peers, will shutdown the profiler, when done.
static void master_controller_cb(void *cls, const struct GNUNET_TESTBED_EventInformation *event)
Function that will be called whenever something in the testbed changes.
int main(int argc, char *const *argv)
Main function.
static struct GNUNET_TESTBED_HostRegistrationHandle * reg_handle
Host registration handle.
static void search_timed_out(void *cls)
Connect by string timeout task.
static void regex_found_handler(void *cls, const struct GNUNET_PeerIdentity *id, const struct GNUNET_DHT_PathElement *get_path, unsigned int get_path_length, const struct GNUNET_DHT_PathElement *put_path, unsigned int put_path_length)
Method called when we've found a peer that announced a regex that matches our search string.
static char * regex_prefix
Prefix used for regex announcing.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
static unsigned int num_peers
Number of peers to be started by the profiler.
static struct GNUNET_DISK_FileHandle * data_file
File to log statistics to.
static struct RegexPeer * peers
The array of peers; we fill this as the peers are given to us by the testbed.
static void find_timed_out(void *cls)
Search timed out.
static struct GNUNET_TESTBED_Peer ** testbed_peers
Handles to all of the running peers.
static char buf[2048]
static const struct GNUNET_CONFIGURATION_Handle * config
static char * subsystem
Set to subsystem that we're going to get stats for (or NULL for all).
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-uri.c:37
#define GNUNET_log(kind,...)
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
void GNUNET_CONFIGURATION_set_value_string(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, const char *value)
Set a configuration value that should be a string.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
void GNUNET_CONFIGURATION_destroy(struct GNUNET_CONFIGURATION_Handle *cfg)
Destroy configuration object.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
void GNUNET_CONFIGURATION_set_value_number(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long number)
Set a configuration value that should be a number.
struct GNUNET_CONFIGURATION_Handle * GNUNET_CONFIGURATION_dup(const struct GNUNET_CONFIGURATION_Handle *cfg)
Duplicate an existing configuration object.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
@ GNUNET_CRYPTO_QUALITY_WEAK
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
struct GNUNET_DHT_Handle * GNUNET_DHT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int ht_len)
Initialize the connection with the DHT service.
Definition: dht_api.c:883
void GNUNET_DHT_disconnect(struct GNUNET_DHT_Handle *handle)
Shutdown connection with the DHT service.
Definition: dht_api.c:906
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1235
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition: disk.c:482
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:686
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:222
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:404
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1306
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:664
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:814
@ GNUNET_DISK_OPEN_TRUNCATE
Truncate file if it exists.
@ GNUNET_DISK_OPEN_CREATE
Create file if it doesn't exist.
@ GNUNET_DISK_OPEN_READWRITE
Open the file for both reading and writing.
@ GNUNET_DISK_PERM_USER_READ
Owner can read.
@ GNUNET_DISK_PERM_USER_WRITE
Owner can write.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_relative_time(char shortName, const char *name, const char *argumentHelp, const char *description, struct GNUNET_TIME_Relative *val)
Allow user to specify a struct GNUNET_TIME_Relative (using human-readable "fancy" time).
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
int GNUNET_snprintf(char *buf, size_t size, const char *format,...) __attribute__((format(printf
Like snprintf, just aborts if the buffer is of insufficient size.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:399
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:531
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1296
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:972
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1269
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
struct GNUNET_STATISTICS_GetHandle * GNUNET_STATISTICS_get(struct GNUNET_STATISTICS_Handle *handle, const char *subsystem, const char *name, GNUNET_STATISTICS_Callback cont, GNUNET_STATISTICS_Iterator proc, void *cls)
Get statistic from the peer.
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1196
void GNUNET_TESTBED_operation_done(struct GNUNET_TESTBED_Operation *operation)
This function is used to signal that the event information (struct GNUNET_TESTBED_EventInformation) f...
Definition: testbed_api.c:2045
void GNUNET_TESTBED_cancel_registration(struct GNUNET_TESTBED_HostRegistrationHandle *handle)
Cancel the pending registration.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_peer_manage_service(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_OperationCompletionCallback cb, void *cb_cls, unsigned int start)
Start or stop given service at a peer.
void GNUNET_TESTBED_run(const char *host_filename, const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int num_peers, uint64_t event_mask, GNUNET_TESTBED_ControllerCallback cc, void *cc_cls, GNUNET_TESTBED_TestMaster test_master, void *test_master_cls)
Convenience method for running a testbed with a single call.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_service_connect(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_ServiceConnectCompletionCallback cb, void *cb_cls, GNUNET_TESTBED_ConnectAdapter ca, GNUNET_TESTBED_DisconnectAdapter da, void *cada_cls)
Connect to a service offered by the given peer.
void GNUNET_TESTBED_controller_stop(struct GNUNET_TESTBED_ControllerProc *cproc)
Stop the controller process (also will terminate all peers and controllers dependent on this controll...
void GNUNET_TESTBED_controller_disconnect(struct GNUNET_TESTBED_Controller *c)
Stop the given controller (also will terminate all peers and controllers dependent on this controller...
Definition: testbed_api.c:1721
@ GNUNET_TESTBED_ET_CONNECT
A connection between two peers was established.
@ GNUNET_TESTBED_ET_PEER_START
A peer has been started.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:405
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:500
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:94
#define GNUNET_TIME_UNIT_MINUTES
One minute.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:554
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:453
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:543
static int op_result(struct GNUNET_OP_Handle *h, uint64_t op_id, int64_t result_code, const void *data, uint16_t data_size, void **ctx, uint8_t cancel)
Remove an operation, and call its result callback (unless it was cancelled).
Definition: op.c:245
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
const char * name
struct REGEX_INTERNAL_Search * REGEX_INTERNAL_search(struct GNUNET_DHT_Handle *dht, const char *string, REGEX_INTERNAL_Found callback, void *callback_cls, struct GNUNET_STATISTICS_Handle *stats)
Search for a peer offering a regex matching certain string in the DHT.
void REGEX_INTERNAL_search_cancel(struct REGEX_INTERNAL_Search *h)
Cancel an ongoing regex search in the DHT and free all resources.
library to parse regular expressions into dfa
DLL of operations.
struct DLLOperation * next
The next pointer for DLL.
struct GNUNET_TESTBED_Operation * op
The testbed operation handle.
struct DLLOperation * prev
The prev pointer for DLL.
void * cls
Closure.
Connection to the DHT service.
Definition: dht_api.c:203
A (signed) path tracking a block's flow through the DHT is represented by an array of path elements,...
Handle used to access files (and pipes).
Definition of a command line option.
The identity of the host (wraps the signing key of the peer).
Entry in list of pending tasks.
Definition: scheduler.c:135
Handle for the service.
Handle for controller process.
Handle to interact with a GNUnet testbed controller.
Definition: testbed_api.h:194
Argument to GNUNET_TESTBED_ControllerCallback with details about the event.
enum GNUNET_TESTBED_EventType type
Type of the event.
handle for host registration
Opaque handle to an abstract operation to be executed by the testing framework.
A peer controlled by the testing framework.
Time for absolute times used by GNUnet, in microseconds.
Time for relative time used by GNUnet, in microseconds.
Struct to keep information of searches of services described by a regex using a user-provided string ...
Peer handles.
int search_str_matched
Set to GNUNET_YES if the peer successfully matched the above search string.
const char * search_str
Peer's search string.
struct GNUNET_TESTBED_Operation * daemon_op
Daemon start.
struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
unsigned int id
Peer id.
struct GNUNET_STATISTICS_Handle * stats_handle
Peers's statistics handle.
struct GNUNET_CONFIGURATION_Handle * cfg
Peer configuration handle.
struct GNUNET_TESTBED_Operation * op_handle
Testbed operation handle for DHT.
struct GNUNET_TESTBED_Peer * peer_handle
The actual testbed peer handle.
struct GNUNET_DHT_Handle * dht_handle
Peer's DHT handle.
struct REGEX_INTERNAL_Search * search_handle
Handle to a running regex search.
struct GNUNET_SCHEDULER_Task * timeout
Operation timeout.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.