GNUnet  0.17.6
gnunet-regex-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011 - 2017 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
27 #include "platform.h"
28 #include "gnunet_applications.h"
29 #include "gnunet_util_lib.h"
30 #include "regex_internal_lib.h"
31 #include "gnunet_arm_service.h"
32 #include "gnunet_dht_service.h"
33 #include "gnunet_testbed_service.h"
34 
35 #define FIND_TIMEOUT \
36  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90)
37 
42 {
47 
51  void *cls;
52 
56  struct DLLOperation *next;
57 
61  struct DLLOperation *prev;
62 };
63 
64 
68 enum State
69 {
74 
79 
84 
89 
94 
99 
105 };
106 
107 
111 struct RegexPeer
112 {
116  unsigned int id;
117 
122 
127 
131  const char *search_str;
132 
140 
145 
150 
155 
160 
165 
170 
175 };
176 
180 static int in_shutdown;
181 
185 static struct RegexPeer *peers;
186 
191 
196 
201 
206 
211 
216 
220 static uint64_t event_mask;
221 
226 
230 static struct GNUNET_TIME_Relative prof_time;
231 
235 static unsigned int num_peers;
236 
240 static int result;
241 
245 enum State state;
246 
250 static char *policy_dir;
251 
255 static char *hosts_file;
256 
260 static char *strings_file;
261 
265 static char **search_strings;
266 
270 static long long unsigned int init_parallel_searches;
271 
275 static unsigned int parallel_searches;
276 
280 static unsigned int strings_found;
281 
285 static unsigned int next_search;
286 
291 
295 static struct GNUNET_TIME_Relative search_timeout_time = { 60000 };
296 
301 
305 static char *data_filename;
306 
311 static char *regex_prefix;
312 
317 
318 
319 /******************************************************************************/
320 /****************************** DECLARATIONS ********************************/
321 /******************************************************************************/
322 
333 static void
334 dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
335  void *ca_result, const char *emsg);
336 
345 static void *
346 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg);
347 
348 
356 static void
357 dht_da (void *cls, void *op_result);
358 
359 
369 static void
370 stats_connect_cb (void *cls,
372  void *ca_result,
373  const char *emsg);
374 
375 
381 static void
382 announce_next_regex (void *cls);
383 
384 
385 /******************************************************************************/
386 /******************************** SHUTDOWN **********************************/
387 /******************************************************************************/
388 
389 
395 static void
396 do_shutdown (void *cls)
397 {
398  struct RegexPeer *peer;
399  unsigned int peer_cnt;
400  unsigned int search_str_cnt;
401  char output_buffer[512];
402  size_t size;
403 
404  if (NULL != abort_task)
405  {
407  abort_task = NULL;
408  }
409  if (NULL != register_hosts_task)
410  {
412  register_hosts_task = NULL;
413  }
414  for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++)
415  {
416  peer = &peers[peer_cnt];
417 
418  if ((GNUNET_YES != peer->search_str_matched) && (NULL != data_file) )
419  {
420  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
421  size =
422  GNUNET_snprintf (output_buffer,
423  sizeof(output_buffer),
424  "%p Search string not found: %s (%d)\n"
425  "%p On peer: %u (%p)\n"
426  "%p After: %s\n",
427  peer, peer->search_str, peer->search_str_matched,
428  peer, peer->id, peer,
429  peer,
431  GNUNET_NO));
432  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
433  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
434  }
435 
436  if (NULL != peers[peer_cnt].op_handle)
438  }
439 
440  if (NULL != data_file)
441  {
443  data_file = NULL;
444  }
445  for (search_str_cnt = 0;
446  search_str_cnt < num_peers && NULL != search_strings;
447  search_str_cnt++)
448  {
449  GNUNET_free (search_strings[search_str_cnt]);
450  }
452  search_strings = NULL;
453 
454  if (NULL != reg_handle)
455  {
457  reg_handle = NULL;
458  }
459  if (NULL != mc)
460  {
462  mc = NULL;
463  }
464  if (NULL != mc_proc)
465  {
467  mc_proc = NULL;
468  }
469  if (NULL != cfg)
470  {
472  cfg = NULL;
473  }
474 }
475 
476 
482 static void
483 do_abort (void *cls)
484 {
485  unsigned long i = (unsigned long) cls;
486 
488  "Aborting from line %lu...\n", i);
489  abort_task = NULL;
492 }
493 
494 
495 /******************************************************************************/
496 /********************* STATISTICS SERVICE CONNECTIONS ***********************/
497 /******************************************************************************/
498 
509 static void *
510 stats_ca (void *cls,
511  const struct GNUNET_CONFIGURATION_Handle *cfg)
512 {
513  return GNUNET_STATISTICS_create ("<driver>", cfg);
514 }
515 
516 
524 static void
525 stats_da (void *cls, void *op_result)
526 {
527  struct RegexPeer *peer = cls;
528 
529  GNUNET_assert (op_result == peer->stats_handle);
530 
531  GNUNET_STATISTICS_destroy (peer->stats_handle, GNUNET_NO);
532  peer->stats_handle = NULL;
533 }
534 
535 
546 static int
547 stats_iterator (void *cls,
548  const char *subsystem,
549  const char *name,
550  uint64_t value, int is_persistent)
551 {
552  struct RegexPeer *peer = cls;
553  char output_buffer[512];
554  size_t size;
555 
556  if (NULL == data_file)
557  {
559  "%p -> %s [%s]: %llu\n",
560  peer,
561  subsystem,
562  name,
563  (unsigned long long) value);
564  return GNUNET_OK;
565  }
566  size =
567  GNUNET_snprintf (output_buffer,
568  sizeof(output_buffer),
569  "%p [%s] %llu %s\n",
570  peer,
571  subsystem,
572  (unsigned long long) value,
573  name);
574  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
576  "Unable to write to file!\n");
577 
578  return GNUNET_OK;
579 }
580 
581 
590 static void
591 stats_cb (void *cls,
592  int success)
593 {
594  static unsigned int peer_cnt;
595  struct RegexPeer *peer = cls;
596 
597  if (GNUNET_OK != success)
598  {
600  "Getting statistics for peer %u failed!\n",
601  peer->id);
602  return;
603  }
604 
605  GNUNET_assert (NULL != peer->op_handle);
606 
607  GNUNET_TESTBED_operation_done (peer->op_handle);
608  peer->op_handle = NULL;
609 
610  peer_cnt++;
611  peer = &peers[peer_cnt];
612 
613  fprintf (stderr, "s");
614  if (peer_cnt == num_peers)
615  {
617  "\nCollecting stats finished. Shutting down.\n");
619  result = GNUNET_OK;
620  }
621  else
622  {
623  peer->op_handle =
625  peer->peer_handle,
626  "statistics",
628  peer,
629  &stats_ca,
630  &stats_da,
631  peer);
632  }
633 }
634 
635 
645 static void
646 stats_connect_cb (void *cls,
648  void *ca_result,
649  const char *emsg)
650 {
651  struct RegexPeer *peer = cls;
652 
653  if ((NULL == ca_result) || (NULL != emsg))
654  {
656  "Failed to connect to statistics service on peer %u: %s\n",
657  peer->id, emsg);
658 
659  peer->stats_handle = NULL;
660  return;
661  }
662 
663  peer->stats_handle = ca_result;
664 
665  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
666  &stats_cb,
667  &stats_iterator, peer))
668  {
670  "Could not get statistics of peer %u!\n", peer->id);
671  }
672 }
673 
674 
681 static void
682 do_collect_stats (void *cls)
683 {
684  struct RegexPeer *peer = &peers[0];
685 
686  GNUNET_assert (NULL != peer->peer_handle);
687 
688  peer->op_handle =
690  peer->peer_handle,
691  "statistics",
693  peer,
694  &stats_ca,
695  &stats_da,
696  peer);
697 }
698 
699 
700 /******************************************************************************/
701 /************************ REGEX FIND CONNECTIONS **************************/
702 /******************************************************************************/
703 
704 
710 static void
711 find_string (void *cls);
712 
713 
725 static void
727  const struct GNUNET_PeerIdentity *id,
728  const struct GNUNET_DHT_PathElement *get_path,
729  unsigned int get_path_length,
730  const struct GNUNET_DHT_PathElement *put_path,
731  unsigned int put_path_length)
732 {
733  struct RegexPeer *peer = cls;
734  char output_buffer[512];
735  size_t size;
736 
737  if (GNUNET_YES == peer->search_str_matched)
738  {
740  "String %s on peer %u already matched!\n",
741  peer->search_str, peer->id);
742  return;
743  }
744 
745  strings_found++;
747 
748  if (NULL != peer->timeout)
749  {
750  GNUNET_SCHEDULER_cancel (peer->timeout);
751  peer->timeout = NULL;
752  if (GNUNET_NO == in_shutdown)
754  }
755 
756  if (NULL == id)
757  {
758  // FIXME not possible right now
760  "String matching timed out for string %s on peer %u (%i/%i)\n",
761  peer->search_str, peer->id, strings_found, num_peers);
762  peer->search_str_matched = GNUNET_SYSERR;
763  }
764  else
765  {
766  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
767 
769  "String %s found on peer %u after %s (%i/%i) (%u||)\n",
770  peer->search_str, peer->id,
773 
774  peer->search_str_matched = GNUNET_YES;
775 
776  if (NULL != data_file)
777  {
778  size =
779  GNUNET_snprintf (output_buffer,
780  sizeof(output_buffer),
781  "%p Peer: %u\n"
782  "%p Search string: %s\n"
783  "%p Search duration: %s\n\n",
784  peer, peer->id,
785  peer, peer->search_str,
786  peer,
788  GNUNET_NO));
789 
790  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
791  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
792  }
793  }
794 
795  GNUNET_TESTBED_operation_done (peer->op_handle);
796  peer->op_handle = NULL;
797 
798  if (strings_found == num_peers)
799  {
802  "All strings successfully matched in %s\n",
804 
805  if (NULL != search_timeout_task)
806  {
808  search_timeout_task = NULL;
809  }
810 
811  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n");
813  }
814 }
815 
816 
823 static void
824 search_timed_out (void *cls)
825 {
826  unsigned int i;
827 
829  "Finding matches to all strings did not succeed after %s.\n",
831  GNUNET_NO));
833  "Found %i of %i strings\n", strings_found, num_peers);
834 
836  "Search timed out after %s."
837  "Collecting stats and shutting down.\n",
839  GNUNET_NO));
840 
842  for (i = 0; i < num_peers; i++)
843  {
844  if (NULL != peers[i].op_handle)
845  {
847  peers[i].op_handle = NULL;
848  }
849  }
851 }
852 
853 
860 static void
861 find_timed_out (void *cls)
862 {
863  struct RegexPeer *p = cls;
864 
865  p->timeout = NULL;
867  "Searching for string \"%s\" on peer %d timed out.\n",
868  p->search_str,
869  p->id);
870  if (GNUNET_NO == in_shutdown)
872 }
873 
874 
880 static void
881 find_string (void *cls)
882 {
883  unsigned int search_peer = (unsigned int) (long) cls;
884 
885  if ((search_peer >= num_peers) ||
886  (GNUNET_YES == in_shutdown))
887  return;
888 
890  "Searching for string \"%s\" on peer %d (%u||)\n",
891  peers[search_peer].search_str,
892  search_peer,
894 
895  peers[search_peer].op_handle =
897  peers[search_peer].peer_handle,
898  "dht",
900  &peers[search_peer],
901  &dht_ca,
902  &dht_da,
903  &peers[search_peer]);
904  GNUNET_assert (NULL != peers[search_peer].op_handle);
905  peers[search_peer].timeout
908  &peers[search_peer]);
909 }
910 
911 
919 static void
920 daemon_started (void *cls,
922  const char *emsg)
923 {
924  struct RegexPeer *peer = (struct RegexPeer *) cls;
925  unsigned long search_peer;
926  unsigned int i;
927 
928  GNUNET_TESTBED_operation_done (peer->daemon_op);
929  peer->daemon_op = NULL;
930  if (NULL != emsg)
931  {
933  "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg);
934  GNUNET_assert (0);
935  }
936  else
937  {
939  "Daemon %u started successfully\n", peer->id);
940  }
941 
942  /* Find a peer to look for a string matching the regex announced */
944  num_peers);
945  for (i = 0; peers[search_peer].search_str != NULL; i++)
946  {
947  search_peer = (search_peer + 1) % num_peers;
948  if (i > num_peers)
949  GNUNET_assert (0); /* we ran out of peers, must be a bug */
950  }
951  peers[search_peer].search_str = search_strings[peer->id];
952  peers[search_peer].search_str_matched = GNUNET_NO;
955  2),
956  &find_string,
957  (void *) search_peer);
958 }
959 
960 
968 static void
969 do_announce (void *cls)
970 {
971  unsigned int i;
972 
973  if (GNUNET_YES == in_shutdown)
974  return;
976  "Starting announce.\n");
977  for (i = 0; i < init_parallel_searches; i++)
978  {
980  " scheduling announce %u\n",
981  i);
983  }
984 }
985 
986 
992 static void
994 {
995  struct RegexPeer *peer;
996 
997  if (GNUNET_YES == in_shutdown)
998  return;
999  if (next_search >= num_peers)
1000  {
1001  if (strings_found != num_peers)
1002  {
1003  struct GNUNET_TIME_Relative new_delay;
1004  if (NULL != search_timeout_task)
1009  NULL);
1010  }
1011  return;
1012  }
1013 
1014  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search);
1015  peer = &peers[next_search];
1016  peer->daemon_op =
1018  peer->peer_handle,
1019  "regexprofiler",
1020  &daemon_started,
1021  peer,
1022  1);
1023  next_search++;
1025 }
1026 
1027 
1028 static void
1029 dht_connect_cb (void *cls,
1030  struct GNUNET_TESTBED_Operation *op,
1031  void *ca_result,
1032  const char *emsg)
1033 {
1034  struct RegexPeer *peer = (struct RegexPeer *) cls;
1035 
1036  if ((NULL != emsg) || (NULL == op) || (NULL == ca_result))
1037  {
1038  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg);
1039  GNUNET_assert (0);
1040  }
1041 
1042  GNUNET_assert (NULL != peer->dht_handle);
1043  GNUNET_assert (peer->op_handle == op);
1044  GNUNET_assert (peer->dht_handle == ca_result);
1045 
1046  peer->search_str_matched = GNUNET_NO;
1047  peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle,
1048  peer->search_str,
1050  NULL);
1051  peer->prof_start_time = GNUNET_TIME_absolute_get ();
1052 }
1053 
1054 
1055 static void *
1056 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
1057 {
1058  struct RegexPeer *peer = cls;
1059 
1060  peer->dht_handle = GNUNET_DHT_connect (cfg, 32);
1061 
1062  return peer->dht_handle;
1063 }
1064 
1065 
1066 static void
1067 dht_da (void *cls, void *op_result)
1068 {
1069  struct RegexPeer *peer = (struct RegexPeer *) cls;
1070 
1071  GNUNET_assert (peer->dht_handle == op_result);
1072 
1073  if (NULL != peer->search_handle)
1074  {
1075  REGEX_INTERNAL_search_cancel (peer->search_handle);
1076  peer->search_handle = NULL;
1077  }
1078 
1079  if (NULL != peer->dht_handle)
1080  {
1081  GNUNET_DHT_disconnect (peer->dht_handle);
1082  peer->dht_handle = NULL;
1083  }
1084 }
1085 
1086 
1100 static void
1101 test_master (void *cls,
1102  struct GNUNET_TESTBED_RunHandle *h,
1103  unsigned int num_peers_,
1105  unsigned int links_succeeded,
1106  unsigned int links_failed)
1107 {
1108  unsigned int i;
1109 
1110  GNUNET_assert (num_peers_ == num_peers);
1111 
1114  "Testbed started in %s\n",
1116 
1117  if (NULL != abort_task)
1118  {
1120  abort_task = NULL;
1121  }
1122 
1123  for (i = 0; i < num_peers; i++)
1124  {
1126  }
1127  if (GNUNET_NO ==
1128  GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT"))
1129  {
1130  struct GNUNET_TIME_Relative settle_time;
1131 
1132  settle_time =
1134  10 * num_peers);
1136  "Waiting for DHT for %s to settle new connections.\n\n",
1138  GNUNET_NO));
1139  GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL);
1140  }
1141  else
1142  {
1144  }
1147 }
1148 
1149 
1156 static void
1158  const struct GNUNET_TESTBED_EventInformation *event)
1159 {
1160  switch (event->type)
1161  {
1163  printf (".");
1164  break;
1165 
1167  printf ("#");
1168  break;
1169 
1170  default:
1171  break;
1172  }
1173  fflush (stdout);
1174 }
1175 
1176 
1177 /******************************************************************************/
1178 /*************************** TESTBED PEER SETUP *****************************/
1179 /******************************************************************************/
1180 
1191 static int
1193  uint64_t data_size,
1194  unsigned int str_max)
1195 {
1196  char *buf; // Keep track of last string to skip blank lines
1197  unsigned int offset;
1198  unsigned int str_cnt;
1199 
1200  buf = data;
1201  offset = 0;
1202  str_cnt = 0;
1203  while ((offset < (data_size - 1)) && (str_cnt < str_max))
1204  {
1205  offset++;
1206  if (((data[offset] == '\n')) &&
1207  (buf != &data[offset]))
1208  {
1209  data[offset] = '\0';
1210  str_cnt++;
1211  buf = &data[offset + 1];
1212  }
1213  else if ((data[offset] == '\n') ||
1214  (data[offset] == '\0'))
1215  buf = &data[offset + 1];
1216  }
1217  return str_cnt;
1218 }
1219 
1220 
1234 static int
1236  char ***strings, unsigned int str_cnt)
1237 {
1238  uint64_t offset;
1239  uint64_t len;
1240  unsigned int i;
1241 
1242  *strings = GNUNET_malloc (sizeof(char *) * str_cnt);
1243  offset = 0;
1244  for (i = 0; i < str_cnt; i++)
1245  {
1246  len = strlen (&data[offset]);
1247  if (offset + len >= data_size)
1248  {
1249  GNUNET_free (*strings);
1250  *strings = NULL;
1251  return GNUNET_SYSERR;
1252  }
1253  if (0 == len) // empty line
1254  {
1255  offset++;
1256  i--;
1257  continue;
1258  }
1259 
1260  GNUNET_asprintf (&(*strings)[i],
1261  "%s%s",
1262  regex_prefix,
1263  &data[offset]);
1264  offset += len + 1;
1265  }
1266  return GNUNET_OK;
1267 }
1268 
1269 
1279 static int
1281  char ***strings,
1282  unsigned int limit)
1283 {
1284  char *data;
1285  uint64_t filesize;
1286  int str_cnt;
1287 
1288  /* Sanity checks */
1289  if (NULL == filename)
1290  {
1291  return GNUNET_SYSERR;
1292  }
1294  {
1296  "Could not find search strings file %s\n", filename);
1297  return GNUNET_SYSERR;
1298  }
1299  if (GNUNET_OK !=
1301  &filesize,
1302  GNUNET_YES,
1303  GNUNET_YES))
1304  {
1306  "Search strings file %s cannot be read.\n",
1307  filename);
1308  return GNUNET_SYSERR;
1309  }
1310  if (0 == filesize)
1311  {
1313  "Search strings file %s is empty.\n",
1314  filename);
1315  return GNUNET_SYSERR;
1316  }
1317 
1318  /* Read data into memory */
1319  data = GNUNET_malloc (filesize + 1);
1320  if (filesize != GNUNET_DISK_fn_read (filename,
1321  data,
1322  filesize))
1323  {
1324  GNUNET_free (data);
1326  "Could not read search strings file %s.\n",
1327  filename);
1328  return GNUNET_SYSERR;
1329  }
1330 
1331  /* Process buffer and build array */
1332  str_cnt = count_and_separate_strings (data, filesize, limit);
1333  if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt))
1334  {
1335  str_cnt = GNUNET_SYSERR;
1336  }
1337  GNUNET_free (data);
1338  return str_cnt;
1339 }
1340 
1341 
1350 static void
1351 run (void *cls,
1352  char *const *args,
1353  const char *cfgfile,
1354  const struct GNUNET_CONFIGURATION_Handle *config)
1355 {
1356  unsigned int nsearchstrs;
1357  unsigned int i;
1358  struct GNUNET_TIME_Relative abort_time;
1359 
1361 
1362  /* Check config */
1363  if (NULL == config)
1364  {
1366  _ ("No configuration file given. Exiting\n"));
1368  return;
1369  }
1371  if (GNUNET_OK !=
1372  GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER",
1373  "REGEX_PREFIX",
1374  &regex_prefix))
1375  {
1377  "regexprofiler",
1378  "regex_prefix");
1380  return;
1381  }
1382  if (GNUNET_OK !=
1383  GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER",
1384  "PARALLEL_SEARCHES",
1386  {
1388  "Configuration option \"PARALLEL_SEARCHES\" missing."
1389  " Using default (%d)\n", 10);
1391  }
1392  if (GNUNET_OK !=
1393  GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER",
1394  "REANNOUNCE_PERIOD_MAX",
1396  {
1398  "reannounce_period_max not given. Using 10 minutes.\n");
1401  }
1402 
1403  /* Check arguments */
1404  if (NULL == policy_dir)
1405  {
1407  _ (
1408  "No policy directory specified on command line. Exiting.\n"));
1409  return;
1410  }
1412  {
1414  _ ("Specified policies directory does not exist. Exiting.\n"));
1416  return;
1417  }
1418  if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL,
1419  NULL)))
1420  {
1422  _ ("No files found in `%s'\n"),
1423  policy_dir);
1424  return;
1425  }
1426  GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER",
1427  "POLICY_DIR", policy_dir);
1429  {
1431  _ ("No search strings file given. Exiting.\n"));
1433  return;
1434  }
1435  nsearchstrs = load_search_strings (strings_file,
1436  &search_strings,
1437  num_peers);
1438  if (num_peers != nsearchstrs)
1439  {
1441  "Error loading search strings.\n");
1443  "File (%s) does not contain enough strings (%u/%u).\n",
1444  strings_file, nsearchstrs, num_peers);
1446  return;
1447  }
1448  if ((0 == num_peers) || (NULL == search_strings))
1449  {
1451  _ ("Error loading search strings. Exiting.\n"));
1453  return;
1454  }
1455  for (i = 0; i < num_peers; i++)
1457  "search string: %s\n",
1458  search_strings[i]);
1459 
1460  /* Check logfile */
1461  if ((NULL != data_filename) &&
1462  (NULL == (data_file =
1469  {
1471  "open",
1472  data_filename);
1473  return;
1474  }
1475 
1476  /* Initialize peers */
1477  peers = GNUNET_malloc (sizeof(struct RegexPeer) * num_peers);
1478  for (i = 0; i < num_peers; i++)
1479  peers[i].id = i;
1480 
1482  "TESTBED", "OVERLAY_RANDOM_LINKS",
1483  num_peers * 20);
1485  "DHT", "FORCE_NSE",
1486  (long long unsigned)
1487  (log (num_peers) / log (2.0)));
1488  event_mask = 0LL;
1489 /* For feedback about the start process activate these and pass master_cb */
1491 // event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
1493 // event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);
1496  cfg,
1497  num_peers,
1498  event_mask,
1500  NULL, /* master_controller_cb cls */
1501  &test_master,
1502  NULL); /* test_master cls */
1503  if (GNUNET_OK !=
1505  "SETUP_TIMEOUT",
1506  &abort_time))
1507  {
1509  "SETUP_TIMEOUT not given. Using 15 minutes.\n");
1510  abort_time =
1512  }
1513  abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES);
1514  abort_task =
1515  GNUNET_SCHEDULER_add_delayed (abort_time,
1516  &do_abort,
1517  (void *) __LINE__);
1519  "setup_timeout: %s\n",
1521 }
1522 
1523 
1531 int
1532 main (int argc, char *const *argv)
1533 {
1536  'o',
1537  "output-file",
1538  "FILENAME",
1539  gettext_noop (
1540  "name of the file for writing statistics"),
1541  &data_filename),
1543  't',
1544  "matching-timeout",
1545  "TIMEOUT",
1546  gettext_noop (
1547  "wait TIMEOUT before ending the experiment"),
1550  'p',
1551  "policy-dir",
1552  "DIRECTORY",
1553  gettext_noop ("directory with policy files"),
1554  &policy_dir),
1556  's',
1557  "strings-file",
1558  "FILENAME",
1559  gettext_noop (
1560  "name of file with input strings"),
1561  &strings_file),
1563  'H',
1564  "hosts-file",
1565  "FILENAME",
1566  gettext_noop (
1567  "name of file with hosts' names"),
1568  &hosts_file),
1569 
1571  };
1572  int ret;
1573 
1574  if (GNUNET_OK !=
1575  GNUNET_STRINGS_get_utf8_args (argc, argv,
1576  &argc, &argv))
1577  return 2;
1579  ret =
1580  GNUNET_PROGRAM_run (argc, argv,
1581  "gnunet-regex-profiler",
1582  _ ("Profiler for regex"),
1583  options,
1584  &run, NULL);
1585  if (GNUNET_OK != ret)
1586  return ret;
1587  if (GNUNET_OK != result)
1588  return 1;
1589  return 0;
1590 }
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
#define gettext_noop(String)
Definition: gettext.h:69
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:144
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
static char * filename
uint32_t data
The data value.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
static char * value
Value of the record to add/remove.
enum State state
current state of profiling
static unsigned int next_search
Index of peer to start next announce/search.
static char * data_filename
Filename to log statistics to.
static struct GNUNET_SCHEDULER_Task * abort_task
Abort task identifier.
static long long unsigned int init_parallel_searches
How many searches are we going to start in parallel.
static void stats_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
Function called by testbed once we are connected to stats service.
static int stats_iterator(void *cls, const char *subsystem, const char *name, uint64_t value, int is_persistent)
Process statistic values.
static int count_and_separate_strings(char *data, uint64_t data_size, unsigned int str_max)
Process the text buffer counting the non-empty lines and separating them with NULL characters,...
static void stats_cb(void *cls, int success)
Stats callback.
static struct GNUNET_TESTBED_Controller * mc
Handle to the master controller.
static struct GNUNET_TIME_Relative reannounce_period_max
What's the maximum regex reannounce period.
static char * policy_dir
Folder where policy files are stored.
static unsigned int parallel_searches
How many searches are running in parallel.
static struct GNUNET_TIME_Relative prof_time
Duration profiling step has taken.
static void stats_da(void *cls, void *op_result)
Adapter function called to destroy a connection to statistics service.
static void test_master(void *cls, struct GNUNET_TESTBED_RunHandle *h, unsigned int num_peers_, struct GNUNET_TESTBED_Peer **testbed_peers, unsigned int links_succeeded, unsigned int links_failed)
Signature of a main function for a testcase.
State
Available states during profiling.
@ STATE_PEERS_CREATING
Creating peers.
@ STATE_SLAVES_STARTING
Starting slaves.
@ STATE_INIT
Initial state.
@ STATE_PEERS_LINKING
Linking peers.
@ STATE_SEARCH_REGEX
Matching strings against announced regexes.
@ STATE_PEERS_STARTING
Starting peers.
@ STATE_PEERS_DESTROYING
Destroying peers; we can do this as the controller takes care of stopping a peer if it is running.
static void do_abort(void *cls)
abort task to run on test timed out
static int in_shutdown
Set when shutting down to avoid making more queries.
static struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
static void do_announce(void *cls)
Task to start the daemons on each peer so that the regexes are announced into the DHT.
static struct GNUNET_SCHEDULER_Task * register_hosts_task
Host registration task identifier.
static struct GNUNET_CONFIGURATION_Handle * cfg
Handle to global configuration.
static void do_shutdown(void *cls)
Shutdown nicely.
static uint64_t event_mask
Global event mask for all testbed events.
static char * hosts_file
File with hostnames where to execute the test.
static struct GNUNET_SCHEDULER_Task * search_timeout_task
Search timeout task identifier.
static unsigned int strings_found
Number of strings found in the published regexes.
static int result
Global testing status.
static void announce_next_regex(void *cls)
Start announcing the next regex in the DHT.
static void find_string(void *cls)
Start searching for the next string in the DHT.
static struct GNUNET_TIME_Relative search_timeout_time
Search timeout in seconds.
static char * strings_file
File with the strings to look for.
static void dht_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
DHT connect callback.
static int load_search_strings(const char *filename, char ***strings, unsigned int limit)
Load search strings from given filename.
static struct GNUNET_TESTBED_ControllerProc * mc_proc
Handle to the master controller process.
static int create_string_array(char *data, uint64_t data_size, char ***strings, unsigned int str_cnt)
Allocate a string array and fill it with the prefixed strings from a pre-processed,...
static void dht_da(void *cls, void *op_result)
Adapter function called to destroy a connection to the DHT service.
static void daemon_started(void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
Callback called when testbed has started the daemon we asked for.
static void * dht_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
DHT connect adapter.
static char ** search_strings
Search strings (num_peers of them).
#define FIND_TIMEOUT
static void * stats_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
Adapter function called to establish a connection to statistics service.
static void do_collect_stats(void *cls)
Task to collect all statistics from all peers, will shutdown the profiler, when done.
static void master_controller_cb(void *cls, const struct GNUNET_TESTBED_EventInformation *event)
Function that will be called whenever something in the testbed changes.
int main(int argc, char *const *argv)
Main function.
static struct GNUNET_TESTBED_HostRegistrationHandle * reg_handle
Host registration handle.
static void search_timed_out(void *cls)
Connect by string timeout task.
static void regex_found_handler(void *cls, const struct GNUNET_PeerIdentity *id, const struct GNUNET_DHT_PathElement *get_path, unsigned int get_path_length, const struct GNUNET_DHT_PathElement *put_path, unsigned int put_path_length)
Method called when we've found a peer that announced a regex that matches our search string.
static char * regex_prefix
Prefix used for regex announcing.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
static unsigned int num_peers
Number of peers to be started by the profiler.
static struct GNUNET_DISK_FileHandle * data_file
File to log statistics to.
static struct RegexPeer * peers
The array of peers; we fill this as the peers are given to us by the testbed.
static void find_timed_out(void *cls)
Search timed out.
static struct GNUNET_TESTBED_Peer ** testbed_peers
Handles to all of the running peers.
static char buf[2048]
static const struct GNUNET_CONFIGURATION_Handle * config
static char * subsystem
Set to subsystem that we're going to get stats for (or NULL for all).
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-uri.c:37
API to the DHT service.
API for writing tests and creating large-scale emulation testbeds for GNUnet.
void GNUNET_CONFIGURATION_set_value_string(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, const char *value)
Set a configuration value that should be a string.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
void GNUNET_CONFIGURATION_destroy(struct GNUNET_CONFIGURATION_Handle *cfg)
Destroy configuration object.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
void GNUNET_CONFIGURATION_set_value_number(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long number)
Set a configuration value that should be a number.
struct GNUNET_CONFIGURATION_Handle * GNUNET_CONFIGURATION_dup(const struct GNUNET_CONFIGURATION_Handle *cfg)
Duplicate an existing configuration object.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
@ GNUNET_CRYPTO_QUALITY_WEAK
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
struct GNUNET_DHT_Handle * GNUNET_DHT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int ht_len)
Initialize the connection with the DHT service.
Definition: dht_api.c:1039
void GNUNET_DHT_disconnect(struct GNUNET_DHT_Handle *handle)
Shutdown connection with the DHT service.
Definition: dht_api.c:1062
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1235
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition: disk.c:482
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:686
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:222
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:404
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1306
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:664
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:814
@ GNUNET_DISK_OPEN_TRUNCATE
Truncate file if it exists.
@ GNUNET_DISK_OPEN_CREATE
Create file if it doesn't exist.
@ GNUNET_DISK_OPEN_READWRITE
Open the file for both reading and writing.
@ GNUNET_DISK_PERM_USER_READ
Owner can read.
@ GNUNET_DISK_PERM_USER_WRITE
Owner can write.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_relative_time(char shortName, const char *name, const char *argumentHelp, const char *description, struct GNUNET_TIME_Relative *val)
Allow user to specify a struct GNUNET_TIME_Relative (using human-readable "fancy" time).
#define GNUNET_log(kind,...)
@ GNUNET_OK
Definition: gnunet_common.h:99
@ GNUNET_YES
@ GNUNET_NO
Definition: gnunet_common.h:98
@ GNUNET_SYSERR
Definition: gnunet_common.h:97
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
int GNUNET_snprintf(char *buf, size_t size, const char *format,...) __attribute__((format(printf
Like snprintf, just aborts if the buffer is of insufficient size.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:399
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:533
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1281
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:957
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1254
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
struct GNUNET_STATISTICS_GetHandle * GNUNET_STATISTICS_get(struct GNUNET_STATISTICS_Handle *handle, const char *subsystem, const char *name, GNUNET_STATISTICS_Callback cont, GNUNET_STATISTICS_Iterator proc, void *cls)
Get statistic from the peer.
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1223
void GNUNET_TESTBED_operation_done(struct GNUNET_TESTBED_Operation *operation)
This function is used to signal that the event information (struct GNUNET_TESTBED_EventInformation) f...
Definition: testbed_api.c:2045
void GNUNET_TESTBED_cancel_registration(struct GNUNET_TESTBED_HostRegistrationHandle *handle)
Cancel the pending registration.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_peer_manage_service(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_OperationCompletionCallback cb, void *cb_cls, unsigned int start)
Start or stop given service at a peer.
void GNUNET_TESTBED_run(const char *host_filename, const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int num_peers, uint64_t event_mask, GNUNET_TESTBED_ControllerCallback cc, void *cc_cls, GNUNET_TESTBED_TestMaster test_master, void *test_master_cls)
Convenience method for running a testbed with a single call.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_service_connect(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_ServiceConnectCompletionCallback cb, void *cb_cls, GNUNET_TESTBED_ConnectAdapter ca, GNUNET_TESTBED_DisconnectAdapter da, void *cada_cls)
Connect to a service offered by the given peer.
void GNUNET_TESTBED_controller_stop(struct GNUNET_TESTBED_ControllerProc *cproc)
Stop the controller process (also will terminate all peers and controllers dependent on this controll...
void GNUNET_TESTBED_controller_disconnect(struct GNUNET_TESTBED_Controller *c)
Stop the given controller (also will terminate all peers and controllers dependent on this controller...
Definition: testbed_api.c:1721
@ GNUNET_TESTBED_ET_CONNECT
A connection between two peers was established.
@ GNUNET_TESTBED_ET_PEER_START
A peer has been started.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:435
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:530
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:110
#define GNUNET_TIME_UNIT_MINUTES
One minute.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:584
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:483
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:570
static int op_result(struct GNUNET_OP_Handle *h, uint64_t op_id, int64_t result_code, const void *data, uint16_t data_size, void **ctx, uint8_t cancel)
Remove an operation, and call its result callback (unless it was cancelled).
Definition: op.c:245
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
const char * name
struct REGEX_INTERNAL_Search * REGEX_INTERNAL_search(struct GNUNET_DHT_Handle *dht, const char *string, REGEX_INTERNAL_Found callback, void *callback_cls, struct GNUNET_STATISTICS_Handle *stats)
Search for a peer offering a regex matching certain string in the DHT.
void REGEX_INTERNAL_search_cancel(struct REGEX_INTERNAL_Search *h)
Cancel an ongoing regex search in the DHT and free all resources.
library to parse regular expressions into dfa
DLL of operations.
struct DLLOperation * next
The next pointer for DLL.
struct GNUNET_TESTBED_Operation * op
The testbed operation handle.
struct DLLOperation * prev
The prev pointer for DLL.
void * cls
Closure.
Connection to the DHT service.
Definition: dht_api.c:237
A (signed) path tracking a block's flow through the DHT is represented by an array of path elements,...
Handle used to access files (and pipes).
Definition of a command line option.
The identity of the host (wraps the signing key of the peer).
Entry in list of pending tasks.
Definition: scheduler.c:135
Handle for the service.
Handle for controller process.
Handle to interact with a GNUnet testbed controller.
Definition: testbed_api.h:194
Argument to GNUNET_TESTBED_ControllerCallback with details about the event.
enum GNUNET_TESTBED_EventType type
Type of the event.
handle for host registration
Opaque handle to an abstract operation to be executed by the testing framework.
A peer controlled by the testing framework.
Time for absolute times used by GNUnet, in microseconds.
Time for relative time used by GNUnet, in microseconds.
Struct to keep information of searches of services described by a regex using a user-provided string ...
Peer handles.
int search_str_matched
Set to GNUNET_YES if the peer successfully matched the above search string.
const char * search_str
Peer's search string.
struct GNUNET_TESTBED_Operation * daemon_op
Daemon start.
struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
unsigned int id
Peer id.
struct GNUNET_STATISTICS_Handle * stats_handle
Peers's statistics handle.
struct GNUNET_CONFIGURATION_Handle * cfg
Peer configuration handle.
struct GNUNET_TESTBED_Operation * op_handle
Testbed operation handle for DHT.
struct GNUNET_TESTBED_Peer * peer_handle
The actual testbed peer handle.
struct GNUNET_DHT_Handle * dht_handle
Peer's DHT handle.
struct REGEX_INTERNAL_Search * search_handle
Handle to a running regex search.
struct GNUNET_SCHEDULER_Task * timeout
Operation timeout.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.