GNUnet  0.19.2
gnunet-regex-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011 - 2017 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
27 #include "platform.h"
28 #include "gnunet_applications.h"
29 #include "gnunet_util_lib.h"
30 #include "regex_internal_lib.h"
31 #include "gnunet_arm_service.h"
32 #include "gnunet_dht_service.h"
33 #include "gnunet_testbed_service.h"
34 
35 #define FIND_TIMEOUT \
36  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90)
37 
42 {
47 
51  void *cls;
52 
56  struct DLLOperation *next;
57 
61  struct DLLOperation *prev;
62 };
63 
64 
68 enum State
69 {
74 
79 
84 
89 
94 
99 
105 };
106 
107 
111 struct RegexPeer
112 {
116  unsigned int id;
117 
122 
127 
131  const char *search_str;
132 
140 
145 
150 
155 
160 
165 
170 
175 };
176 
180 static int in_shutdown;
181 
185 static struct RegexPeer *peers;
186 
191 
196 
201 
206 
211 
216 
220 static uint64_t event_mask;
221 
226 
230 static struct GNUNET_TIME_Relative prof_time;
231 
235 static unsigned int num_peers;
236 
240 static int result;
241 
245 enum State state;
246 
250 static char *policy_dir;
251 
255 static char *hosts_file;
256 
260 static char *strings_file;
261 
265 static char **search_strings;
266 
270 static long long unsigned int init_parallel_searches;
271 
275 static unsigned int parallel_searches;
276 
280 static unsigned int strings_found;
281 
285 static unsigned int next_search;
286 
291 
295 static struct GNUNET_TIME_Relative search_timeout_time = { 60000 };
296 
301 
305 static char *data_filename;
306 
311 static char *regex_prefix;
312 
317 
318 
319 /******************************************************************************/
320 /****************************** DECLARATIONS ********************************/
321 /******************************************************************************/
322 
333 static void
334 dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
335  void *ca_result, const char *emsg);
336 
345 static void *
346 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg);
347 
348 
356 static void
357 dht_da (void *cls, void *op_result);
358 
359 
369 static void
370 stats_connect_cb (void *cls,
372  void *ca_result,
373  const char *emsg);
374 
375 
381 static void
382 announce_next_regex (void *cls);
383 
384 
385 /******************************************************************************/
386 /******************************** SHUTDOWN **********************************/
387 /******************************************************************************/
388 
389 
395 static void
396 do_shutdown (void *cls)
397 {
398  struct RegexPeer *peer;
399  unsigned int peer_cnt;
400  unsigned int search_str_cnt;
401  char output_buffer[512];
402  size_t size;
403 
404  if (NULL != abort_task)
405  {
407  abort_task = NULL;
408  }
409  if (NULL != register_hosts_task)
410  {
412  register_hosts_task = NULL;
413  }
414  for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++)
415  {
416  peer = &peers[peer_cnt];
417 
418  if ((GNUNET_YES != peer->search_str_matched) && (NULL != data_file) )
419  {
420  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
421  size =
422  GNUNET_snprintf (output_buffer,
423  sizeof(output_buffer),
424  "%p Search string not found: %s (%d)\n"
425  "%p On peer: %u (%p)\n"
426  "%p After: %s\n",
427  peer, peer->search_str, peer->search_str_matched,
428  peer, peer->id, peer,
429  peer,
431  GNUNET_NO));
432  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
433  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
434  }
435 
436  if (NULL != peers[peer_cnt].op_handle)
438  }
439 
440  if (NULL != data_file)
441  {
443  data_file = NULL;
444  }
445  for (search_str_cnt = 0;
446  search_str_cnt < num_peers && NULL != search_strings;
447  search_str_cnt++)
448  {
449  GNUNET_free (search_strings[search_str_cnt]);
450  }
452  search_strings = NULL;
453 
454  if (NULL != reg_handle)
455  {
457  reg_handle = NULL;
458  }
459  if (NULL != mc)
460  {
462  mc = NULL;
463  }
464  if (NULL != mc_proc)
465  {
467  mc_proc = NULL;
468  }
469  if (NULL != cfg)
470  {
472  cfg = NULL;
473  }
474 }
475 
476 
482 static void
483 do_abort (void *cls)
484 {
485  unsigned long i = (unsigned long) cls;
486 
488  "Aborting from line %lu...\n", i);
489  abort_task = NULL;
492 }
493 
494 
495 /******************************************************************************/
496 /********************* STATISTICS SERVICE CONNECTIONS ***********************/
497 /******************************************************************************/
498 
509 static void *
510 stats_ca (void *cls,
511  const struct GNUNET_CONFIGURATION_Handle *cfg)
512 {
513  return GNUNET_STATISTICS_create ("<driver>", cfg);
514 }
515 
516 
524 static void
525 stats_da (void *cls, void *op_result)
526 {
527  struct RegexPeer *peer = cls;
528 
529  GNUNET_assert (op_result == peer->stats_handle);
530 
531  GNUNET_STATISTICS_destroy (peer->stats_handle, GNUNET_NO);
532  peer->stats_handle = NULL;
533 }
534 
535 
546 static int
547 stats_iterator (void *cls,
548  const char *subsystem,
549  const char *name,
550  uint64_t value, int is_persistent)
551 {
552  struct RegexPeer *peer = cls;
553  char output_buffer[512];
554  size_t size;
555 
556  if (NULL == data_file)
557  {
559  "%p -> %s [%s]: %llu\n",
560  peer,
561  subsystem,
562  name,
563  (unsigned long long) value);
564  return GNUNET_OK;
565  }
566  size =
567  GNUNET_snprintf (output_buffer,
568  sizeof(output_buffer),
569  "%p [%s] %llu %s\n",
570  peer,
571  subsystem,
572  (unsigned long long) value,
573  name);
574  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
576  "Unable to write to file!\n");
577 
578  return GNUNET_OK;
579 }
580 
581 
590 static void
591 stats_cb (void *cls,
592  int success)
593 {
594  static unsigned int peer_cnt;
595  struct RegexPeer *peer = cls;
596 
597  if (GNUNET_OK != success)
598  {
600  "Getting statistics for peer %u failed!\n",
601  peer->id);
602  return;
603  }
604 
605  GNUNET_assert (NULL != peer->op_handle);
606 
607  GNUNET_TESTBED_operation_done (peer->op_handle);
608  peer->op_handle = NULL;
609 
610  peer_cnt++;
611  peer = &peers[peer_cnt];
612 
613  fprintf (stderr, "s");
614  if (peer_cnt == num_peers)
615  {
617  "\nCollecting stats finished. Shutting down.\n");
619  result = GNUNET_OK;
620  }
621  else
622  {
623  peer->op_handle =
625  peer->peer_handle,
626  "statistics",
628  peer,
629  &stats_ca,
630  &stats_da,
631  peer);
632  }
633 }
634 
635 
645 static void
646 stats_connect_cb (void *cls,
648  void *ca_result,
649  const char *emsg)
650 {
651  struct RegexPeer *peer = cls;
652 
653  if ((NULL == ca_result) || (NULL != emsg))
654  {
656  "Failed to connect to statistics service on peer %u: %s\n",
657  peer->id, emsg);
658 
659  peer->stats_handle = NULL;
660  return;
661  }
662 
663  peer->stats_handle = ca_result;
664 
665  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
666  &stats_cb,
667  &stats_iterator, peer))
668  {
670  "Could not get statistics of peer %u!\n", peer->id);
671  }
672 }
673 
674 
681 static void
682 do_collect_stats (void *cls)
683 {
684  struct RegexPeer *peer = &peers[0];
685 
686  GNUNET_assert (NULL != peer->peer_handle);
687 
688  peer->op_handle =
690  peer->peer_handle,
691  "statistics",
693  peer,
694  &stats_ca,
695  &stats_da,
696  peer);
697 }
698 
699 
700 /******************************************************************************/
701 /************************ REGEX FIND CONNECTIONS **************************/
702 /******************************************************************************/
703 
704 
710 static void
711 find_string (void *cls);
712 
713 
725 static void
727  const struct GNUNET_PeerIdentity *id,
728  const struct GNUNET_DHT_PathElement *get_path,
729  unsigned int get_path_length,
730  const struct GNUNET_DHT_PathElement *put_path,
731  unsigned int put_path_length)
732 {
733  struct RegexPeer *peer = cls;
734  char output_buffer[512];
735  size_t size;
736 
737  if (GNUNET_YES == peer->search_str_matched)
738  {
740  "String %s on peer %u already matched!\n",
741  peer->search_str, peer->id);
742  return;
743  }
744 
745  strings_found++;
747 
748  if (NULL != peer->timeout)
749  {
750  GNUNET_SCHEDULER_cancel (peer->timeout);
751  peer->timeout = NULL;
752  if (GNUNET_NO == in_shutdown)
754  }
755 
756  if (NULL == id)
757  {
758  // FIXME not possible right now
760  "String matching timed out for string %s on peer %u (%i/%i)\n",
761  peer->search_str, peer->id, strings_found, num_peers);
762  peer->search_str_matched = GNUNET_SYSERR;
763  }
764  else
765  {
766  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
767 
769  "String %s found on peer %u after %s (%i/%i) (%u||)\n",
770  peer->search_str, peer->id,
773 
774  peer->search_str_matched = GNUNET_YES;
775 
776  if (NULL != data_file)
777  {
778  size =
779  GNUNET_snprintf (output_buffer,
780  sizeof(output_buffer),
781  "%p Peer: %u\n"
782  "%p Search string: %s\n"
783  "%p Search duration: %s\n\n",
784  peer, peer->id,
785  peer, peer->search_str,
786  peer,
788  GNUNET_NO));
789 
790  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
791  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
792  }
793  }
794 
795  GNUNET_TESTBED_operation_done (peer->op_handle);
796  peer->op_handle = NULL;
797 
798  if (strings_found == num_peers)
799  {
802  "All strings successfully matched in %s\n",
804 
805  if (NULL != search_timeout_task)
806  {
808  search_timeout_task = NULL;
809  }
810 
811  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n");
813  }
814 }
815 
816 
823 static void
824 search_timed_out (void *cls)
825 {
826  unsigned int i;
827 
829  "Finding matches to all strings did not succeed after %s.\n",
831  GNUNET_NO));
833  "Found %i of %i strings\n", strings_found, num_peers);
834 
836  "Search timed out after %s."
837  "Collecting stats and shutting down.\n",
839  GNUNET_NO));
840 
842  for (i = 0; i < num_peers; i++)
843  {
844  if (NULL != peers[i].op_handle)
845  {
847  peers[i].op_handle = NULL;
848  }
849  }
851 }
852 
853 
860 static void
861 find_timed_out (void *cls)
862 {
863  struct RegexPeer *p = cls;
864 
865  p->timeout = NULL;
867  "Searching for string \"%s\" on peer %d timed out.\n",
868  p->search_str,
869  p->id);
870  if (GNUNET_NO == in_shutdown)
872 }
873 
874 
880 static void
881 find_string (void *cls)
882 {
883  unsigned int search_peer = (unsigned int) (long) cls;
884 
885  if ((search_peer >= num_peers) ||
886  (GNUNET_YES == in_shutdown))
887  return;
888 
890  "Searching for string \"%s\" on peer %d (%u||)\n",
891  peers[search_peer].search_str,
892  search_peer,
894 
895  peers[search_peer].op_handle =
897  peers[search_peer].peer_handle,
898  "dht",
900  &peers[search_peer],
901  &dht_ca,
902  &dht_da,
903  &peers[search_peer]);
904  GNUNET_assert (NULL != peers[search_peer].op_handle);
905  peers[search_peer].timeout
908  &peers[search_peer]);
909 }
910 
911 
919 static void
920 daemon_started (void *cls,
922  const char *emsg)
923 {
924  struct RegexPeer *peer = (struct RegexPeer *) cls;
925  unsigned long search_peer;
926  unsigned int i;
927 
928  GNUNET_TESTBED_operation_done (peer->daemon_op);
929  peer->daemon_op = NULL;
930  if (NULL != emsg)
931  {
933  "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg);
934  GNUNET_assert (0);
935  }
936  else
937  {
939  "Daemon %u started successfully\n", peer->id);
940  }
941 
942  /* Find a peer to look for a string matching the regex announced */
944  num_peers);
945  for (i = 0; peers[search_peer].search_str != NULL; i++)
946  {
947  search_peer = (search_peer + 1) % num_peers;
948  if (i > num_peers)
949  GNUNET_assert (0); /* we ran out of peers, must be a bug */
950  }
951  peers[search_peer].search_str = search_strings[peer->id];
952  peers[search_peer].search_str_matched = GNUNET_NO;
955  2),
956  &find_string,
957  (void *) search_peer);
958 }
959 
960 
967 static void
968 do_announce (void *cls)
969 {
970  unsigned int i;
971 
972  if (GNUNET_YES == in_shutdown)
973  return;
975  "Starting announce.\n");
976  for (i = 0; i < init_parallel_searches; i++)
977  {
979  " scheduling announce %u\n",
980  i);
982  }
983 }
984 
985 
991 static void
993 {
994  struct RegexPeer *peer;
995 
996  if (GNUNET_YES == in_shutdown)
997  return;
998  if (next_search >= num_peers)
999  {
1000  if (strings_found != num_peers)
1001  {
1002  struct GNUNET_TIME_Relative new_delay;
1003  if (NULL != search_timeout_task)
1008  NULL);
1009  }
1010  return;
1011  }
1012 
1013  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search);
1014  peer = &peers[next_search];
1015  peer->daemon_op =
1017  peer->peer_handle,
1018  "regexprofiler",
1019  &daemon_started,
1020  peer,
1021  1);
1022  next_search++;
1024 }
1025 
1026 
1027 static void
1028 dht_connect_cb (void *cls,
1029  struct GNUNET_TESTBED_Operation *op,
1030  void *ca_result,
1031  const char *emsg)
1032 {
1033  struct RegexPeer *peer = (struct RegexPeer *) cls;
1034 
1035  if ((NULL != emsg) || (NULL == op) || (NULL == ca_result))
1036  {
1037  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg);
1038  GNUNET_assert (0);
1039  }
1040 
1041  GNUNET_assert (NULL != peer->dht_handle);
1042  GNUNET_assert (peer->op_handle == op);
1043  GNUNET_assert (peer->dht_handle == ca_result);
1044 
1045  peer->search_str_matched = GNUNET_NO;
1046  peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle,
1047  peer->search_str,
1049  NULL);
1050  peer->prof_start_time = GNUNET_TIME_absolute_get ();
1051 }
1052 
1053 
1054 static void *
1055 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
1056 {
1057  struct RegexPeer *peer = cls;
1058 
1059  peer->dht_handle = GNUNET_DHT_connect (cfg, 32);
1060 
1061  return peer->dht_handle;
1062 }
1063 
1064 
1065 static void
1066 dht_da (void *cls, void *op_result)
1067 {
1068  struct RegexPeer *peer = (struct RegexPeer *) cls;
1069 
1070  GNUNET_assert (peer->dht_handle == op_result);
1071 
1072  if (NULL != peer->search_handle)
1073  {
1074  REGEX_INTERNAL_search_cancel (peer->search_handle);
1075  peer->search_handle = NULL;
1076  }
1077 
1078  if (NULL != peer->dht_handle)
1079  {
1080  GNUNET_DHT_disconnect (peer->dht_handle);
1081  peer->dht_handle = NULL;
1082  }
1083 }
1084 
1085 
1099 static void
1100 test_master (void *cls,
1101  struct GNUNET_TESTBED_RunHandle *h,
1102  unsigned int num_peers_,
1104  unsigned int links_succeeded,
1105  unsigned int links_failed)
1106 {
1107  unsigned int i;
1108 
1109  GNUNET_assert (num_peers_ == num_peers);
1110 
1113  "Testbed started in %s\n",
1115 
1116  if (NULL != abort_task)
1117  {
1119  abort_task = NULL;
1120  }
1121 
1122  for (i = 0; i < num_peers; i++)
1123  {
1125  }
1126  if (GNUNET_NO ==
1127  GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT"))
1128  {
1129  struct GNUNET_TIME_Relative settle_time;
1130 
1131  settle_time =
1133  10 * num_peers);
1135  "Waiting for DHT for %s to settle new connections.\n\n",
1137  GNUNET_NO));
1138  GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL);
1139  }
1140  else
1141  {
1143  }
1146 }
1147 
1148 
1155 static void
1157  const struct GNUNET_TESTBED_EventInformation *event)
1158 {
1159  switch (event->type)
1160  {
1162  printf (".");
1163  break;
1164 
1166  printf ("#");
1167  break;
1168 
1169  default:
1170  break;
1171  }
1172  fflush (stdout);
1173 }
1174 
1175 
1176 /******************************************************************************/
1177 /*************************** TESTBED PEER SETUP *****************************/
1178 /******************************************************************************/
1179 
1190 static int
1192  uint64_t data_size,
1193  unsigned int str_max)
1194 {
1195  char *buf; // Keep track of last string to skip blank lines
1196  unsigned int offset;
1197  unsigned int str_cnt;
1198 
1199  buf = data;
1200  offset = 0;
1201  str_cnt = 0;
1202  while ((offset < (data_size - 1)) && (str_cnt < str_max))
1203  {
1204  offset++;
1205  if (((data[offset] == '\n')) &&
1206  (buf != &data[offset]))
1207  {
1208  data[offset] = '\0';
1209  str_cnt++;
1210  buf = &data[offset + 1];
1211  }
1212  else if ((data[offset] == '\n') ||
1213  (data[offset] == '\0'))
1214  buf = &data[offset + 1];
1215  }
1216  return str_cnt;
1217 }
1218 
1219 
1233 static int
1235  char ***strings, unsigned int str_cnt)
1236 {
1237  uint64_t offset;
1238  uint64_t len;
1239  unsigned int i;
1240 
1241  *strings = GNUNET_malloc (sizeof(char *) * str_cnt);
1242  offset = 0;
1243  for (i = 0; i < str_cnt; i++)
1244  {
1245  len = strlen (&data[offset]);
1246  if (offset + len >= data_size)
1247  {
1248  GNUNET_free (*strings);
1249  *strings = NULL;
1250  return GNUNET_SYSERR;
1251  }
1252  if (0 == len) // empty line
1253  {
1254  offset++;
1255  i--;
1256  continue;
1257  }
1258 
1259  GNUNET_asprintf (&(*strings)[i],
1260  "%s%s",
1261  regex_prefix,
1262  &data[offset]);
1263  offset += len + 1;
1264  }
1265  return GNUNET_OK;
1266 }
1267 
1268 
1278 static int
1280  char ***strings,
1281  unsigned int limit)
1282 {
1283  char *data;
1284  uint64_t filesize;
1285  int str_cnt;
1286 
1287  /* Sanity checks */
1288  if (NULL == filename)
1289  {
1290  return GNUNET_SYSERR;
1291  }
1293  {
1295  "Could not find search strings file %s\n", filename);
1296  return GNUNET_SYSERR;
1297  }
1298  if (GNUNET_OK !=
1300  &filesize,
1301  GNUNET_YES,
1302  GNUNET_YES))
1303  {
1305  "Search strings file %s cannot be read.\n",
1306  filename);
1307  return GNUNET_SYSERR;
1308  }
1309  if (0 == filesize)
1310  {
1312  "Search strings file %s is empty.\n",
1313  filename);
1314  return GNUNET_SYSERR;
1315  }
1316 
1317  /* Read data into memory */
1318  data = GNUNET_malloc (filesize + 1);
1319  if (filesize != GNUNET_DISK_fn_read (filename,
1320  data,
1321  filesize))
1322  {
1323  GNUNET_free (data);
1325  "Could not read search strings file %s.\n",
1326  filename);
1327  return GNUNET_SYSERR;
1328  }
1329 
1330  /* Process buffer and build array */
1331  str_cnt = count_and_separate_strings (data, filesize, limit);
1332  if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt))
1333  {
1334  str_cnt = GNUNET_SYSERR;
1335  }
1336  GNUNET_free (data);
1337  return str_cnt;
1338 }
1339 
1340 
1349 static void
1350 run (void *cls,
1351  char *const *args,
1352  const char *cfgfile,
1353  const struct GNUNET_CONFIGURATION_Handle *config)
1354 {
1355  unsigned int nsearchstrs;
1356  unsigned int i;
1357  struct GNUNET_TIME_Relative abort_time;
1358 
1360 
1361  /* Check config */
1362  if (NULL == config)
1363  {
1365  _ ("No configuration file given. Exiting\n"));
1367  return;
1368  }
1370  if (GNUNET_OK !=
1371  GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER",
1372  "REGEX_PREFIX",
1373  &regex_prefix))
1374  {
1376  "regexprofiler",
1377  "regex_prefix");
1379  return;
1380  }
1381  if (GNUNET_OK !=
1382  GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER",
1383  "PARALLEL_SEARCHES",
1385  {
1387  "Configuration option \"PARALLEL_SEARCHES\" missing."
1388  " Using default (%d)\n", 10);
1390  }
1391  if (GNUNET_OK !=
1392  GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER",
1393  "REANNOUNCE_PERIOD_MAX",
1395  {
1397  "reannounce_period_max not given. Using 10 minutes.\n");
1400  }
1401 
1402  /* Check arguments */
1403  if (NULL == policy_dir)
1404  {
1406  _ (
1407  "No policy directory specified on command line. Exiting.\n"));
1408  return;
1409  }
1411  {
1413  _ ("Specified policies directory does not exist. Exiting.\n"));
1415  return;
1416  }
1417  if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL,
1418  NULL)))
1419  {
1421  _ ("No files found in `%s'\n"),
1422  policy_dir);
1423  return;
1424  }
1425  GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER",
1426  "POLICY_DIR", policy_dir);
1428  {
1430  _ ("No search strings file given. Exiting.\n"));
1432  return;
1433  }
1434  nsearchstrs = load_search_strings (strings_file,
1435  &search_strings,
1436  num_peers);
1437  if (num_peers != nsearchstrs)
1438  {
1440  "Error loading search strings.\n");
1442  "File (%s) does not contain enough strings (%u/%u).\n",
1443  strings_file, nsearchstrs, num_peers);
1445  return;
1446  }
1447  if ((0 == num_peers) || (NULL == search_strings))
1448  {
1450  _ ("Error loading search strings. Exiting.\n"));
1452  return;
1453  }
1454  for (i = 0; i < num_peers; i++)
1456  "search string: %s\n",
1457  search_strings[i]);
1458 
1459  /* Check logfile */
1460  if ((NULL != data_filename) &&
1461  (NULL == (data_file =
1468  {
1470  "open",
1471  data_filename);
1472  return;
1473  }
1474 
1475  /* Initialize peers */
1476  peers = GNUNET_malloc (sizeof(struct RegexPeer) * num_peers);
1477  for (i = 0; i < num_peers; i++)
1478  peers[i].id = i;
1479 
1481  "TESTBED", "OVERLAY_RANDOM_LINKS",
1482  num_peers * 20);
1484  "DHT", "FORCE_NSE",
1485  (long long unsigned)
1486  (log (num_peers) / log (2.0)));
1487  event_mask = 0LL;
1488 /* For feedback about the start process activate these and pass master_cb */
1490 // event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
1492 // event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);
1495  cfg,
1496  num_peers,
1497  event_mask,
1499  NULL, /* master_controller_cb cls */
1500  &test_master,
1501  NULL); /* test_master cls */
1502  if (GNUNET_OK !=
1504  "SETUP_TIMEOUT",
1505  &abort_time))
1506  {
1508  "SETUP_TIMEOUT not given. Using 15 minutes.\n");
1509  abort_time =
1511  }
1512  abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES);
1513  abort_task =
1514  GNUNET_SCHEDULER_add_delayed (abort_time,
1515  &do_abort,
1516  (void *) __LINE__);
1518  "setup_timeout: %s\n",
1520 }
1521 
1522 
1530 int
1531 main (int argc, char *const *argv)
1532 {
1535  'o',
1536  "output-file",
1537  "FILENAME",
1538  gettext_noop (
1539  "name of the file for writing statistics"),
1540  &data_filename),
1542  't',
1543  "matching-timeout",
1544  "TIMEOUT",
1545  gettext_noop (
1546  "wait TIMEOUT before ending the experiment"),
1549  'p',
1550  "policy-dir",
1551  "DIRECTORY",
1552  gettext_noop ("directory with policy files"),
1553  &policy_dir),
1555  's',
1556  "strings-file",
1557  "FILENAME",
1558  gettext_noop (
1559  "name of file with input strings"),
1560  &strings_file),
1562  'H',
1563  "hosts-file",
1564  "FILENAME",
1565  gettext_noop (
1566  "name of file with hosts' names"),
1567  &hosts_file),
1568 
1570  };
1571  int ret;
1572 
1573  if (GNUNET_OK !=
1574  GNUNET_STRINGS_get_utf8_args (argc, argv,
1575  &argc, &argv))
1576  return 2;
1578  ret =
1579  GNUNET_PROGRAM_run (argc, argv,
1580  "gnunet-regex-profiler",
1581  _ ("Profiler for regex"),
1582  options,
1583  &run, NULL);
1584  if (GNUNET_OK != ret)
1585  return ret;
1586  if (GNUNET_OK != result)
1587  return 1;
1588  return 0;
1589 }
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
#define gettext_noop(String)
Definition: gettext.h:70
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:144
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
static char * filename
uint32_t data
The data value.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
static char * value
Value of the record to add/remove.
enum State state
current state of profiling
static unsigned int next_search
Index of peer to start next announce/search.
static char * data_filename
Filename to log statistics to.
static struct GNUNET_SCHEDULER_Task * abort_task
Abort task identifier.
static long long unsigned int init_parallel_searches
How many searches are we going to start in parallel.
static void stats_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
Function called by testbed once we are connected to stats service.
static int stats_iterator(void *cls, const char *subsystem, const char *name, uint64_t value, int is_persistent)
Process statistic values.
static int count_and_separate_strings(char *data, uint64_t data_size, unsigned int str_max)
Process the text buffer counting the non-empty lines and separating them with NULL characters,...
static void stats_cb(void *cls, int success)
Stats callback.
static struct GNUNET_TESTBED_Controller * mc
Handle to the master controller.
static struct GNUNET_TIME_Relative reannounce_period_max
What's the maximum regex reannounce period.
static char * policy_dir
Folder where policy files are stored.
static unsigned int parallel_searches
How many searches are running in parallel.
static struct GNUNET_TIME_Relative prof_time
Duration profiling step has taken.
static void stats_da(void *cls, void *op_result)
Adapter function called to destroy a connection to statistics service.
static void test_master(void *cls, struct GNUNET_TESTBED_RunHandle *h, unsigned int num_peers_, struct GNUNET_TESTBED_Peer **testbed_peers, unsigned int links_succeeded, unsigned int links_failed)
Signature of a main function for a testcase.
State
Available states during profiling.
@ STATE_PEERS_CREATING
Creating peers.
@ STATE_SLAVES_STARTING
Starting slaves.
@ STATE_INIT
Initial state.
@ STATE_PEERS_LINKING
Linking peers.
@ STATE_SEARCH_REGEX
Matching strings against announced regexes.
@ STATE_PEERS_STARTING
Starting peers.
@ STATE_PEERS_DESTROYING
Destroying peers; we can do this as the controller takes care of stopping a peer if it is running.
static void do_abort(void *cls)
abort task to run on test timed out
static int in_shutdown
Set when shutting down to avoid making more queries.
static struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
static void do_announce(void *cls)
Task to start the daemons on each peer so that the regexes are announced into the DHT.
static struct GNUNET_SCHEDULER_Task * register_hosts_task
Host registration task identifier.
static struct GNUNET_CONFIGURATION_Handle * cfg
Handle to global configuration.
static void do_shutdown(void *cls)
Shutdown nicely.
static uint64_t event_mask
Global event mask for all testbed events.
static char * hosts_file
File with hostnames where to execute the test.
static struct GNUNET_SCHEDULER_Task * search_timeout_task
Search timeout task identifier.
static unsigned int strings_found
Number of strings found in the published regexes.
static int result
Global testing status.
static void announce_next_regex(void *cls)
Start announcing the next regex in the DHT.
static void find_string(void *cls)
Start searching for the next string in the DHT.
static struct GNUNET_TIME_Relative search_timeout_time
Search timeout in seconds.
static char * strings_file
File with the strings to look for.
static void dht_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
DHT connect callback.
static int load_search_strings(const char *filename, char ***strings, unsigned int limit)
Load search strings from given filename.
static struct GNUNET_TESTBED_ControllerProc * mc_proc
Handle to the master controller process.
static int create_string_array(char *data, uint64_t data_size, char ***strings, unsigned int str_cnt)
Allocate a string array and fill it with the prefixed strings from a pre-processed,...
static void dht_da(void *cls, void *op_result)
Adapter function called to destroy a connection to the DHT service.
static void daemon_started(void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
Callback called when testbed has started the daemon we asked for.
static void * dht_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
DHT connect adapter.
static char ** search_strings
Search strings (num_peers of them).
#define FIND_TIMEOUT
static void * stats_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
Adapter function called to establish a connection to statistics service.
static void do_collect_stats(void *cls)
Task to collect all statistics from all peers, will shutdown the profiler, when done.
static void master_controller_cb(void *cls, const struct GNUNET_TESTBED_EventInformation *event)
Function that will be called whenever something in the testbed changes.
int main(int argc, char *const *argv)
Main function.
static struct GNUNET_TESTBED_HostRegistrationHandle * reg_handle
Host registration handle.
static void search_timed_out(void *cls)
Connect by string timeout task.
static void regex_found_handler(void *cls, const struct GNUNET_PeerIdentity *id, const struct GNUNET_DHT_PathElement *get_path, unsigned int get_path_length, const struct GNUNET_DHT_PathElement *put_path, unsigned int put_path_length)
Method called when we've found a peer that announced a regex that matches our search string.
static char * regex_prefix
Prefix used for regex announcing.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
static unsigned int num_peers
Number of peers to be started by the profiler.
static struct GNUNET_DISK_FileHandle * data_file
File to log statistics to.
static struct RegexPeer * peers
The array of peers; we fill this as the peers are given to us by the testbed.
static void find_timed_out(void *cls)
Search timed out.
static struct GNUNET_TESTBED_Peer ** testbed_peers
Handles to all of the running peers.
static char buf[2048]
static const struct GNUNET_CONFIGURATION_Handle * config
static char * subsystem
Set to subsystem that we're going to get stats for (or NULL for all).
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-uri.c:38
Constants for network applications operating on top of the CADET service.
API to the DHT service.
API for writing tests and creating large-scale emulation testbeds for GNUnet.
void GNUNET_CONFIGURATION_set_value_string(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, const char *value)
Set a configuration value that should be a string.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
void GNUNET_CONFIGURATION_destroy(struct GNUNET_CONFIGURATION_Handle *cfg)
Destroy configuration object.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
void GNUNET_CONFIGURATION_set_value_number(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long number)
Set a configuration value that should be a number.
struct GNUNET_CONFIGURATION_Handle * GNUNET_CONFIGURATION_dup(const struct GNUNET_CONFIGURATION_Handle *cfg)
Duplicate an existing configuration object.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
@ GNUNET_CRYPTO_QUALITY_WEAK
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
struct GNUNET_DHT_Handle * GNUNET_DHT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int ht_len)
Initialize the connection with the DHT service.
Definition: dht_api.c:1039
void GNUNET_DHT_disconnect(struct GNUNET_DHT_Handle *handle)
Shutdown connection with the DHT service.
Definition: dht_api.c:1062
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1234
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition: disk.c:481
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:685
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:221
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:403
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1305
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:663
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:813
@ GNUNET_DISK_OPEN_TRUNCATE
Truncate file if it exists.
@ GNUNET_DISK_OPEN_CREATE
Create file if it doesn't exist.
@ GNUNET_DISK_OPEN_READWRITE
Open the file for both reading and writing.
@ GNUNET_DISK_PERM_USER_READ
Owner can read.
@ GNUNET_DISK_PERM_USER_WRITE
Owner can write.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_relative_time(char shortName, const char *name, const char *argumentHelp, const char *description, struct GNUNET_TIME_Relative *val)
Allow user to specify a struct GNUNET_TIME_Relative (using human-readable "fancy" time).
#define GNUNET_log(kind,...)
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
int GNUNET_snprintf(char *buf, size_t size, const char *format,...) __attribute__((format(printf
Like snprintf, just aborts if the buffer is of insufficient size.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:400
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:534
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1268
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:944
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1241
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
struct GNUNET_STATISTICS_GetHandle * GNUNET_STATISTICS_get(struct GNUNET_STATISTICS_Handle *handle, const char *subsystem, const char *name, GNUNET_STATISTICS_Callback cont, GNUNET_STATISTICS_Iterator proc, void *cls)
Get statistic from the peer.
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1222
void GNUNET_TESTBED_operation_done(struct GNUNET_TESTBED_Operation *operation)
This function is used to signal that the event information (struct GNUNET_TESTBED_EventInformation) f...
Definition: testbed_api.c:2021
void GNUNET_TESTBED_cancel_registration(struct GNUNET_TESTBED_HostRegistrationHandle *handle)
Cancel the pending registration.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_peer_manage_service(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_OperationCompletionCallback cb, void *cb_cls, unsigned int start)
Start or stop given service at a peer.
void GNUNET_TESTBED_run(const char *host_filename, const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int num_peers, uint64_t event_mask, GNUNET_TESTBED_ControllerCallback cc, void *cc_cls, GNUNET_TESTBED_TestMaster test_master, void *test_master_cls)
Convenience method for running a testbed with a single call.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_service_connect(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_ServiceConnectCompletionCallback cb, void *cb_cls, GNUNET_TESTBED_ConnectAdapter ca, GNUNET_TESTBED_DisconnectAdapter da, void *cada_cls)
Connect to a service offered by the given peer.
void GNUNET_TESTBED_controller_stop(struct GNUNET_TESTBED_ControllerProc *cproc)
Stop the controller process (also will terminate all peers and controllers dependent on this controll...
void GNUNET_TESTBED_controller_disconnect(struct GNUNET_TESTBED_Controller *c)
Stop the given controller (also will terminate all peers and controllers dependent on this controller...
Definition: testbed_api.c:1721
@ GNUNET_TESTBED_ET_CONNECT
A connection between two peers was established.
@ GNUNET_TESTBED_ET_PEER_START
A peer has been started.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:436
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:531
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:111
#define GNUNET_TIME_UNIT_MINUTES
One minute.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:585
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:484
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:569
static int op_result(struct GNUNET_OP_Handle *h, uint64_t op_id, int64_t result_code, const void *data, uint16_t data_size, void **ctx, uint8_t cancel)
Remove an operation, and call its result callback (unless it was cancelled).
Definition: op.c:246
static unsigned int size
Size of the "table".
Definition: peer.c:68
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
const char * name
struct REGEX_INTERNAL_Search * REGEX_INTERNAL_search(struct GNUNET_DHT_Handle *dht, const char *string, REGEX_INTERNAL_Found callback, void *callback_cls, struct GNUNET_STATISTICS_Handle *stats)
Search for a peer offering a regex matching certain string in the DHT.
void REGEX_INTERNAL_search_cancel(struct REGEX_INTERNAL_Search *h)
Cancel an ongoing regex search in the DHT and free all resources.
library to parse regular expressions into dfa
DLL of operations.
struct DLLOperation * next
The next pointer for DLL.
struct GNUNET_TESTBED_Operation * op
The testbed operation handle.
struct DLLOperation * prev
The prev pointer for DLL.
void * cls
Closure.
Connection to the DHT service.
Definition: dht_api.c:237
A (signed) path tracking a block's flow through the DHT is represented by an array of path elements,...
Handle used to access files (and pipes).
Definition of a command line option.
The identity of the host (wraps the signing key of the peer).
Entry in list of pending tasks.
Definition: scheduler.c:136
Handle for the service.
Handle for controller process.
Handle to interact with a GNUnet testbed controller.
Definition: testbed_api.h:194
Argument to GNUNET_TESTBED_ControllerCallback with details about the event.
enum GNUNET_TESTBED_EventType type
Type of the event.
handle for host registration
Opaque handle to an abstract operation to be executed by the testing framework.
A peer controlled by the testing framework.
Time for absolute times used by GNUnet, in microseconds.
Time for relative time used by GNUnet, in microseconds.
Struct to keep information of searches of services described by a regex using a user-provided string ...
Peer handles.
int search_str_matched
Set to GNUNET_YES if the peer successfully matched the above search string.
const char * search_str
Peer's search string.
struct GNUNET_TESTBED_Operation * daemon_op
Daemon start.
struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
unsigned int id
Peer id.
struct GNUNET_STATISTICS_Handle * stats_handle
Peers's statistics handle.
struct GNUNET_CONFIGURATION_Handle * cfg
Peer configuration handle.
struct GNUNET_TESTBED_Operation * op_handle
Testbed operation handle for DHT.
struct GNUNET_TESTBED_Peer * peer_handle
The actual testbed peer handle.
struct GNUNET_DHT_Handle * dht_handle
Peer's DHT handle.
struct REGEX_INTERNAL_Search * search_handle
Handle to a running regex search.
struct GNUNET_SCHEDULER_Task * timeout
Operation timeout.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.