GNUnet  0.11.x
gnunet-regex-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011 - 2017 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
29 #include <string.h>
30 
31 #include "platform.h"
32 #include "gnunet_applications.h"
33 #include "gnunet_util_lib.h"
34 #include "regex_internal_lib.h"
35 #include "gnunet_arm_service.h"
36 #include "gnunet_dht_service.h"
37 #include "gnunet_testbed_service.h"
38 
39 #define FIND_TIMEOUT \
40  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90)
41 
46 {
51 
55  void *cls;
56 
60  struct DLLOperation *next;
61 
65  struct DLLOperation *prev;
66 };
67 
68 
72 enum State
73 {
78 
83 
88 
93 
98 
103 
109 };
110 
111 
115 struct RegexPeer
116 {
120  unsigned int id;
121 
126 
131 
135  const char *search_str;
136 
144 
149 
154 
159 
164 
169 
174 
179 };
180 
184 static int in_shutdown;
185 
189 static struct RegexPeer *peers;
190 
195 
200 
205 
210 
215 
220 
224 static uint64_t event_mask;
225 
230 
234 static struct GNUNET_TIME_Relative prof_time;
235 
239 static unsigned int num_peers;
240 
244 static int result;
245 
249 enum State state;
250 
254 static char *policy_dir;
255 
259 static char *hosts_file;
260 
264 static char *strings_file;
265 
269 static char **search_strings;
270 
274 static long long unsigned int init_parallel_searches;
275 
279 static unsigned int parallel_searches;
280 
284 static unsigned int strings_found;
285 
289 static unsigned int next_search;
290 
295 
299 static struct GNUNET_TIME_Relative search_timeout_time = { 60000 };
300 
305 
309 static char *data_filename;
310 
315 static char *regex_prefix;
316 
321 
322 
323 /******************************************************************************/
324 /****************************** DECLARATIONS ********************************/
325 /******************************************************************************/
326 
335 static void
336 dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op,
337  void *ca_result, const char *emsg);
338 
347 static void *
348 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg);
349 
350 
358 static void
359 dht_da (void *cls, void *op_result);
360 
361 
371 static void
372 stats_connect_cb (void *cls,
374  void *ca_result,
375  const char *emsg);
376 
377 
383 static void
384 announce_next_regex (void *cls);
385 
386 
387 /******************************************************************************/
388 /******************************** SHUTDOWN **********************************/
389 /******************************************************************************/
390 
391 
397 static void
398 do_shutdown (void *cls)
399 {
400  struct RegexPeer *peer;
401  unsigned int peer_cnt;
402  unsigned int search_str_cnt;
403  char output_buffer[512];
404  size_t size;
405 
406  if (NULL != abort_task)
407  {
409  abort_task = NULL;
410  }
411  if (NULL != register_hosts_task)
412  {
414  register_hosts_task = NULL;
415  }
416  for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++)
417  {
418  peer = &peers[peer_cnt];
419 
420  if ((GNUNET_YES != peer->search_str_matched) && (NULL != data_file) )
421  {
422  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
423  size =
424  GNUNET_snprintf (output_buffer,
425  sizeof(output_buffer),
426  "%p Search string not found: %s (%d)\n"
427  "%p On peer: %u (%p)\n"
428  "%p After: %s\n",
429  peer, peer->search_str, peer->search_str_matched,
430  peer, peer->id, peer,
431  peer,
433  GNUNET_NO));
434  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
435  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
436  }
437 
438  if (NULL != peers[peer_cnt].op_handle)
440  }
441 
442  if (NULL != data_file)
443  {
445  data_file = NULL;
446  }
447  for (search_str_cnt = 0;
448  search_str_cnt < num_peers && NULL != search_strings;
449  search_str_cnt++)
450  {
451  GNUNET_free (search_strings[search_str_cnt]);
452  }
454  search_strings = NULL;
455 
456  if (NULL != reg_handle)
457  {
459  reg_handle = NULL;
460  }
461  if (NULL != mc)
462  {
464  mc = NULL;
465  }
466  if (NULL != mc_proc)
467  {
469  mc_proc = NULL;
470  }
471  if (NULL != cfg)
472  {
474  cfg = NULL;
475  }
476 }
477 
478 
484 static void
485 do_abort (void *cls)
486 {
487  unsigned long i = (unsigned long) cls;
488 
490  "Aborting from line %lu...\n", i);
491  abort_task = NULL;
494 }
495 
496 
497 /******************************************************************************/
498 /********************* STATISTICS SERVICE CONNECTIONS ***********************/
499 /******************************************************************************/
500 
511 static void *
512 stats_ca (void *cls,
513  const struct GNUNET_CONFIGURATION_Handle *cfg)
514 {
515  return GNUNET_STATISTICS_create ("<driver>", cfg);
516 }
517 
518 
526 static void
527 stats_da (void *cls, void *op_result)
528 {
529  struct RegexPeer *peer = cls;
530 
531  GNUNET_assert (op_result == peer->stats_handle);
532 
533  GNUNET_STATISTICS_destroy (peer->stats_handle, GNUNET_NO);
534  peer->stats_handle = NULL;
535 }
536 
537 
548 static int
549 stats_iterator (void *cls,
550  const char *subsystem,
551  const char *name,
552  uint64_t value, int is_persistent)
553 {
554  struct RegexPeer *peer = cls;
555  char output_buffer[512];
556  size_t size;
557 
558  if (NULL == data_file)
559  {
561  "%p -> %s [%s]: %llu\n",
562  peer,
563  subsystem,
564  name,
565  (unsigned long long) value);
566  return GNUNET_OK;
567  }
568  size =
569  GNUNET_snprintf (output_buffer,
570  sizeof(output_buffer),
571  "%p [%s] %llu %s\n",
572  peer,
573  subsystem,
574  (unsigned long long) value,
575  name);
576  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
578  "Unable to write to file!\n");
579 
580  return GNUNET_OK;
581 }
582 
583 
592 static void
593 stats_cb (void *cls,
594  int success)
595 {
596  static unsigned int peer_cnt;
597  struct RegexPeer *peer = cls;
598 
599  if (GNUNET_OK != success)
600  {
602  "Getting statistics for peer %u failed!\n",
603  peer->id);
604  return;
605  }
606 
607  GNUNET_assert (NULL != peer->op_handle);
608 
609  GNUNET_TESTBED_operation_done (peer->op_handle);
610  peer->op_handle = NULL;
611 
612  peer_cnt++;
613  peer = &peers[peer_cnt];
614 
615  fprintf (stderr, "s");
616  if (peer_cnt == num_peers)
617  {
619  "\nCollecting stats finished. Shutting down.\n");
621  result = GNUNET_OK;
622  }
623  else
624  {
625  peer->op_handle =
627  peer->peer_handle,
628  "statistics",
630  peer,
631  &stats_ca,
632  &stats_da,
633  peer);
634  }
635 }
636 
637 
647 static void
648 stats_connect_cb (void *cls,
650  void *ca_result,
651  const char *emsg)
652 {
653  struct RegexPeer *peer = cls;
654 
655  if ((NULL == ca_result) || (NULL != emsg))
656  {
658  "Failed to connect to statistics service on peer %u: %s\n",
659  peer->id, emsg);
660 
661  peer->stats_handle = NULL;
662  return;
663  }
664 
665  peer->stats_handle = ca_result;
666 
667  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
668  &stats_cb,
669  &stats_iterator, peer))
670  {
672  "Could not get statistics of peer %u!\n", peer->id);
673  }
674 }
675 
676 
683 static void
684 do_collect_stats (void *cls)
685 {
686  struct RegexPeer *peer = &peers[0];
687 
688  GNUNET_assert (NULL != peer->peer_handle);
689 
690  peer->op_handle =
692  peer->peer_handle,
693  "statistics",
695  peer,
696  &stats_ca,
697  &stats_da,
698  peer);
699 }
700 
701 
702 /******************************************************************************/
703 /************************ REGEX FIND CONNECTIONS **************************/
704 /******************************************************************************/
705 
706 
712 static void
713 find_string (void *cls);
714 
715 
727 static void
729  const struct GNUNET_PeerIdentity *id,
730  const struct GNUNET_PeerIdentity *get_path,
731  unsigned int get_path_length,
732  const struct GNUNET_PeerIdentity *put_path,
733  unsigned int put_path_length)
734 {
735  struct RegexPeer *peer = cls;
736  char output_buffer[512];
737  size_t size;
738 
739  if (GNUNET_YES == peer->search_str_matched)
740  {
742  "String %s on peer %u already matched!\n",
743  peer->search_str, peer->id);
744  return;
745  }
746 
747  strings_found++;
749 
750  if (NULL != peer->timeout)
751  {
752  GNUNET_SCHEDULER_cancel (peer->timeout);
753  peer->timeout = NULL;
754  if (GNUNET_NO == in_shutdown)
756  }
757 
758  if (NULL == id)
759  {
760  // FIXME not possible right now
762  "String matching timed out for string %s on peer %u (%i/%i)\n",
763  peer->search_str, peer->id, strings_found, num_peers);
764  peer->search_str_matched = GNUNET_SYSERR;
765  }
766  else
767  {
768  prof_time = GNUNET_TIME_absolute_get_duration (peer->prof_start_time);
769 
771  "String %s found on peer %u after %s (%i/%i) (%u||)\n",
772  peer->search_str, peer->id,
775 
776  peer->search_str_matched = GNUNET_YES;
777 
778  if (NULL != data_file)
779  {
780  size =
781  GNUNET_snprintf (output_buffer,
782  sizeof(output_buffer),
783  "%p Peer: %u\n"
784  "%p Search string: %s\n"
785  "%p Search duration: %s\n\n",
786  peer, peer->id,
787  peer, peer->search_str,
788  peer,
790  GNUNET_NO));
791 
792  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
793  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
794  }
795  }
796 
797  GNUNET_TESTBED_operation_done (peer->op_handle);
798  peer->op_handle = NULL;
799 
800  if (strings_found == num_peers)
801  {
804  "All strings successfully matched in %s\n",
806 
807  if (NULL != search_timeout_task)
808  {
810  search_timeout_task = NULL;
811  }
812 
813  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n");
815  }
816 }
817 
818 
825 static void
826 search_timed_out (void *cls)
827 {
828  unsigned int i;
829 
831  "Finding matches to all strings did not succeed after %s.\n",
833  GNUNET_NO));
835  "Found %i of %i strings\n", strings_found, num_peers);
836 
838  "Search timed out after %s."
839  "Collecting stats and shutting down.\n",
841  GNUNET_NO));
842 
844  for (i = 0; i < num_peers; i++)
845  {
846  if (NULL != peers[i].op_handle)
847  {
849  peers[i].op_handle = NULL;
850  }
851  }
853 }
854 
855 
862 static void
863 find_timed_out (void *cls)
864 {
865  struct RegexPeer *p = cls;
866 
867  p->timeout = NULL;
869  "Searching for string \"%s\" on peer %d timed out.\n",
870  p->search_str,
871  p->id);
872  if (GNUNET_NO == in_shutdown)
874 }
875 
876 
882 static void
883 find_string (void *cls)
884 {
885  unsigned int search_peer = (unsigned int) (long) cls;
886 
887  if ((search_peer >= num_peers) ||
888  (GNUNET_YES == in_shutdown))
889  return;
890 
892  "Searching for string \"%s\" on peer %d (%u||)\n",
893  peers[search_peer].search_str,
894  search_peer,
896 
897  peers[search_peer].op_handle =
899  peers[search_peer].peer_handle,
900  "dht",
902  &peers[search_peer],
903  &dht_ca,
904  &dht_da,
905  &peers[search_peer]);
906  GNUNET_assert (NULL != peers[search_peer].op_handle);
907  peers[search_peer].timeout
910  &peers[search_peer]);
911 }
912 
913 
921 static void
922 daemon_started (void *cls,
924  const char *emsg)
925 {
926  struct RegexPeer *peer = (struct RegexPeer *) cls;
927  unsigned long search_peer;
928  unsigned int i;
929 
930  GNUNET_TESTBED_operation_done (peer->daemon_op);
931  peer->daemon_op = NULL;
932  if (NULL != emsg)
933  {
935  "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg);
936  GNUNET_assert (0);
937  }
938  else
939  {
941  "Daemon %u started successfully\n", peer->id);
942  }
943 
944  /* Find a peer to look for a string matching the regex announced */
946  num_peers);
947  for (i = 0; peers[search_peer].search_str != NULL; i++)
948  {
949  search_peer = (search_peer + 1) % num_peers;
950  if (i > num_peers)
951  GNUNET_assert (0); /* we ran out of peers, must be a bug */
952  }
953  peers[search_peer].search_str = search_strings[peer->id];
954  peers[search_peer].search_str_matched = GNUNET_NO;
957  2),
958  &find_string,
959  (void *) search_peer);
960 }
961 
962 
970 static void
971 do_announce (void *cls)
972 {
973  unsigned int i;
974 
975  if (GNUNET_YES == in_shutdown)
976  return;
978  "Starting announce.\n");
979  for (i = 0; i < init_parallel_searches; i++)
980  {
982  " scheduling announce %u\n",
983  i);
985  }
986 }
987 
988 
994 static void
996 {
997  struct RegexPeer *peer;
998 
999  if (GNUNET_YES == in_shutdown)
1000  return;
1001  if (next_search >= num_peers)
1002  {
1003  if (strings_found != num_peers)
1004  {
1005  struct GNUNET_TIME_Relative new_delay;
1006  if (NULL != search_timeout_task)
1011  NULL);
1012  }
1013  return;
1014  }
1015 
1016  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search);
1017  peer = &peers[next_search];
1018  peer->daemon_op =
1020  peer->peer_handle,
1021  "regexprofiler",
1022  &daemon_started,
1023  peer,
1024  1);
1025  next_search++;
1027 }
1028 
1029 
1040 static void
1041 dht_connect_cb (void *cls,
1042  struct GNUNET_TESTBED_Operation *op,
1043  void *ca_result,
1044  const char *emsg)
1045 {
1046  struct RegexPeer *peer = (struct RegexPeer *) cls;
1047 
1048  if ((NULL != emsg) || (NULL == op) || (NULL == ca_result))
1049  {
1050  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg);
1051  GNUNET_assert (0);
1052  }
1053 
1054  GNUNET_assert (NULL != peer->dht_handle);
1055  GNUNET_assert (peer->op_handle == op);
1056  GNUNET_assert (peer->dht_handle == ca_result);
1057 
1058  peer->search_str_matched = GNUNET_NO;
1059  peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle,
1060  peer->search_str,
1062  NULL);
1063  peer->prof_start_time = GNUNET_TIME_absolute_get ();
1064 }
1065 
1066 
1075 static void *
1076 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
1077 {
1078  struct RegexPeer *peer = cls;
1079 
1080  peer->dht_handle = GNUNET_DHT_connect (cfg, 32);
1081 
1082  return peer->dht_handle;
1083 }
1084 
1085 
1092 static void
1093 dht_da (void *cls, void *op_result)
1094 {
1095  struct RegexPeer *peer = (struct RegexPeer *) cls;
1096 
1097  GNUNET_assert (peer->dht_handle == op_result);
1098 
1099  if (NULL != peer->search_handle)
1100  {
1101  REGEX_INTERNAL_search_cancel (peer->search_handle);
1102  peer->search_handle = NULL;
1103  }
1104 
1105  if (NULL != peer->dht_handle)
1106  {
1107  GNUNET_DHT_disconnect (peer->dht_handle);
1108  peer->dht_handle = NULL;
1109  }
1110 }
1111 
1112 
1126 static void
1127 test_master (void *cls,
1128  struct GNUNET_TESTBED_RunHandle *h,
1129  unsigned int num_peers_,
1131  unsigned int links_succeeded,
1132  unsigned int links_failed)
1133 {
1134  unsigned int i;
1135 
1136  GNUNET_assert (num_peers_ == num_peers);
1137 
1140  "Testbed started in %s\n",
1142 
1143  if (NULL != abort_task)
1144  {
1146  abort_task = NULL;
1147  }
1148 
1149  for (i = 0; i < num_peers; i++)
1150  {
1152  }
1153  if (GNUNET_NO ==
1154  GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT"))
1155  {
1156  struct GNUNET_TIME_Relative settle_time;
1157 
1158  settle_time =
1160  10 * num_peers);
1162  "Waiting for DHT for %s to settle new connections.\n\n",
1164  GNUNET_NO));
1165  GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL);
1166  }
1167  else
1168  {
1170  }
1173 }
1174 
1175 
1182 static void
1184  const struct GNUNET_TESTBED_EventInformation *event)
1185 {
1186  switch (event->type)
1187  {
1189  printf (".");
1190  break;
1191 
1193  printf ("#");
1194  break;
1195 
1196  default:
1197  break;
1198  }
1199  fflush (stdout);
1200 }
1201 
1202 
1203 /******************************************************************************/
1204 /*************************** TESTBED PEER SETUP *****************************/
1205 /******************************************************************************/
1206 
1217 static int
1219  uint64_t data_size,
1220  unsigned int str_max)
1221 {
1222  char *buf; // Keep track of last string to skip blank lines
1223  unsigned int offset;
1224  unsigned int str_cnt;
1225 
1226  buf = data;
1227  offset = 0;
1228  str_cnt = 0;
1229  while ((offset < (data_size - 1)) && (str_cnt < str_max))
1230  {
1231  offset++;
1232  if (((data[offset] == '\n')) &&
1233  (buf != &data[offset]))
1234  {
1235  data[offset] = '\0';
1236  str_cnt++;
1237  buf = &data[offset + 1];
1238  }
1239  else if ((data[offset] == '\n') ||
1240  (data[offset] == '\0'))
1241  buf = &data[offset + 1];
1242  }
1243  return str_cnt;
1244 }
1245 
1246 
1260 static int
1262  char ***strings, unsigned int str_cnt)
1263 {
1264  uint64_t offset;
1265  uint64_t len;
1266  unsigned int i;
1267 
1268  *strings = GNUNET_malloc (sizeof(char *) * str_cnt);
1269  offset = 0;
1270  for (i = 0; i < str_cnt; i++)
1271  {
1272  len = strlen (&data[offset]);
1273  if (offset + len >= data_size)
1274  {
1275  GNUNET_free (*strings);
1276  *strings = NULL;
1277  return GNUNET_SYSERR;
1278  }
1279  if (0 == len) // empty line
1280  {
1281  offset++;
1282  i--;
1283  continue;
1284  }
1285 
1286  GNUNET_asprintf (&(*strings)[i],
1287  "%s%s",
1288  regex_prefix,
1289  &data[offset]);
1290  offset += len + 1;
1291  }
1292  return GNUNET_OK;
1293 }
1294 
1295 
1305 static int
1307  char ***strings,
1308  unsigned int limit)
1309 {
1310  char *data;
1311  uint64_t filesize;
1312  int str_cnt;
1313 
1314  /* Sanity checks */
1315  if (NULL == filename)
1316  {
1317  return GNUNET_SYSERR;
1318  }
1320  {
1322  "Could not find search strings file %s\n", filename);
1323  return GNUNET_SYSERR;
1324  }
1325  if (GNUNET_OK !=
1327  &filesize,
1328  GNUNET_YES,
1329  GNUNET_YES))
1330  {
1332  "Search strings file %s cannot be read.\n",
1333  filename);
1334  return GNUNET_SYSERR;
1335  }
1336  if (0 == filesize)
1337  {
1339  "Search strings file %s is empty.\n",
1340  filename);
1341  return GNUNET_SYSERR;
1342  }
1343 
1344  /* Read data into memory */
1345  data = GNUNET_malloc (filesize + 1);
1346  if (filesize != GNUNET_DISK_fn_read (filename,
1347  data,
1348  filesize))
1349  {
1350  GNUNET_free (data);
1352  "Could not read search strings file %s.\n",
1353  filename);
1354  return GNUNET_SYSERR;
1355  }
1356 
1357  /* Process buffer and build array */
1358  str_cnt = count_and_separate_strings (data, filesize, limit);
1359  if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt))
1360  {
1361  str_cnt = GNUNET_SYSERR;
1362  }
1363  GNUNET_free (data);
1364  return str_cnt;
1365 }
1366 
1367 
1376 static void
1377 run (void *cls,
1378  char *const *args,
1379  const char *cfgfile,
1380  const struct GNUNET_CONFIGURATION_Handle *config)
1381 {
1382  unsigned int nsearchstrs;
1383  unsigned int i;
1384  struct GNUNET_TIME_Relative abort_time;
1385 
1387 
1388  /* Check config */
1389  if (NULL == config)
1390  {
1392  _ ("No configuration file given. Exiting\n"));
1394  return;
1395  }
1397  if (GNUNET_OK !=
1398  GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER",
1399  "REGEX_PREFIX",
1400  &regex_prefix))
1401  {
1403  "regexprofiler",
1404  "regex_prefix");
1406  return;
1407  }
1408  if (GNUNET_OK !=
1409  GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER",
1410  "PARALLEL_SEARCHES",
1412  {
1414  "Configuration option \"PARALLEL_SEARCHES\" missing."
1415  " Using default (%d)\n", 10);
1417  }
1418  if (GNUNET_OK !=
1419  GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER",
1420  "REANNOUNCE_PERIOD_MAX",
1422  {
1424  "reannounce_period_max not given. Using 10 minutes.\n");
1427  }
1428 
1429  /* Check arguments */
1430  if (NULL == policy_dir)
1431  {
1433  _ (
1434  "No policy directory specified on command line. Exiting.\n"));
1435  return;
1436  }
1438  {
1440  _ ("Specified policies directory does not exist. Exiting.\n"));
1442  return;
1443  }
1444  if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL,
1445  NULL)))
1446  {
1448  _ ("No files found in `%s'\n"),
1449  policy_dir);
1450  return;
1451  }
1452  GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER",
1453  "POLICY_DIR", policy_dir);
1455  {
1457  _ ("No search strings file given. Exiting.\n"));
1459  return;
1460  }
1461  nsearchstrs = load_search_strings (strings_file,
1462  &search_strings,
1463  num_peers);
1464  if (num_peers != nsearchstrs)
1465  {
1467  "Error loading search strings.\n");
1469  "File (%s) does not contain enough strings (%u/%u).\n",
1470  strings_file, nsearchstrs, num_peers);
1472  return;
1473  }
1474  if ((0 == num_peers) || (NULL == search_strings))
1475  {
1477  _ ("Error loading search strings. Exiting.\n"));
1479  return;
1480  }
1481  for (i = 0; i < num_peers; i++)
1483  "search string: %s\n",
1484  search_strings[i]);
1485 
1486  /* Check logfile */
1487  if ((NULL != data_filename) &&
1488  (NULL == (data_file =
1495  {
1497  "open",
1498  data_filename);
1499  return;
1500  }
1501 
1502  /* Initialize peers */
1503  peers = GNUNET_malloc (sizeof(struct RegexPeer) * num_peers);
1504  for (i = 0; i < num_peers; i++)
1505  peers[i].id = i;
1506 
1508  "TESTBED", "OVERLAY_RANDOM_LINKS",
1509  num_peers * 20);
1511  "DHT", "FORCE_NSE",
1512  (long long unsigned)
1513  (log (num_peers) / log (2.0)));
1514  event_mask = 0LL;
1515 /* For feedback about the start process activate these and pass master_cb */
1517 // event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
1519 // event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);
1522  cfg,
1523  num_peers,
1524  event_mask,
1526  NULL, /* master_controller_cb cls */
1527  &test_master,
1528  NULL); /* test_master cls */
1529  if (GNUNET_OK !=
1531  "SETUP_TIMEOUT",
1532  &abort_time))
1533  {
1535  "SETUP_TIMEOUT not given. Using 15 minutes.\n");
1536  abort_time =
1538  }
1539  abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES);
1540  abort_task =
1541  GNUNET_SCHEDULER_add_delayed (abort_time,
1542  &do_abort,
1543  (void *) __LINE__);
1545  "setup_timeout: %s\n",
1547 }
1548 
1549 
1557 int
1558 main (int argc, char *const *argv)
1559 {
1562  "output-file",
1563  "FILENAME",
1564  gettext_noop (
1565  "name of the file for writing statistics"),
1566  &data_filename),
1567 
1569  "matching-timeout",
1570  "TIMEOUT",
1571  gettext_noop (
1572  "wait TIMEOUT before ending the experiment"),
1574 
1576  "policy-dir",
1577  "DIRECTORY",
1578  gettext_noop ("directory with policy files"),
1579  &policy_dir),
1580 
1581 
1583  "strings-file",
1584  "FILENAME",
1585  gettext_noop (
1586  "name of file with input strings"),
1587  &strings_file),
1588 
1590  "hosts-file",
1591  "FILENAME",
1592  gettext_noop (
1593  "name of file with hosts' names"),
1594  &hosts_file),
1595 
1597  };
1598  int ret;
1599 
1600  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
1601  return 2;
1603  ret =
1604  GNUNET_PROGRAM_run (argc, argv,
1605  "gnunet-regex-profiler",
1606  _ ("Profiler for regex"),
1607  options, &run, NULL);
1608  if (GNUNET_OK != ret)
1609  return ret;
1610  if (GNUNET_OK != result)
1611  return 1;
1612  return 0;
1613 }
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
#define gettext_noop(String)
Definition: gettext.h:69
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
static struct GNUNET_ARM_Operation * op
Current operation.
Definition: gnunet-arm.c:144
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
static char * filename
uint32_t data
The data value.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
static char * value
Value of the record to add/remove.
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-qr.c:59
enum State state
current state of profiling
static unsigned int next_search
Index of peer to start next announce/search.
static char * data_filename
Filename to log statistics to.
static struct GNUNET_SCHEDULER_Task * abort_task
Abort task identifier.
static long long unsigned int init_parallel_searches
How many searches are we going to start in parallel.
static void stats_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
Function called by testbed once we are connected to stats service.
static int stats_iterator(void *cls, const char *subsystem, const char *name, uint64_t value, int is_persistent)
Process statistic values.
static int count_and_separate_strings(char *data, uint64_t data_size, unsigned int str_max)
Process the text buffer counting the non-empty lines and separating them with NULL characters,...
static void stats_cb(void *cls, int success)
Stats callback.
static struct GNUNET_TESTBED_Controller * mc
Handle to the master controller.
static struct GNUNET_TIME_Relative reannounce_period_max
What's the maximum regex reannounce period.
static char * policy_dir
Folder where policy files are stored.
static unsigned int parallel_searches
How many searches are running in parallel.
static struct GNUNET_TIME_Relative prof_time
Duration profiling step has taken.
static void stats_da(void *cls, void *op_result)
Adapter function called to destroy a connection to statistics service.
static void test_master(void *cls, struct GNUNET_TESTBED_RunHandle *h, unsigned int num_peers_, struct GNUNET_TESTBED_Peer **testbed_peers, unsigned int links_succeeded, unsigned int links_failed)
Signature of a main function for a testcase.
State
Available states during profiling.
@ STATE_PEERS_CREATING
Creating peers.
@ STATE_SLAVES_STARTING
Starting slaves.
@ STATE_INIT
Initial state.
@ STATE_PEERS_LINKING
Linking peers.
@ STATE_SEARCH_REGEX
Matching strings against announced regexes.
@ STATE_PEERS_STARTING
Starting peers.
@ STATE_PEERS_DESTROYING
Destroying peers; we can do this as the controller takes care of stopping a peer if it is running.
static void do_abort(void *cls)
abort task to run on test timed out
static int in_shutdown
Set when shutting down to avoid making more queries.
static struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
static void do_announce(void *cls)
Task to start the daemons on each peer so that the regexes are announced into the DHT.
static struct GNUNET_SCHEDULER_Task * register_hosts_task
Host registration task identifier.
static struct GNUNET_CONFIGURATION_Handle * cfg
Handle to global configuration.
static void do_shutdown(void *cls)
Shutdown nicely.
static uint64_t event_mask
Global event mask for all testbed events.
static char * hosts_file
File with hostnames where to execute the test.
static struct GNUNET_SCHEDULER_Task * search_timeout_task
Search timeout task identifier.
static unsigned int strings_found
Number of strings found in the published regexes.
static int result
Global testing status.
static void announce_next_regex(void *cls)
Start announcing the next regex in the DHT.
static void find_string(void *cls)
Start searching for the next string in the DHT.
static struct GNUNET_TIME_Relative search_timeout_time
Search timeout in seconds.
static char * strings_file
File with the strings to look for.
static void dht_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
DHT connect callback.
static int load_search_strings(const char *filename, char ***strings, unsigned int limit)
Load search strings from given filename.
static struct GNUNET_TESTBED_ControllerProc * mc_proc
Handle to the master controller process.
static int create_string_array(char *data, uint64_t data_size, char ***strings, unsigned int str_cnt)
Allocate a string array and fill it with the prefixed strings from a pre-processed,...
static void dht_da(void *cls, void *op_result)
Adapter function called to destroy a connection to the DHT service.
static void daemon_started(void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
Callback called when testbed has started the daemon we asked for.
static void * dht_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
DHT connect adapter.
static char ** search_strings
Search strings (num_peers of them).
#define FIND_TIMEOUT
static void * stats_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
Adapter function called to establish a connection to statistics service.
static void do_collect_stats(void *cls)
Task to collect all statistics from all peers, will shutdown the profiler, when done.
static void master_controller_cb(void *cls, const struct GNUNET_TESTBED_EventInformation *event)
Function that will be called whenever something in the testbed changes.
static void regex_found_handler(void *cls, const struct GNUNET_PeerIdentity *id, const struct GNUNET_PeerIdentity *get_path, unsigned int get_path_length, const struct GNUNET_PeerIdentity *put_path, unsigned int put_path_length)
Method called when we've found a peer that announced a regex that matches our search string.
int main(int argc, char *const *argv)
Main function.
static struct GNUNET_TESTBED_HostRegistrationHandle * reg_handle
Host registration handle.
static void search_timed_out(void *cls)
Connect by string timeout task.
static char * regex_prefix
Prefix used for regex announcing.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
static unsigned int num_peers
Number of peers to be started by the profiler.
static struct GNUNET_DISK_FileHandle * data_file
File to log statistics to.
static struct RegexPeer * peers
The array of peers; we fill this as the peers are given to us by the testbed.
static void find_timed_out(void *cls)
Search timed out.
static struct GNUNET_TESTBED_Peer ** testbed_peers
Handles to all of the running peers.
static char buf[2048]
static const struct GNUNET_CONFIGURATION_Handle * config
static char * subsystem
Set to subsystem that we're going to get stats for (or NULL for all).
#define GNUNET_log(kind,...)
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
void GNUNET_CONFIGURATION_set_value_string(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, const char *value)
Set a configuration value that should be a string.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
void GNUNET_CONFIGURATION_destroy(struct GNUNET_CONFIGURATION_Handle *cfg)
Destroy configuration object.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
void GNUNET_CONFIGURATION_set_value_number(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long number)
Set a configuration value that should be a number.
struct GNUNET_CONFIGURATION_Handle * GNUNET_CONFIGURATION_dup(const struct GNUNET_CONFIGURATION_Handle *cfg)
Duplicate an existing configuration object.
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
@ GNUNET_CRYPTO_QUALITY_WEAK
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
struct GNUNET_DHT_Handle * GNUNET_DHT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int ht_len)
Initialize the connection with the DHT service.
Definition: dht_api.c:890
void GNUNET_DHT_disconnect(struct GNUNET_DHT_Handle *handle)
Shutdown connection with the DHT service.
Definition: dht_api.c:918
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1235
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition: disk.c:482
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:686
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:222
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:404
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1306
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:664
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:814
@ GNUNET_DISK_OPEN_TRUNCATE
Truncate file if it exists.
@ GNUNET_DISK_OPEN_CREATE
Create file if it doesn't exist.
@ GNUNET_DISK_OPEN_READWRITE
Open the file for both reading and writing.
@ GNUNET_DISK_PERM_USER_READ
Owner can read.
@ GNUNET_DISK_PERM_USER_WRITE
Owner can write.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_relative_time(char shortName, const char *name, const char *argumentHelp, const char *description, struct GNUNET_TIME_Relative *val)
Allow user to specify a struct GNUNET_TIME_Relative (using human-readable "fancy" time).
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
int GNUNET_snprintf(char *buf, size_t size, const char *format,...) __attribute__((format(printf
Like snprintf, just aborts if the buffer is of insufficient size.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:364
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:531
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1296
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:972
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1269
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
struct GNUNET_STATISTICS_GetHandle * GNUNET_STATISTICS_get(struct GNUNET_STATISTICS_Handle *handle, const char *subsystem, const char *name, GNUNET_STATISTICS_Callback cont, GNUNET_STATISTICS_Iterator proc, void *cls)
Get statistic from the peer.
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1209
void GNUNET_TESTBED_operation_done(struct GNUNET_TESTBED_Operation *operation)
This function is used to signal that the event information (struct GNUNET_TESTBED_EventInformation) f...
Definition: testbed_api.c:2045
void GNUNET_TESTBED_cancel_registration(struct GNUNET_TESTBED_HostRegistrationHandle *handle)
Cancel the pending registration.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_peer_manage_service(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_OperationCompletionCallback cb, void *cb_cls, unsigned int start)
Start or stop given service at a peer.
void GNUNET_TESTBED_run(const char *host_filename, const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int num_peers, uint64_t event_mask, GNUNET_TESTBED_ControllerCallback cc, void *cc_cls, GNUNET_TESTBED_TestMaster test_master, void *test_master_cls)
Convenience method for running a testbed with a single call.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_service_connect(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_ServiceConnectCompletionCallback cb, void *cb_cls, GNUNET_TESTBED_ConnectAdapter ca, GNUNET_TESTBED_DisconnectAdapter da, void *cada_cls)
Connect to a service offered by the given peer.
void GNUNET_TESTBED_controller_stop(struct GNUNET_TESTBED_ControllerProc *cproc)
Stop the controller process (also will terminate all peers and controllers dependent on this controll...
void GNUNET_TESTBED_controller_disconnect(struct GNUNET_TESTBED_Controller *c)
Stop the given controller (also will terminate all peers and controllers dependent on this controller...
Definition: testbed_api.c:1721
@ GNUNET_TESTBED_ET_CONNECT
A connection between two peers was established.
@ GNUNET_TESTBED_ET_PEER_START
A peer has been started.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:263
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:357
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:86
#define GNUNET_TIME_UNIT_MINUTES
One minute.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:411
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:311
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:557
static int op_result(struct GNUNET_OP_Handle *h, uint64_t op_id, int64_t result_code, const void *data, uint16_t data_size, void **ctx, uint8_t cancel)
Remove an operation, and call its result callback (unless it was cancelled).
Definition: op.c:245
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
const char * name
struct REGEX_INTERNAL_Search * REGEX_INTERNAL_search(struct GNUNET_DHT_Handle *dht, const char *string, REGEX_INTERNAL_Found callback, void *callback_cls, struct GNUNET_STATISTICS_Handle *stats)
Search for a peer offering a regex matching certain string in the DHT.
void REGEX_INTERNAL_search_cancel(struct REGEX_INTERNAL_Search *h)
Cancel an ongoing regex search in the DHT and free all resources.
library to parse regular expressions into dfa
DLL of operations.
struct DLLOperation * next
The next pointer for DLL.
struct GNUNET_TESTBED_Operation * op
The testbed operation handle.
struct DLLOperation * prev
The prev pointer for DLL.
void * cls
Closure.
Connection to the DHT service.
Definition: dht_api.c:202
Handle used to access files (and pipes).
Definition of a command line option.
The identity of the host (wraps the signing key of the peer).
Entry in list of pending tasks.
Definition: scheduler.c:135
Handle for the service.
Handle for controller process.
Handle to interact with a GNUnet testbed controller.
Definition: testbed_api.h:194
Argument to GNUNET_TESTBED_ControllerCallback with details about the event.
enum GNUNET_TESTBED_EventType type
Type of the event.
handle for host registration
Opaque handle to an abstract operation to be executed by the testing framework.
A peer controlled by the testing framework.
Time for absolute times used by GNUnet, in microseconds.
Time for relative time used by GNUnet, in microseconds.
Struct to keep information of searches of services described by a regex using a user-provided string ...
Peer handles.
int search_str_matched
Set to GNUNET_YES if the peer successfully matched the above search string.
const char * search_str
Peer's search string.
struct GNUNET_TESTBED_Operation * daemon_op
Daemon start.
struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
unsigned int id
Peer id.
struct GNUNET_STATISTICS_Handle * stats_handle
Peers's statistics handle.
struct GNUNET_CONFIGURATION_Handle * cfg
Peer configuration handle.
struct GNUNET_TESTBED_Operation * op_handle
Testbed operation handle for DHT.
struct GNUNET_TESTBED_Peer * peer_handle
The actual testbed peer handle.
struct GNUNET_DHT_Handle * dht_handle
Peer's DHT handle.
struct REGEX_INTERNAL_Search * search_handle
Handle to a running regex search.
struct GNUNET_SCHEDULER_Task * timeout
Operation timeout.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.