GNUnet  0.11.x
gnunet-regex-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011 - 2017 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
29 #include <string.h>
30 
31 #include "platform.h"
32 #include "gnunet_applications.h"
33 #include "gnunet_util_lib.h"
34 #include "regex_internal_lib.h"
35 #include "gnunet_arm_service.h"
36 #include "gnunet_dht_service.h"
37 #include "gnunet_testbed_service.h"
38 
39 #define FIND_TIMEOUT \
40  GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 90)
41 
46 {
51 
55  void *cls;
56 
60  struct DLLOperation *next;
61 
65  struct DLLOperation *prev;
66 };
67 
68 
72 enum State
73 {
78 
83 
88 
93 
98 
103 
109 };
110 
111 
115 struct RegexPeer
116 {
120  unsigned int id;
121 
126 
131 
135  const char *search_str;
136 
144 
149 
154 
159 
164 
169 
174 
179 };
180 
184 static int in_shutdown;
185 
189 static struct RegexPeer *peers;
190 
195 
200 
205 
210 
215 
220 
224 static uint64_t event_mask;
225 
230 
235 
239 static unsigned int num_peers;
240 
244 static int result;
245 
250 
254 static char *policy_dir;
255 
259 static char *hosts_file;
260 
264 static char *strings_file;
265 
269 static char **search_strings;
270 
274 static long long unsigned int init_parallel_searches;
275 
279 static unsigned int parallel_searches;
280 
284 static unsigned int strings_found;
285 
289 static unsigned int next_search;
290 
295 
299 static struct GNUNET_TIME_Relative search_timeout_time = { 60000 };
300 
305 
309 static char *data_filename;
310 
315 static char *regex_prefix;
316 
321 
322 
323 /******************************************************************************/
324 /****************************** DECLARATIONS ********************************/
325 /******************************************************************************/
326 
335 static void
337  void *ca_result, const char *emsg);
338 
347 static void *
348 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg);
349 
350 
358 static void
359 dht_da (void *cls, void *op_result);
360 
361 
371 static void
372 stats_connect_cb (void *cls,
374  void *ca_result,
375  const char *emsg);
376 
377 
383 static void
384 announce_next_regex (void *cls);
385 
386 
387 /******************************************************************************/
388 /******************************** SHUTDOWN **********************************/
389 /******************************************************************************/
390 
391 
397 static void
399 {
400  struct RegexPeer *peer;
401  unsigned int peer_cnt;
402  unsigned int search_str_cnt;
403  char output_buffer[512];
404  size_t size;
405 
406  if (NULL != abort_task)
407  {
408  GNUNET_SCHEDULER_cancel (abort_task);
409  abort_task = NULL;
410  }
411  if (NULL != register_hosts_task)
412  {
413  GNUNET_SCHEDULER_cancel (register_hosts_task);
414  register_hosts_task = NULL;
415  }
416  for (peer_cnt = 0; peer_cnt < num_peers; peer_cnt++)
417  {
418  peer = &peers[peer_cnt];
419 
420  if ((GNUNET_YES != peer->search_str_matched) && (NULL != data_file) )
421  {
423  size =
424  GNUNET_snprintf (output_buffer,
425  sizeof(output_buffer),
426  "%p Search string not found: %s (%d)\n"
427  "%p On peer: %u (%p)\n"
428  "%p After: %s\n",
429  peer, peer->search_str, peer->search_str_matched,
430  peer, peer->id, peer,
431  peer,
433  GNUNET_NO));
434  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
435  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
436  }
437 
438  if (NULL != peers[peer_cnt].op_handle)
439  GNUNET_TESTBED_operation_done (peers[peer_cnt].op_handle);
440  }
441 
442  if (NULL != data_file)
443  {
444  GNUNET_DISK_file_close (data_file);
445  data_file = NULL;
446  }
447  for (search_str_cnt = 0;
448  search_str_cnt < num_peers && NULL != search_strings;
449  search_str_cnt++)
450  {
451  GNUNET_free (search_strings[search_str_cnt]);
452  }
454  search_strings = NULL;
455 
456  if (NULL != reg_handle)
457  {
459  reg_handle = NULL;
460  }
461  if (NULL != mc)
462  {
464  mc = NULL;
465  }
466  if (NULL != mc_proc)
467  {
469  mc_proc = NULL;
470  }
471  if (NULL != cfg)
472  {
474  cfg = NULL;
475  }
476 }
477 
478 
484 static void
485 do_abort (void *cls)
486 {
487  unsigned long i = (unsigned long) cls;
488 
490  "Aborting from line %lu...\n", i);
491  abort_task = NULL;
494 }
495 
496 
497 /******************************************************************************/
498 /********************* STATISTICS SERVICE CONNECTIONS ***********************/
499 /******************************************************************************/
500 
511 static void *
512 stats_ca (void *cls,
513  const struct GNUNET_CONFIGURATION_Handle *cfg)
514 {
515  return GNUNET_STATISTICS_create ("<driver>", cfg);
516 }
517 
518 
526 static void
527 stats_da (void *cls, void *op_result)
528 {
529  struct RegexPeer *peer = cls;
530 
531  GNUNET_assert (op_result == peer->stats_handle);
532 
534  peer->stats_handle = NULL;
535 }
536 
537 
548 static int
550  const char *subsystem,
551  const char *name,
552  uint64_t value, int is_persistent)
553 {
554  struct RegexPeer *peer = cls;
555  char output_buffer[512];
556  size_t size;
557 
558  if (NULL == data_file)
559  {
561  "%p -> %s [%s]: %llu\n",
562  peer,
563  subsystem,
564  name,
565  (unsigned long long) value);
566  return GNUNET_OK;
567  }
568  size =
569  GNUNET_snprintf (output_buffer,
570  sizeof(output_buffer),
571  "%p [%s] %llu %s\n",
572  peer,
573  subsystem,
574  (unsigned long long) value,
575  name);
576  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
578  "Unable to write to file!\n");
579 
580  return GNUNET_OK;
581 }
582 
583 
592 static void
593 stats_cb (void *cls,
594  int success)
595 {
596  static unsigned int peer_cnt;
597  struct RegexPeer *peer = cls;
598 
599  if (GNUNET_OK != success)
600  {
602  "Getting statistics for peer %u failed!\n",
603  peer->id);
604  return;
605  }
606 
607  GNUNET_assert (NULL != peer->op_handle);
608 
610  peer->op_handle = NULL;
611 
612  peer_cnt++;
613  peer = &peers[peer_cnt];
614 
615  fprintf (stderr, "s");
616  if (peer_cnt == num_peers)
617  {
619  "\nCollecting stats finished. Shutting down.\n");
621  result = GNUNET_OK;
622  }
623  else
624  {
625  peer->op_handle =
627  peer->peer_handle,
628  "statistics",
630  peer,
631  &stats_ca,
632  &stats_da,
633  peer);
634  }
635 }
636 
637 
647 static void
650  void *ca_result,
651  const char *emsg)
652 {
653  struct RegexPeer *peer = cls;
654 
655  if ((NULL == ca_result) || (NULL != emsg))
656  {
658  "Failed to connect to statistics service on peer %u: %s\n",
659  peer->id, emsg);
660 
661  peer->stats_handle = NULL;
662  return;
663  }
664 
665  peer->stats_handle = ca_result;
666 
667  if (NULL == GNUNET_STATISTICS_get (peer->stats_handle, NULL, NULL,
668  &stats_cb,
669  &stats_iterator, peer))
670  {
672  "Could not get statistics of peer %u!\n", peer->id);
673  }
674 }
675 
676 
683 static void
685 {
686  struct RegexPeer *peer = &peers[0];
687 
688  GNUNET_assert (NULL != peer->peer_handle);
689 
690  peer->op_handle =
692  peer->peer_handle,
693  "statistics",
695  peer,
696  &stats_ca,
697  &stats_da,
698  peer);
699 }
700 
701 
702 /******************************************************************************/
703 /************************ REGEX FIND CONNECTIONS **************************/
704 /******************************************************************************/
705 
706 
712 static void
713 find_string (void *cls);
714 
715 
727 static void
729  const struct GNUNET_PeerIdentity *id,
730  const struct GNUNET_PeerIdentity *get_path,
731  unsigned int get_path_length,
732  const struct GNUNET_PeerIdentity *put_path,
733  unsigned int put_path_length)
734 {
735  struct RegexPeer *peer = cls;
736  char output_buffer[512];
737  size_t size;
738 
739  if (GNUNET_YES == peer->search_str_matched)
740  {
742  "String %s on peer %u already matched!\n",
743  peer->search_str, peer->id);
744  return;
745  }
746 
747  strings_found++;
749 
750  if (NULL != peer->timeout)
751  {
753  peer->timeout = NULL;
754  if (GNUNET_NO == in_shutdown)
756  }
757 
758  if (NULL == id)
759  {
760  // FIXME not possible right now
762  "String matching timed out for string %s on peer %u (%i/%i)\n",
763  peer->search_str, peer->id, strings_found, num_peers);
765  }
766  else
767  {
769 
771  "String %s found on peer %u after %s (%i/%i) (%u||)\n",
772  peer->search_str, peer->id,
775 
777 
778  if (NULL != data_file)
779  {
780  size =
781  GNUNET_snprintf (output_buffer,
782  sizeof(output_buffer),
783  "%p Peer: %u\n"
784  "%p Search string: %s\n"
785  "%p Search duration: %s\n\n",
786  peer, peer->id,
787  peer, peer->search_str,
788  peer,
790  GNUNET_NO));
791 
792  if (size != GNUNET_DISK_file_write (data_file, output_buffer, size))
793  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n");
794  }
795  }
796 
798  peer->op_handle = NULL;
799 
800  if (strings_found == num_peers)
801  {
804  "All strings successfully matched in %s\n",
806 
807  if (NULL != search_timeout_task)
808  {
809  GNUNET_SCHEDULER_cancel (search_timeout_task);
810  search_timeout_task = NULL;
811  }
812 
813  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats.\n");
815  }
816 }
817 
818 
825 static void
827 {
828  unsigned int i;
829 
831  "Finding matches to all strings did not succeed after %s.\n",
832  GNUNET_STRINGS_relative_time_to_string (search_timeout_time,
833  GNUNET_NO));
835  "Found %i of %i strings\n", strings_found, num_peers);
836 
838  "Search timed out after %s."
839  "Collecting stats and shutting down.\n",
840  GNUNET_STRINGS_relative_time_to_string (search_timeout_time,
841  GNUNET_NO));
842 
844  for (i = 0; i < num_peers; i++)
845  {
846  if (NULL != peers[i].op_handle)
847  {
848  GNUNET_TESTBED_operation_done (peers[i].op_handle);
849  peers[i].op_handle = NULL;
850  }
851  }
853 }
854 
855 
862 static void
864 {
865  struct RegexPeer *p = cls;
866 
867  p->timeout = NULL;
869  "Searching for string \"%s\" on peer %d timed out.\n",
870  p->search_str,
871  p->id);
872  if (GNUNET_NO == in_shutdown)
874 }
875 
876 
882 static void
884 {
885  unsigned int search_peer = (unsigned int) (long) cls;
886 
887  if ((search_peer >= num_peers) ||
888  (GNUNET_YES == in_shutdown))
889  return;
890 
892  "Searching for string \"%s\" on peer %d (%u||)\n",
893  peers[search_peer].search_str,
894  search_peer,
896 
897  peers[search_peer].op_handle =
899  peers[search_peer].peer_handle,
900  "dht",
902  &peers[search_peer],
903  &dht_ca,
904  &dht_da,
905  &peers[search_peer]);
906  GNUNET_assert (NULL != peers[search_peer].op_handle);
907  peers[search_peer].timeout
910  &peers[search_peer]);
911 }
912 
913 
921 static void
924  const char *emsg)
925 {
926  struct RegexPeer *peer = (struct RegexPeer *) cls;
927  unsigned long search_peer;
928  unsigned int i;
929 
931  peer->daemon_op = NULL;
932  if (NULL != emsg)
933  {
935  "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg);
936  GNUNET_assert (0);
937  }
938  else
939  {
941  "Daemon %u started successfully\n", peer->id);
942  }
943 
944  /* Find a peer to look for a string matching the regex announced */
946  num_peers);
947  for (i = 0; peers[search_peer].search_str != NULL; i++)
948  {
949  search_peer = (search_peer + 1) % num_peers;
950  if (i > num_peers)
951  GNUNET_assert (0); /* we ran out of peers, must be a bug */
952  }
953  peers[search_peer].search_str = search_strings[peer->id];
954  peers[search_peer].search_str_matched = GNUNET_NO;
957  2),
958  &find_string,
959  (void *) search_peer);
960 }
961 
962 
970 static void
972 {
973  unsigned int i;
974 
975  if (GNUNET_YES == in_shutdown)
976  return;
978  "Starting announce.\n");
979  for (i = 0; i < init_parallel_searches; i++)
980  {
982  " scheduling announce %u\n",
983  i);
985  }
986 }
987 
988 
994 static void
996 {
997  struct RegexPeer *peer;
998 
999  if (GNUNET_YES == in_shutdown)
1000  return;
1001  if (next_search >= num_peers)
1002  {
1003  if (strings_found != num_peers)
1004  {
1005  struct GNUNET_TIME_Relative new_delay;
1006  if (NULL != search_timeout_task)
1007  GNUNET_SCHEDULER_cancel (search_timeout_task);
1009  search_timeout_task = GNUNET_SCHEDULER_add_delayed (new_delay,
1011  NULL);
1012  }
1013  return;
1014  }
1015 
1016  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search);
1017  peer = &peers[next_search];
1018  peer->daemon_op =
1020  peer->peer_handle,
1021  "regexprofiler",
1022  &daemon_started,
1023  peer,
1024  1);
1025  next_search++;
1027 }
1028 
1029 
1040 static void
1042  struct GNUNET_TESTBED_Operation *op,
1043  void *ca_result,
1044  const char *emsg)
1045 {
1046  struct RegexPeer *peer = (struct RegexPeer *) cls;
1047 
1048  if ((NULL != emsg) || (NULL == op) || (NULL == ca_result))
1049  {
1050  GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "DHT connect failed: %s\n", emsg);
1051  GNUNET_assert (0);
1052  }
1053 
1054  GNUNET_assert (NULL != peer->dht_handle);
1055  GNUNET_assert (peer->op_handle == op);
1056  GNUNET_assert (peer->dht_handle == ca_result);
1057 
1058  peer->search_str_matched = GNUNET_NO;
1060  peer->search_str,
1061  &regex_found_handler, peer,
1062  NULL);
1064 }
1065 
1066 
1075 static void *
1076 dht_ca (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
1077 {
1078  struct RegexPeer *peer = cls;
1079 
1080  peer->dht_handle = GNUNET_DHT_connect (cfg, 32);
1081 
1082  return peer->dht_handle;
1083 }
1084 
1085 
1092 static void
1093 dht_da (void *cls, void *op_result)
1094 {
1095  struct RegexPeer *peer = (struct RegexPeer *) cls;
1096 
1097  GNUNET_assert (peer->dht_handle == op_result);
1098 
1099  if (NULL != peer->search_handle)
1100  {
1102  peer->search_handle = NULL;
1103  }
1104 
1105  if (NULL != peer->dht_handle)
1106  {
1108  peer->dht_handle = NULL;
1109  }
1110 }
1111 
1112 
1126 static void
1128  struct GNUNET_TESTBED_RunHandle *h,
1129  unsigned int num_peers_,
1131  unsigned int links_succeeded,
1132  unsigned int links_failed)
1133 {
1134  unsigned int i;
1135 
1136  GNUNET_assert (num_peers_ == num_peers);
1137 
1140  "Testbed started in %s\n",
1142 
1143  if (NULL != abort_task)
1144  {
1145  GNUNET_SCHEDULER_cancel (abort_task);
1146  abort_task = NULL;
1147  }
1148 
1149  for (i = 0; i < num_peers; i++)
1150  {
1151  peers[i].peer_handle = testbed_peers[i];
1152  }
1153  if (GNUNET_NO ==
1154  GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT"))
1155  {
1156  struct GNUNET_TIME_Relative settle_time;
1157 
1158  settle_time =
1160  10 * num_peers);
1162  "Waiting for DHT for %s to settle new connections.\n\n",
1164  GNUNET_NO));
1165  GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL);
1166  }
1167  else
1168  {
1170  }
1171  search_timeout_task =
1172  GNUNET_SCHEDULER_add_delayed (search_timeout_time, &search_timed_out, NULL);
1173 }
1174 
1175 
1182 static void
1184  const struct GNUNET_TESTBED_EventInformation *event)
1185 {
1186  switch (event->type)
1187  {
1189  printf (".");
1190  break;
1191 
1193  printf ("#");
1194  break;
1195 
1196  default:
1197  break;
1198  }
1199  fflush (stdout);
1200 }
1201 
1202 
1203 /******************************************************************************/
1204 /*************************** TESTBED PEER SETUP *****************************/
1205 /******************************************************************************/
1206 
1217 static int
1219  uint64_t data_size,
1220  unsigned int str_max)
1221 {
1222  char *buf; // Keep track of last string to skip blank lines
1223  unsigned int offset;
1224  unsigned int str_cnt;
1225 
1226  buf = data;
1227  offset = 0;
1228  str_cnt = 0;
1229  while ((offset < (data_size - 1)) && (str_cnt < str_max))
1230  {
1231  offset++;
1232  if (((data[offset] == '\n')) &&
1233  (buf != &data[offset]))
1234  {
1235  data[offset] = '\0';
1236  str_cnt++;
1237  buf = &data[offset + 1];
1238  }
1239  else if ((data[offset] == '\n') ||
1240  (data[offset] == '\0'))
1241  buf = &data[offset + 1];
1242  }
1243  return str_cnt;
1244 }
1245 
1246 
1260 static int
1262  char ***strings, unsigned int str_cnt)
1263 {
1264  uint64_t offset;
1265  uint64_t len;
1266  unsigned int i;
1267 
1268  *strings = GNUNET_malloc (sizeof(char *) * str_cnt);
1269  offset = 0;
1270  for (i = 0; i < str_cnt; i++)
1271  {
1272  len = strlen (&data[offset]);
1273  if (offset + len >= data_size)
1274  {
1275  GNUNET_free (*strings);
1276  *strings = NULL;
1277  return GNUNET_SYSERR;
1278  }
1279  if (0 == len) // empty line
1280  {
1281  offset++;
1282  i--;
1283  continue;
1284  }
1285 
1286  GNUNET_asprintf (&(*strings)[i],
1287  "%s%s",
1288  regex_prefix,
1289  &data[offset]);
1290  offset += len + 1;
1291  }
1292  return GNUNET_OK;
1293 }
1294 
1295 
1305 static int
1307  char ***strings,
1308  unsigned int limit)
1309 {
1310  char *data;
1311  uint64_t filesize;
1312  int str_cnt;
1313 
1314  /* Sanity checks */
1315  if (NULL == filename)
1316  {
1317  return GNUNET_SYSERR;
1318  }
1319  if (GNUNET_YES != GNUNET_DISK_file_test (filename))
1320  {
1322  "Could not find search strings file %s\n", filename);
1323  return GNUNET_SYSERR;
1324  }
1325  if (GNUNET_OK !=
1326  GNUNET_DISK_file_size (filename,
1327  &filesize,
1328  GNUNET_YES,
1329  GNUNET_YES))
1330  {
1332  "Search strings file %s cannot be read.\n",
1333  filename);
1334  return GNUNET_SYSERR;
1335  }
1336  if (0 == filesize)
1337  {
1339  "Search strings file %s is empty.\n",
1340  filename);
1341  return GNUNET_SYSERR;
1342  }
1343 
1344  /* Read data into memory */
1345  data = GNUNET_malloc (filesize + 1);
1346  if (filesize != GNUNET_DISK_fn_read (filename,
1347  data,
1348  filesize))
1349  {
1350  GNUNET_free (data);
1352  "Could not read search strings file %s.\n",
1353  filename);
1354  return GNUNET_SYSERR;
1355  }
1356 
1357  /* Process buffer and build array */
1358  str_cnt = count_and_separate_strings (data, filesize, limit);
1359  if (GNUNET_OK != create_string_array (data, filesize, strings, str_cnt))
1360  {
1361  str_cnt = GNUNET_SYSERR;
1362  }
1363  GNUNET_free (data);
1364  return str_cnt;
1365 }
1366 
1367 
1376 static void
1377 run (void *cls,
1378  char *const *args,
1379  const char *cfgfile,
1380  const struct GNUNET_CONFIGURATION_Handle *config)
1381 {
1382  unsigned int nsearchstrs;
1383  unsigned int i;
1384  struct GNUNET_TIME_Relative abort_time;
1385 
1387 
1388  /* Check config */
1389  if (NULL == config)
1390  {
1392  _ ("No configuration file given. Exiting\n"));
1394  return;
1395  }
1396  cfg = GNUNET_CONFIGURATION_dup (config);
1397  if (GNUNET_OK !=
1398  GNUNET_CONFIGURATION_get_value_string (cfg, "REGEXPROFILER",
1399  "REGEX_PREFIX",
1400  &regex_prefix))
1401  {
1403  "regexprofiler",
1404  "regex_prefix");
1406  return;
1407  }
1408  if (GNUNET_OK !=
1409  GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER",
1410  "PARALLEL_SEARCHES",
1412  {
1414  "Configuration option \"PARALLEL_SEARCHES\" missing."
1415  " Using default (%d)\n", 10);
1417  }
1418  if (GNUNET_OK !=
1419  GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER",
1420  "REANNOUNCE_PERIOD_MAX",
1422  {
1424  "reannounce_period_max not given. Using 10 minutes.\n");
1427  }
1428 
1429  /* Check arguments */
1430  if (NULL == policy_dir)
1431  {
1433  _ (
1434  "No policy directory specified on command line. Exiting.\n"));
1435  return;
1436  }
1438  {
1440  _ ("Specified policies directory does not exist. Exiting.\n"));
1442  return;
1443  }
1444  if (0 >= (int) (num_peers = GNUNET_DISK_directory_scan (policy_dir, NULL,
1445  NULL)))
1446  {
1448  _ ("No files found in `%s'\n"),
1449  policy_dir);
1450  return;
1451  }
1452  GNUNET_CONFIGURATION_set_value_string (cfg, "REGEXPROFILER",
1453  "POLICY_DIR", policy_dir);
1455  {
1457  _ ("No search strings file given. Exiting.\n"));
1459  return;
1460  }
1461  nsearchstrs = load_search_strings (strings_file,
1462  &search_strings,
1463  num_peers);
1464  if (num_peers != nsearchstrs)
1465  {
1467  "Error loading search strings.\n");
1469  "File (%s) does not contain enough strings (%u/%u).\n",
1470  strings_file, nsearchstrs, num_peers);
1472  return;
1473  }
1474  if ((0 == num_peers) || (NULL == search_strings))
1475  {
1477  _ ("Error loading search strings. Exiting.\n"));
1479  return;
1480  }
1481  for (i = 0; i < num_peers; i++)
1483  "search string: %s\n",
1484  search_strings[i]);
1485 
1486  /* Check logfile */
1487  if ((NULL != data_filename) &&
1488  (NULL == (data_file =
1495  {
1497  "open",
1498  data_filename);
1499  return;
1500  }
1501 
1502  /* Initialize peers */
1503  peers = GNUNET_malloc (sizeof(struct RegexPeer) * num_peers);
1504  for (i = 0; i < num_peers; i++)
1505  peers[i].id = i;
1506 
1508  "TESTBED", "OVERLAY_RANDOM_LINKS",
1509  num_peers * 20);
1511  "DHT", "FORCE_NSE",
1512  (long long unsigned)
1513  (log (num_peers) / log (2.0)));
1514  event_mask = 0LL;
1515 /* For feedback about the start process activate these and pass master_cb */
1517 // event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
1519 // event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);
1522  cfg,
1523  num_peers,
1524  event_mask,
1526  NULL, /* master_controller_cb cls */
1527  &test_master,
1528  NULL); /* test_master cls */
1529  if (GNUNET_OK !=
1530  GNUNET_CONFIGURATION_get_value_time (cfg, "TESTBED",
1531  "SETUP_TIMEOUT",
1532  &abort_time))
1533  {
1535  "SETUP_TIMEOUT not given. Using 15 minutes.\n");
1536  abort_time =
1538  }
1539  abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES);
1540  abort_task =
1541  GNUNET_SCHEDULER_add_delayed (abort_time,
1542  &do_abort,
1543  (void *) __LINE__);
1545  "setup_timeout: %s\n",
1547 }
1548 
1549 
1557 int
1558 main (int argc, char *const *argv)
1559 {
1560  struct GNUNET_GETOPT_CommandLineOption options[] = {
1562  "output-file",
1563  "FILENAME",
1564  gettext_noop (
1565  "name of the file for writing statistics"),
1566  &data_filename),
1567 
1569  "matching-timeout",
1570  "TIMEOUT",
1571  gettext_noop (
1572  "wait TIMEOUT before ending the experiment"),
1573  &search_timeout_time),
1574 
1576  "policy-dir",
1577  "DIRECTORY",
1578  gettext_noop ("directory with policy files"),
1579  &policy_dir),
1580 
1581 
1583  "strings-file",
1584  "FILENAME",
1585  gettext_noop (
1586  "name of file with input strings"),
1587  &strings_file),
1588 
1590  "hosts-file",
1591  "FILENAME",
1592  gettext_noop (
1593  "name of file with hosts' names"),
1594  &hosts_file),
1595 
1597  };
1598  int ret;
1599 
1600  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
1601  return 2;
1603  ret =
1604  GNUNET_PROGRAM_run (argc, argv,
1605  "gnunet-regex-profiler",
1606  _ ("Profiler for regex"),
1607  options, &run, NULL);
1608  if (GNUNET_OK != ret)
1609  return ret;
1610  if (GNUNET_OK != result)
1611  return 1;
1612  return 0;
1613 }
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_relative_time(char shortName, const char *name, const char *argumentHelp, const char *description, struct GNUNET_TIME_Relative *val)
Allow user to specify a struct GNUNET_TIME_Relative (using human-readable "fancy" time)...
static char * regex_prefix
Prefix used for regex announcing.
void GNUNET_TESTBED_cancel_registration(struct GNUNET_TESTBED_HostRegistrationHandle *handle)
Cancel the pending registration.
struct GNUNET_STATISTICS_Handle * stats_handle
Peers&#39;s statistics handle.
Create file if it doesn&#39;t exist.
unsigned int id
Peer id.
static char * subsystem
Set to subsystem that we&#39;re going to get stats for (or NULL for all).
void GNUNET_TESTBED_controller_disconnect(struct GNUNET_TESTBED_Controller *c)
Stop the given controller (also will terminate all peers and controllers dependent on this controller...
Definition: testbed_api.c:1721
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
struct GNUNET_CONFIGURATION_Handle * cfg
Peer configuration handle.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration, parse options).
Definition: program.c:363
static char * strings_file
File with the strings to look for.
static int in_shutdown
Set when shutting down to avoid making more queries.
Handle to interact with a GNUnet testbed controller.
Definition: testbed_api.h:193
static int count_and_separate_strings(char *data, uint64_t data_size, unsigned int str_max)
Process the text buffer counting the non-empty lines and separating them with NULL characters...
static struct GNUNET_SCHEDULER_Task * register_hosts_task
Host registration task identifier.
static void find_timed_out(void *cls)
Search timed out.
static void stats_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
Function called by testbed once we are connected to stats service.
Destroying peers; we can do this as the controller takes care of stopping a peer if it is running...
static char * data_filename
Filename to log statistics to.
struct GNUNET_DHT_Handle * dht_handle
Peer&#39;s DHT handle.
#define FIND_TIMEOUT
Matching strings against announced regexes.
#define GNUNET_TIME_UNIT_MINUTES
One minute.
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
static struct GNUNET_SCHEDULER_Task * abort_task
Abort task identifier.
int GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1459
void * cls
Closure.
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
uint32_t GNUNET_CRYPTO_random_u32(enum GNUNET_CRYPTO_Quality mode, uint32_t i)
Produce a random value.
struct GNUNET_TESTBED_Operation * op_handle
Testbed operation handle for DHT.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
Peer handles.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1126
static int op_result(struct GNUNET_OP_Handle *h, uint64_t op_id, int64_t result_code, const void *data, uint16_t data_size, void **ctx, uint8_t cancel)
Remove an operation, and call its result callback (unless it was cancelled).
Definition: op.c:245
struct GNUNET_TESTBED_Operation * daemon_op
Daemon start.
void GNUNET_TESTBED_controller_stop(struct GNUNET_TESTBED_ControllerProc *cproc)
Stop the controller process (also will terminate all peers and controllers dependent on this controll...
static void dht_connect_cb(void *cls, struct GNUNET_TESTBED_Operation *op, void *ca_result, const char *emsg)
DHT connect callback.
static void dht_da(void *cls, void *op_result)
Adapter function called to destroy a connection to the DHT service.
static void * dht_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
DHT connect adapter.
Argument to GNUNET_TESTBED_ControllerCallback with details about the event.
struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
void GNUNET_CONFIGURATION_set_value_number(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long number)
Set a configuration value that should be a number.
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_number(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *number)
Get a configuration value that should be a number.
struct GNUNET_SCHEDULER_Task * timeout
Operation timeout.
static void test_master(void *cls, struct GNUNET_TESTBED_RunHandle *h, unsigned int num_peers_, struct GNUNET_TESTBED_Peer **testbed_peers, unsigned int links_succeeded, unsigned int links_failed)
Signature of a main function for a testcase.
struct GNUNET_TESTBED_Operation * op
The testbed operation handle.
enum GNUNET_TESTBED_EventType type
Type of the event.
Definition of a command line option.
static void stats_da(void *cls, void *op_result)
Adapter function called to destroy a connection to statistics service.
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:794
int main(int argc, char *const *argv)
Main function.
static struct GNUNET_CONFIGURATION_Handle * cfg
Handle to global configuration.
static int create_string_array(char *data, uint64_t data_size, char ***strings, unsigned int str_cnt)
Allocate a string array and fill it with the prefixed strings from a pre-processed, NULL-separated memory region.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:531
Handle for the service.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
Handle for controller process.
struct DLLOperation * prev
The prev pointer for DLL.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
Connection to the DHT service.
Definition: dht_api.c:201
Opaque handle to an abstract operation to be executed by the testing framework.
#define _(String)
GNU gettext support macro.
Definition: platform.h:178
static char * hosts_file
File with hostnames where to execute the test.
int GNUNET_snprintf(char *buf, size_t size, const char *format,...) __attribute__((format(printf
Like snprintf, just aborts if the buffer is of insufficient size.
static void do_abort(void *cls)
abort task to run on test timed out
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
static struct GNUNET_TESTBED_ControllerProc * mc_proc
Handle to the master controller process.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_service_connect(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_ServiceConnectCompletionCallback cb, void *cb_cls, GNUNET_TESTBED_ConnectAdapter ca, GNUNET_TESTBED_DisconnectAdapter da, void *cada_cls)
Connect to a service offered by the given peer.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1269
static void * stats_ca(void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg)
Adapter function called to establish a connection to statistics service.
enum State state
current state of profiling
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-qr.c:59
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
static char * value
Value of the record to add/remove.
library to parse regular expressions into dfa
struct GNUNET_TESTBED_Peer * peer_handle
The actual testbed peer handle.
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level &#39;level&#39; that indicates a failure of the command &#39;cmd&#39; with the mess...
static void do_announce(void *cls)
Task to start the daemons on each peer so that the regexes are announced into the DHT...
Truncate file if it exists.
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:666
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1296
static struct GNUNET_TIME_Relative reannounce_period_max
What&#39;s the maximum regex reannounce period.
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:702
void GNUNET_CONFIGURATION_destroy(struct GNUNET_CONFIGURATION_Handle *cfg)
Destroy configuration object.
static struct GNUNET_TESTBED_HostRegistrationHandle * reg_handle
Host registration handle.
static char buf[2048]
static char * filename
static const struct GNUNET_CONFIGURATION_Handle * config
static int result
Global testing status.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Multiply relative time by a given factor.
Definition: time.c:311
static void regex_found_handler(void *cls, const struct GNUNET_PeerIdentity *id, const struct GNUNET_PeerIdentity *get_path, unsigned int get_path_length, const struct GNUNET_PeerIdentity *put_path, unsigned int put_path_length)
Method called when we&#39;ve found a peer that announced a regex that matches our search string...
void GNUNET_CONFIGURATION_set_value_string(struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, const char *value)
Set a configuration value that should be a string.
static void master_controller_cb(void *cls, const struct GNUNET_TESTBED_EventInformation *event)
Function that will be called whenever something in the testbed changes.
static void daemon_started(void *cls, struct GNUNET_TESTBED_Operation *op, const char *emsg)
Callback called when testbed has started the daemon we asked for.
void REGEX_INTERNAL_search_cancel(struct REGEX_INTERNAL_Search *h)
Cancel an ongoing regex search in the DHT and free all resources.
static void find_string(void *cls)
Start searching for the next string in the DHT.
void GNUNET_TESTBED_operation_done(struct GNUNET_TESTBED_Operation *operation)
This function is used to signal that the event information (struct GNUNET_TESTBED_EventInformation) f...
Definition: testbed_api.c:2045
void GNUNET_TESTBED_run(const char *host_filename, const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int num_peers, uint64_t event_mask, GNUNET_TESTBED_ControllerCallback cc, void *cc_cls, GNUNET_TESTBED_TestMaster test_master, void *test_master_cls)
Convenience method for running a testbed with a single call.
const char * search_str
Peer&#39;s search string.
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.
static struct GNUNET_TESTBED_Controller * mc
Handle to the master controller.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:86
static void do_collect_stats(void *cls)
Task to collect all statistics from all peers, will shutdown the profiler, when done.
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
static struct GNUNET_DISK_FileHandle * data_file
File to log statistics to.
struct GNUNET_TESTBED_Peer * peer
The peer associated with this model.
static unsigned int size
Size of the "table".
Definition: peer.c:67
static struct GNUNET_TESTBED_Peer ** testbed_peers
Handles to all of the running peers.
struct REGEX_INTERNAL_Search * REGEX_INTERNAL_search(struct GNUNET_DHT_Handle *dht, const char *string, REGEX_INTERNAL_Found callback, void *callback_cls, struct GNUNET_STATISTICS_Handle *stats)
Search for a peer offering a regex matching certain string in the DHT.
static void announce_next_regex(void *cls)
Start announcing the next regex in the DHT.
struct GNUNET_DHT_Handle * GNUNET_DHT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, unsigned int ht_len)
Initialize the connection with the DHT service.
Definition: dht_api.c:890
static void stats_cb(void *cls, int success)
Stats callback.
void GNUNET_DHT_disconnect(struct GNUNET_DHT_Handle *handle)
Shutdown connection with the DHT service.
Definition: dht_api.c:918
static unsigned int parallel_searches
How many searches are running in parallel.
static char * policy_dir
Folder where policy files are stored.
State
Available states during profiling.
static long long unsigned int init_parallel_searches
How many searches are we going to start in parallel.
static void search_timed_out(void *cls)
Connect by string timeout task.
A connection between two peers was established.
Struct to keep information of searches of services described by a regex using a user-provided string ...
struct GNUNET_CONFIGURATION_Handle * GNUNET_CONFIGURATION_dup(const struct GNUNET_CONFIGURATION_Handle *cfg)
Duplicate an existing configuration object.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:222
static void do_shutdown(void *cls)
Shutdown nicely.
struct GNUNET_TIME_Relative GNUNET_TIME_relative_add(struct GNUNET_TIME_Relative a1, struct GNUNET_TIME_Relative a2)
Add relative times together.
Definition: time.c:411
The identity of the host (wraps the signing key of the peer).
static struct GNUNET_TIME_Relative prof_time
Duration profiling step has taken.
DLL of operations.
static struct RegexPeer * peers
The array of peers; we fill this as the peers are given to us by the testbed.
static char ** search_strings
Search strings (num_peers of them).
static unsigned int next_search
Index of peer to start next announce/search.
struct GNUNET_STATISTICS_GetHandle * GNUNET_STATISTICS_get(struct GNUNET_STATISTICS_Handle *handle, const char *subsystem, const char *name, GNUNET_STATISTICS_Callback cont, GNUNET_STATISTICS_Iterator proc, void *cls)
Get statistic from the peer.
static uint64_t event_mask
Global event mask for all testbed events.
configuration data
Definition: configuration.c:84
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_time(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, struct GNUNET_TIME_Relative *time)
Get a configuration value that should be a relative time.
static struct GNUNET_TIME_Absolute prof_start_time
The starting time of a profiling step.
const char * name
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:263
#define GNUNET_log(kind,...)
Entry in list of pending tasks.
Definition: scheduler.c:134
struct DLLOperation * next
The next pointer for DLL.
Open the file for both reading and writing.
static int stats_iterator(void *cls, const char *subsystem, const char *name, uint64_t value, int is_persistent)
Process statistic values.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
Initial state.
handle for host registration
Time for absolute times used by GNUnet, in microseconds.
static int load_search_strings(const char *filename, char ***strings, unsigned int limit)
Load search strings from given filename.
static unsigned int num_peers
Number of peers to be started by the profiler.
A peer controlled by the testing framework.
static struct GNUNET_TIME_Relative search_timeout_time
Search timeout in seconds.
struct GNUNET_TESTBED_Operation * GNUNET_TESTBED_peer_manage_service(void *op_cls, struct GNUNET_TESTBED_Peer *peer, const char *service_name, GNUNET_TESTBED_OperationCompletionCallback cb, void *cb_cls, unsigned int start)
Start or stop given service at a peer.
struct REGEX_INTERNAL_Search * search_handle
Handle to a running regex search.
uint32_t data
The data value.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1055
Handle used to access files (and pipes).
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:404
struct GNUNET_TIME_Relative GNUNET_TIME_relative_saturating_multiply(struct GNUNET_TIME_Relative rel, unsigned long long factor)
Saturating multiply relative time by a given factor.
Definition: time.c:357
static unsigned int strings_found
Number of strings found in the published regexes.
int search_str_matched
Set to GNUNET_YES if the peer successfully matched the above search string.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
static struct GNUNET_SCHEDULER_Task * search_timeout_task
Search timeout task identifier.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory)...
Definition: disk.c:437
#define GNUNET_malloc(size)
Wrapper around malloc.
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:644
#define GNUNET_free(ptr)
Wrapper around free.
Time for relative time used by GNUnet, in microseconds.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
#define gettext_noop(String)
Definition: gettext.h:69
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:972