GNUnet  0.11.x
gnunet-regex-simulation-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011, 2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
21 
31 #include "platform.h"
32 #include "gnunet_util_lib.h"
33 #include "regex_internal_lib.h"
34 #include "gnunet_mysql_lib.h"
35 #include "gnunet_my_lib.h"
36 #include <mysql/mysql.h>
37 
41 #define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` " \
42  "(`key`, `label`, `to_key`, `accepting`) " \
43  "VALUES (?, ?, ?, ?);"
44 
48 #define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` " \
49  "WHERE `key` = ? AND `label` = ?;"
50 
56 {
60  unsigned int total;
61 
65  unsigned int modnum;
66 
70  unsigned int dotnum;
71 
75  unsigned int completed;
76 
80  int print;
81 
86 };
87 
88 
92 static struct ProgressMeter *meter;
93 
98 
102 static int result;
103 
108 
113 
118 
122 static char *table_name;
123 
127 static char *policy_dir;
128 
132 static unsigned int num_policy_files;
133 
137 static unsigned int num_policies;
138 
142 static unsigned int max_path_compression;
143 
147 static unsigned long long num_merged_transitions;
148 
152 static unsigned long long num_merged_states;
153 
157 static char *regex_prefix;
158 
159 
170 static struct ProgressMeter *
171 create_meter (unsigned int total, char *start_string, int print)
172 {
173  struct ProgressMeter *ret;
174 
175  ret = GNUNET_new (struct ProgressMeter);
176  ret->print = print;
177  ret->total = total;
178  ret->modnum = total / 4;
179  if (ret->modnum == 0) /* Divide by zero check */
180  ret->modnum = 1;
181  ret->dotnum = (total / 50) + 1;
182  if (start_string != NULL)
183  ret->startup_string = GNUNET_strdup (start_string);
184  else
185  ret->startup_string = GNUNET_strdup ("");
186 
187  return ret;
188 }
189 
190 
199 static int
201 {
202  if (meter->print == GNUNET_YES)
203  {
204  if (meter->completed % meter->modnum == 0)
205  {
206  if (meter->completed == 0)
207  {
208  fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
209  }
210  else
211  fprintf (stdout, "%d%%",
212  (int) (((float) meter->completed / meter->total) * 100));
213  }
214  else if (meter->completed % meter->dotnum == 0)
215  fprintf (stdout, "%s", ".");
216 
217  if (meter->completed + 1 == meter->total)
218  fprintf (stdout, "%d%%]\n", 100);
219  fflush (stdout);
220  }
221  meter->completed++;
222 
223  if (meter->completed == meter->total)
224  return GNUNET_YES;
225  if (meter->completed > meter->total)
226  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n");
227  return GNUNET_NO;
228 }
229 
230 
239 static int
240 reset_meter (struct ProgressMeter *meter)
241 {
242  if (meter == NULL)
243  return GNUNET_SYSERR;
244 
245  meter->completed = 0;
246  return GNUNET_YES;
247 }
248 
249 
255 static void
256 free_meter (struct ProgressMeter *meter)
257 {
259  GNUNET_free (meter);
260 }
261 
262 
268 static void
269 do_shutdown (void *cls)
270 {
271  if (NULL != mysql_ctx)
272  {
273  GNUNET_MYSQL_context_destroy (mysql_ctx);
274  mysql_ctx = NULL;
275  }
276  if (NULL != meter)
277  {
278  free_meter (meter);
279  meter = NULL;
280  }
281 }
282 
283 
295 static void
296 do_abort (void *cls)
297 {
298  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n");
299  if (NULL != scan_task)
300  {
301  GNUNET_SCHEDULER_cancel (scan_task);
302  scan_task = NULL;
303  }
306 }
307 
308 
319 static void
320 regex_iterator (void *cls,
321  const struct GNUNET_HashCode *key,
322  const char *proof,
323  int accepting,
324  unsigned int num_edges,
325  const struct REGEX_BLOCK_Edge *edges)
326 {
327  unsigned int i;
328  int result;
329 
330  uint32_t iaccepting = (uint32_t) accepting;
331  uint64_t total;
332 
333  GNUNET_assert (NULL != mysql_ctx);
334 
335  for (i = 0; i < num_edges; i++)
336  {
337  struct GNUNET_MY_QueryParam params_select[] = {
339  GNUNET_MY_query_param_string (edges[i].label),
341  };
342 
343  struct GNUNET_MY_ResultSpec results_select[] = {
346  };
347 
348  result =
349  GNUNET_MY_exec_prepared (mysql_ctx,
350  select_stmt_handle,
351  params_select);
352 
353  if (GNUNET_SYSERR == result)
354  {
356  "Error executing prepared mysql select statement\n");
358  return;
359  }
360 
361  result =
362  GNUNET_MY_extract_result (select_stmt_handle,
363  results_select);
364 
365  if (GNUNET_SYSERR == result)
366  {
368  "Error extracting result mysql select statement\n");
370  return;
371  }
372 
373  if ((-1 != total) && (total > 0) )
374  {
375  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n",
376  (unsigned long long) total,
377  GNUNET_h2s (key), edges[i].label);
378  }
379 
380  struct GNUNET_MY_QueryParam params_stmt[] = {
382  GNUNET_MY_query_param_string (edges[i].label),
383  GNUNET_MY_query_param_auto_from_type (&edges[i].destination),
384  GNUNET_MY_query_param_uint32 (&iaccepting),
386  };
387 
388  result =
389  GNUNET_MY_exec_prepared (mysql_ctx,
390  stmt_handle,
391  params_stmt);
392 
393  if (0 == result)
394  {
395  char *key_str = GNUNET_strdup (GNUNET_h2s (key));
396  char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination));
397 
398  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n",
399  key_str,
400  edges[i].label,
401  to_key_str,
402  accepting);
403 
404  GNUNET_free (key_str);
405  GNUNET_free (to_key_str);
407  }
408  else if (-1 != total)
409  {
411  }
412 
413  if ((GNUNET_SYSERR == result) || ((1 != result) && (0 != result) ))
414  {
416  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
417  result);
419  }
420  }
421 
422  if (0 == num_edges)
423  {
424  struct GNUNET_MY_QueryParam params_stmt[] = {
428  GNUNET_MY_query_param_uint32 (&iaccepting),
430  };
431 
432  result =
433  GNUNET_MY_exec_prepared (mysql_ctx,
434  stmt_handle,
435  params_stmt);
436 
437  if ((1 != result) && (0 != result) )
438  {
440  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
441  result);
443  }
444  }
445 }
446 
447 
455 static int
456 announce_regex (const char *regex)
457 {
458  struct REGEX_INTERNAL_Automaton *dfa;
459 
460  dfa =
462  strlen (regex),
464 
465  if (NULL == dfa)
466  {
468  "Failed to create DFA for regex %s\n",
469  regex);
471  return GNUNET_SYSERR;
472  }
474  &regex_iterator, NULL);
476 
477  return GNUNET_OK;
478 }
479 
480 
489 static int
490 policy_filename_cb (void *cls, const char *filename)
491 {
492  char *regex;
493  char *data;
494  char *buf;
495  uint64_t filesize;
496  unsigned int offset;
497 
498  GNUNET_assert (NULL != filename);
499 
501  "Announcing regexes from file %s\n",
502  filename);
503 
504  if (GNUNET_YES != GNUNET_DISK_file_test (filename))
505  {
507  "Could not find policy file %s\n",
508  filename);
509  return GNUNET_OK;
510  }
511  if (GNUNET_OK !=
512  GNUNET_DISK_file_size (filename, &filesize,
514  filesize = 0;
515  if (0 == filesize)
516  {
517  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n",
518  filename);
519  return GNUNET_OK;
520  }
521  data = GNUNET_malloc (filesize);
522  if (filesize != GNUNET_DISK_fn_read (filename, data, filesize))
523  {
524  GNUNET_free (data);
526  "Could not read policy file %s.\n",
527  filename);
528  return GNUNET_OK;
529  }
530 
531  update_meter (meter);
532 
533  buf = data;
534  offset = 0;
535  regex = NULL;
536  while (offset < (filesize - 1))
537  {
538  offset++;
539  if (((data[offset] == '\n')) && (buf != &data[offset]))
540  {
541  data[offset] = '|';
542  num_policies++;
543  buf = &data[offset + 1];
544  }
545  else if ((data[offset] == '\n') || (data[offset] == '\0'))
546  buf = &data[offset + 1];
547  }
548  data[offset] = '\0';
549  GNUNET_asprintf (&regex, "%s(%s)", regex_prefix, data);
550  GNUNET_assert (NULL != regex);
552  "Announcing regex: %s\n", regex);
553 
554  if (GNUNET_OK != announce_regex (regex))
555  {
557  "Could not announce regex %s\n",
558  regex);
559  }
560  GNUNET_free (regex);
561  GNUNET_free (data);
562  return GNUNET_OK;
563 }
564 
565 
571 static void
572 do_directory_scan (void *cls)
573 {
574  struct GNUNET_TIME_Absolute start_time;
575  struct GNUNET_TIME_Relative duration;
576  char *stmt;
577 
578  /* Create an MySQL prepared statement for the inserts */
579  scan_task = NULL;
581  stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
582  GNUNET_free (stmt);
583 
585  select_stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
586  GNUNET_free (stmt);
587 
588  GNUNET_assert (NULL != stmt_handle);
589 
591  "Announcing policy files\n",
592  GNUNET_YES);
593  start_time = GNUNET_TIME_absolute_get ();
596  stmt_handle);
597  duration = GNUNET_TIME_absolute_get_duration (start_time);
598  reset_meter (meter);
599  free_meter (meter);
600  meter = NULL;
601 
602  printf ("Announced %u files containing %u policies in %s\n"
603  "Duplicate transitions: %llu\nMerged states: %llu\n",
605  num_policies,
609  result = GNUNET_OK;
611 }
612 
613 
622 static void
623 run (void *cls,
624  char *const *args,
625  const char *cfgfile,
626  const struct GNUNET_CONFIGURATION_Handle *config)
627 {
628  if (NULL == args[0])
629  {
630  fprintf (stderr,
631  _ ("No policy directory specified on command line. Exiting.\n"));
633  return;
634  }
635  if (GNUNET_YES !=
637  {
638  fprintf (stderr,
639  _ ("Specified policies directory does not exist. Exiting.\n"));
641  return;
642  }
643  policy_dir = args[0];
644 
646  NULL, NULL);
647  meter = NULL;
648 
649  if (NULL == table_name)
650  {
652  "No table name specified, using default \"NFA\".\n");
653  table_name = "NFA";
654  }
655 
656  mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql");
657  if (NULL == mysql_ctx)
658  {
660  "Failed to create mysql context\n");
662  return;
663  }
664 
665  if (GNUNET_OK !=
667  "regex-mysql",
668  "REGEX_PREFIX",
669  &regex_prefix))
670  {
672  "regex-mysql",
673  "REGEX_PREFIX");
675  return;
676  }
677 
678  result = GNUNET_OK;
680  NULL);
681  scan_task = GNUNET_SCHEDULER_add_now (&do_directory_scan, NULL);
682 }
683 
684 
692 int
693 main (int argc, char *const *argv)
694 {
695  struct GNUNET_GETOPT_CommandLineOption options[] = {
697  "table",
698  "TABLENAME",
699  gettext_noop (
700  "name of the table to write DFAs"),
701  &table_name),
702 
704  "max-path-compression",
705  "MAX_PATH_COMPRESSION",
706  gettext_noop ("maximum path compression length"),
708 
710  };
711  int ret;
712 
713  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
714  return 2;
715 
717  ret =
718  GNUNET_PROGRAM_run (argc, argv,
719  "gnunet-regex-simulationprofiler [OPTIONS] policy-dir",
720  _ ("Profiler for regex library"), options, &run, NULL);
721  if (GNUNET_OK != ret)
722  return ret;
723  if (GNUNET_OK != result)
724  return 1;
725  return 0;
726 }
int GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory)...
Definition: disk.c:544
static int announce_regex(const char *regex)
Announce a regex by creating the DFA and iterating over each state, inserting each state into a MySQL...
#define GNUNET_MY_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
static char * table_name
MySQL table name.
struct GNUNET_MYSQL_Context * GNUNET_MYSQL_context_create(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section)
Create a mysql context.
Definition: mysql.c:320
#define GNUNET_MY_result_spec_end
End of result speceter specification.
Simple struct to keep track of progress, and print a nice little percentage meter for long running ta...
struct GNUNET_MY_ResultSpec GNUNET_MY_result_spec_uint64(uint64_t *u64)
uint64_t expected.
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.FG.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received, or when GNUNET_SCHEDULER_shutdown() is being invoked.
Definition: scheduler.c:1300
int GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1438
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static void do_directory_scan(void *cls)
Iterate over files contained in policy_dir.
#define INSERT_EDGE_STMT
MySQL statement to insert an edge.
static unsigned int max_path_compression
Maximal path compression length.
static unsigned long long num_merged_transitions
Number of merged transitions.
int GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:503
Information we pass to GNUNET_MY_exec_prepared() to initialize the arguments of the prepared statemen...
Definition: gnunet_my_lib.h:83
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
#define GNUNET_NO
Definition: gnunet_common.h:78
static unsigned long long num_merged_states
Number of merged states from different policies.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
static int reset_meter(struct ProgressMeter *meter)
Reset progress meter.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
Definition of a command line option.
Handle for a prepared statement.
Definition: mysql.c:112
Automaton representation.
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:912
static unsigned int num_policy_files
Number of policy files.
int GNUNET_MY_exec_prepared(struct GNUNET_MYSQL_Context *mc, struct GNUNET_MYSQL_StatementHandle *sh, struct GNUNET_MY_QueryParam *params)
Run a prepared SELECT statement.
Definition: my.c:42
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:526
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
unsigned int modnum
Intervall for printing percentage.
static char * policy_dir
Policy dir containing files that contain policies.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
Edge representation.
static struct GNUNET_MYSQL_Context * mysql_ctx
MySQL context.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_string(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a string.
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
static void regex_iterator(void *cls, const struct GNUNET_HashCode *key, const char *proof, int accepting, unsigned int num_edges, const struct REGEX_BLOCK_Edge *edges)
Iterator over all states that inserts each state into the MySQL db.
int GNUNET_asprintf(char **buf, const char *format,...)
Like asprintf, just portable.
Information we pass to GNUNET_MY_extract_result() to initialize the arguments of the prepared stateme...
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
library to parse regular expressions into dfa
static uint64_t proof
Definition: gnunet-scrypt.c:41
int main(int argc, char *const *argv)
Main function.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1280
void GNUNET_MYSQL_context_destroy(struct GNUNET_MYSQL_Context *mc)
Destroy a mysql context.
Definition: mysql.c:368
static struct ProgressMeter * meter
Handle for the progress meter.
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:687
unsigned int completed
Completed number.
static char buf[2048]
static char * filename
static const struct GNUNET_CONFIGURATION_Handle * config
static struct GNUNET_SCHEDULER_Task * scan_task
Scan task identifier;.
A 512-bit hashcode.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
int GNUNET_MY_extract_result(struct GNUNET_MYSQL_StatementHandle *sh, struct GNUNET_MY_ResultSpec *specs)
Extract results from a query result according to the given specification.
Definition: my.c:136
int GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
struct GNUNET_HashCode key
The key used in the DHT.
#define GNUNET_MY_query_param_end
End of query parameter specification.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int num_policies
Number of policies.
#define SELECT_KEY_STMT
MySQL statement to select a key count.
static void free_meter(struct ProgressMeter *meter)
Release resources for meter.
Mysql context.
Definition: mysql.c:75
struct GNUNET_MYSQL_StatementHandle * GNUNET_MYSQL_statement_prepare(struct GNUNET_MYSQL_Context *mc, const char *query)
Prepare a statement.
Definition: mysql.c:393
void REGEX_INTERNAL_automaton_destroy(struct REGEX_INTERNAL_Automaton *a)
Free the memory allocated by constructing the REGEX_INTERNAL_Automaton data structure.
void REGEX_INTERNAL_iterate_all_edges(struct REGEX_INTERNAL_Automaton *a, REGEX_INTERNAL_KeyIterator iterator, void *iterator_cls)
Iterate over all edges starting from start state of automaton &#39;a&#39;.
int print
Should the meter be printed?
int GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:257
static int update_meter(struct ProgressMeter *meter)
Update progress meter (increment by one).
static void do_shutdown(void *cls)
Shutdown task.
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_uint32(const uint32_t *x)
Generate query parameter for an uint32_t in host byte order.
static int policy_filename_cb(void *cls, const char *filename)
Function called with a filename.
configuration data
Definition: configuration.c:85
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:375
struct REGEX_INTERNAL_Automaton * REGEX_INTERNAL_construct_dfa(const char *regex, const size_t len, unsigned int max_path_len)
Construct DFA for the given &#39;regex&#39; of length &#39;len&#39;.
#define GNUNET_log(kind,...)
Entry in list of pending tasks.
Definition: scheduler.c:134
unsigned int dotnum
Number of dots to print.
static char * regex_prefix
Prefix to add before every regex we&#39;re announcing.
static int result
Global testing status.
static struct GNUNET_MYSQL_StatementHandle * stmt_handle
MySQL prepared statement handle.
unsigned int total
Total number of elements.
int GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration, parse options).
Definition: program.c:367
Time for absolute times used by GNUnet, in microseconds.
#define GNUNET_YES
Definition: gnunet_common.h:77
static void do_abort(void *cls)
Abort task to run on test timed out.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_uint(char shortName, const char *name, const char *argumentHelp, const char *description, unsigned int *val)
Allow user to specify an unsigned int.
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_string(const char *ptr)
Generate query parameter for a string.
char * startup_string
String to print on startup.
uint32_t data
The data value.
static struct ProgressMeter * create_meter(unsigned int total, char *start_string, int print)
Create a meter to keep track of the progress of some task.
static struct GNUNET_MYSQL_StatementHandle * select_stmt_handle
MySQL prepared statement handle for key select.
#define GNUNET_malloc(size)
Wrapper around malloc.
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:794
#define GNUNET_free(ptr)
Wrapper around free.
Time for relative time used by GNUnet, in microseconds.
#define gettext_noop(String)
Definition: gettext.h:69
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:966