GNUnet  0.19.2
gnunet-regex-simulation-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011, 2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
21 
31 #include "platform.h"
32 #include "gnunet_util_lib.h"
33 #include "regex_internal_lib.h"
34 #include "gnunet_mysql_lib.h"
35 #include "gnunet_mysql_compat.h"
36 #include "gnunet_my_lib.h"
37 #include <mysql/mysql.h>
38 
42 #define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` " \
43  "(`key`, `label`, `to_key`, `accepting`) " \
44  "VALUES (?, ?, ?, ?);"
45 
49 #define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` " \
50  "WHERE `key` = ? AND `label` = ?;"
51 
57 {
61  unsigned int total;
62 
66  unsigned int modnum;
67 
71  unsigned int dotnum;
72 
76  unsigned int completed;
77 
81  int print;
82 
87 };
88 
89 
93 static struct ProgressMeter *meter;
94 
99 
103 static int result;
104 
109 
114 
119 
123 static char *table_name;
124 
128 static char *policy_dir;
129 
133 static unsigned int num_policy_files;
134 
138 static unsigned int num_policies;
139 
143 static unsigned int max_path_compression;
144 
148 static unsigned long long num_merged_transitions;
149 
153 static unsigned long long num_merged_states;
154 
158 static char *regex_prefix;
159 
160 
171 static struct ProgressMeter *
172 create_meter (unsigned int total, char *start_string, int print)
173 {
174  struct ProgressMeter *ret;
175 
176  ret = GNUNET_new (struct ProgressMeter);
177  ret->print = print;
178  ret->total = total;
179  ret->modnum = total / 4;
180  if (ret->modnum == 0) /* Divide by zero check */
181  ret->modnum = 1;
182  ret->dotnum = (total / 50) + 1;
183  if (start_string != NULL)
184  ret->startup_string = GNUNET_strdup (start_string);
185  else
186  ret->startup_string = GNUNET_strdup ("");
187 
188  return ret;
189 }
190 
191 
200 static int
202 {
203  if (meter->print == GNUNET_YES)
204  {
205  if (meter->completed % meter->modnum == 0)
206  {
207  if (meter->completed == 0)
208  {
209  fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
210  }
211  else
212  fprintf (stdout, "%d%%",
213  (int) (((float) meter->completed / meter->total) * 100));
214  }
215  else if (meter->completed % meter->dotnum == 0)
216  fprintf (stdout, "%s", ".");
217 
218  if (meter->completed + 1 == meter->total)
219  fprintf (stdout, "%d%%]\n", 100);
220  fflush (stdout);
221  }
222  meter->completed++;
223 
224  if (meter->completed == meter->total)
225  return GNUNET_YES;
226  if (meter->completed > meter->total)
227  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n");
228  return GNUNET_NO;
229 }
230 
231 
240 static int
242 {
243  if (meter == NULL)
244  return GNUNET_SYSERR;
245 
246  meter->completed = 0;
247  return GNUNET_YES;
248 }
249 
250 
256 static void
258 {
260  GNUNET_free (meter);
261 }
262 
263 
269 static void
270 do_shutdown (void *cls)
271 {
272  if (NULL != mysql_ctx)
273  {
275  mysql_ctx = NULL;
276  }
277  if (NULL != meter)
278  {
279  free_meter (meter);
280  meter = NULL;
281  }
282 }
283 
284 
296 static void
297 do_abort (void *cls)
298 {
299  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n");
300  if (NULL != scan_task)
301  {
303  scan_task = NULL;
304  }
307 }
308 
309 
320 static void
321 regex_iterator (void *cls,
322  const struct GNUNET_HashCode *key,
323  const char *proof,
324  int accepting,
325  unsigned int num_edges,
326  const struct REGEX_BLOCK_Edge *edges)
327 {
328  unsigned int i;
329  int result;
330 
331  uint32_t iaccepting = (uint32_t) accepting;
332  uint64_t total;
333 
334  GNUNET_assert (NULL != mysql_ctx);
335 
336  for (i = 0; i < num_edges; i++)
337  {
338  struct GNUNET_MY_QueryParam params_select[] = {
340  GNUNET_MY_query_param_string (edges[i].label),
342  };
343 
344  struct GNUNET_MY_ResultSpec results_select[] = {
347  };
348 
349  result =
352  params_select);
353 
354  if (GNUNET_SYSERR == result)
355  {
357  "Error executing prepared mysql select statement\n");
359  return;
360  }
361 
362  result =
364  results_select);
365 
366  if (GNUNET_SYSERR == result)
367  {
369  "Error extracting result mysql select statement\n");
371  return;
372  }
373 
374  if ((-1 != total) && (total > 0) )
375  {
376  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n",
377  (unsigned long long) total,
378  GNUNET_h2s (key), edges[i].label);
379  }
380 
381  struct GNUNET_MY_QueryParam params_stmt[] = {
383  GNUNET_MY_query_param_string (edges[i].label),
384  GNUNET_MY_query_param_auto_from_type (&edges[i].destination),
385  GNUNET_MY_query_param_uint32 (&iaccepting),
387  };
388 
389  result =
391  stmt_handle,
392  params_stmt);
393 
394  if (0 == result)
395  {
396  char *key_str = GNUNET_strdup (GNUNET_h2s (key));
397  char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination));
398 
399  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n",
400  key_str,
401  edges[i].label,
402  to_key_str,
403  accepting);
404 
405  GNUNET_free (key_str);
406  GNUNET_free (to_key_str);
408  }
409  else if (-1 != total)
410  {
412  }
413 
414  if ((GNUNET_SYSERR == result) || ((1 != result) && (0 != result) ))
415  {
417  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
418  result);
420  }
421  }
422 
423  if (0 == num_edges)
424  {
425  struct GNUNET_MY_QueryParam params_stmt[] = {
429  GNUNET_MY_query_param_uint32 (&iaccepting),
431  };
432 
433  result =
435  stmt_handle,
436  params_stmt);
437 
438  if ((1 != result) && (0 != result) )
439  {
441  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
442  result);
444  }
445  }
446 }
447 
448 
456 static int
457 announce_regex (const char *regex)
458 {
459  struct REGEX_INTERNAL_Automaton *dfa;
460 
461  dfa =
463  strlen (regex),
465 
466  if (NULL == dfa)
467  {
469  "Failed to create DFA for regex %s\n",
470  regex);
472  return GNUNET_SYSERR;
473  }
475  &regex_iterator, NULL);
477 
478  return GNUNET_OK;
479 }
480 
481 
490 static int
491 policy_filename_cb (void *cls, const char *filename)
492 {
493  char *regex;
494  char *data;
495  char *buf;
496  uint64_t filesize;
497  unsigned int offset;
498 
499  GNUNET_assert (NULL != filename);
500 
502  "Announcing regexes from file %s\n",
503  filename);
504 
506  {
508  "Could not find policy file %s\n",
509  filename);
510  return GNUNET_OK;
511  }
512  if (GNUNET_OK !=
513  GNUNET_DISK_file_size (filename, &filesize,
515  filesize = 0;
516  if (0 == filesize)
517  {
518  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n",
519  filename);
520  return GNUNET_OK;
521  }
522  data = GNUNET_malloc (filesize);
523  if (filesize != GNUNET_DISK_fn_read (filename, data, filesize))
524  {
525  GNUNET_free (data);
527  "Could not read policy file %s.\n",
528  filename);
529  return GNUNET_OK;
530  }
531 
533 
534  buf = data;
535  offset = 0;
536  regex = NULL;
537  while (offset < (filesize - 1))
538  {
539  offset++;
540  if (((data[offset] == '\n')) && (buf != &data[offset]))
541  {
542  data[offset] = '|';
543  num_policies++;
544  buf = &data[offset + 1];
545  }
546  else if ((data[offset] == '\n') || (data[offset] == '\0'))
547  buf = &data[offset + 1];
548  }
549  data[offset] = '\0';
550  GNUNET_asprintf (&regex, "%s(%s)", regex_prefix, data);
551  GNUNET_assert (NULL != regex);
553  "Announcing regex: %s\n", regex);
554 
555  if (GNUNET_OK != announce_regex (regex))
556  {
558  "Could not announce regex %s\n",
559  regex);
560  }
561  GNUNET_free (regex);
562  GNUNET_free (data);
563  return GNUNET_OK;
564 }
565 
566 
572 static void
573 do_directory_scan (void *cls)
574 {
577  char *stmt;
578 
579  /* Create an MySQL prepared statement for the inserts */
580  scan_task = NULL;
583  GNUNET_free (stmt);
584 
587  GNUNET_free (stmt);
588 
589  GNUNET_assert (NULL != stmt_handle);
590 
592  "Announcing policy files\n",
593  GNUNET_YES);
597  stmt_handle);
599  reset_meter (meter);
600  free_meter (meter);
601  meter = NULL;
602 
603  printf ("Announced %u files containing %u policies in %s\n"
604  "Duplicate transitions: %llu\nMerged states: %llu\n",
606  num_policies,
610  result = GNUNET_OK;
612 }
613 
614 
623 static void
624 run (void *cls,
625  char *const *args,
626  const char *cfgfile,
627  const struct GNUNET_CONFIGURATION_Handle *config)
628 {
629  if (NULL == args[0])
630  {
631  fprintf (stderr,
632  _ ("No policy directory specified on command line. Exiting.\n"));
634  return;
635  }
636  if (GNUNET_YES !=
638  {
639  fprintf (stderr,
640  _ ("Specified policies directory does not exist. Exiting.\n"));
642  return;
643  }
644  policy_dir = args[0];
645 
647  NULL, NULL);
648  meter = NULL;
649 
650  if (NULL == table_name)
651  {
653  "No table name specified, using default \"NFA\".\n");
654  table_name = "NFA";
655  }
656 
657  mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql");
658  if (NULL == mysql_ctx)
659  {
661  "Failed to create mysql context\n");
663  return;
664  }
665 
666  if (GNUNET_OK !=
668  "regex-mysql",
669  "REGEX_PREFIX",
670  &regex_prefix))
671  {
673  "regex-mysql",
674  "REGEX_PREFIX");
676  return;
677  }
678 
679  result = GNUNET_OK;
681  NULL);
683 }
684 
685 
693 int
694 main (int argc, char *const *argv)
695 {
698  "table",
699  "TABLENAME",
700  gettext_noop (
701  "name of the table to write DFAs"),
702  &table_name),
703 
705  "max-path-compression",
706  "MAX_PATH_COMPRESSION",
707  gettext_noop ("maximum path compression length"),
709 
711  };
712  int ret;
713 
714  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
715  return 2;
716 
718  ret =
719  GNUNET_PROGRAM_run (argc, argv,
720  "gnunet-regex-simulationprofiler [OPTIONS] policy-dir",
721  _ ("Profiler for regex library"), options, &run, NULL);
722  if (GNUNET_OK != ret)
723  return ret;
724  if (GNUNET_OK != result)
725  return 1;
726  return 0;
727 }
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
#define gettext_noop(String)
Definition: gettext.h:70
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static struct GNUNET_TIME_Absolute start_time
Start time of the current round; used to determine how long one iteration takes (which influences how...
struct GNUNET_HashCode key
The key used in the DHT.
static char * filename
uint32_t data
The data value.
static void regex_iterator(void *cls, const struct GNUNET_HashCode *key, const char *proof, int accepting, unsigned int num_edges, const struct REGEX_BLOCK_Edge *edges)
Iterator over all states that inserts each state into the MySQL db.
static void do_directory_scan(void *cls)
Iterate over files contained in policy_dir.
static struct ProgressMeter * meter
Handle for the progress meter.
static void free_meter(struct ProgressMeter *meter)
Release resources for meter.
static unsigned int max_path_compression
Maximal path compression length.
static char * policy_dir
Policy dir containing files that contain policies.
#define SELECT_KEY_STMT
MySQL statement to select a key count.
#define INSERT_EDGE_STMT
MySQL statement to insert an edge.
static void do_abort(void *cls)
Abort task to run on test timed out.
static unsigned int num_policies
Number of policies.
static unsigned long long num_merged_states
Number of merged states from different policies.
static struct GNUNET_MYSQL_Context * mysql_ctx
MySQL context.
static void do_shutdown(void *cls)
Shutdown task.
static struct GNUNET_SCHEDULER_Task * scan_task
Scan task identifier;.
static unsigned int num_policy_files
Number of policy files.
static int policy_filename_cb(void *cls, const char *filename)
Function called with a filename.
static int result
Global testing status.
static char * table_name
MySQL table name.
static int update_meter(struct ProgressMeter *meter)
Update progress meter (increment by one).
static int announce_regex(const char *regex)
Announce a regex by creating the DFA and iterating over each state, inserting each state into a MySQL...
static struct GNUNET_MYSQL_StatementHandle * stmt_handle
MySQL prepared statement handle.
static int reset_meter(struct ProgressMeter *meter)
Reset progress meter.
int main(int argc, char *const *argv)
Main function.
static struct GNUNET_MYSQL_StatementHandle * select_stmt_handle
MySQL prepared statement handle for key select.
static struct ProgressMeter * create_meter(unsigned int total, char *start_string, int print)
Create a meter to keep track of the progress of some task.
static char * regex_prefix
Prefix to add before every regex we're announcing.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
static unsigned long long num_merged_transitions
Number of merged transitions.
static struct GNUNET_TIME_Relative duration
How long do we run the test?
static uint64_t proof
Definition: gnunet-scrypt.c:49
static char buf[2048]
static const struct GNUNET_CONFIGURATION_Handle * config
Helper library to access a MySQL database.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition: disk.c:481
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:221
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:403
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:663
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:813
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_uint(char shortName, const char *name, const char *argumentHelp, const char *description, unsigned int *val)
Allow user to specify an unsigned int.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_string(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a string.
#define GNUNET_log(kind,...)
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
#define GNUNET_MY_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
struct GNUNET_MYSQL_StatementHandle * GNUNET_MYSQL_statement_prepare(struct GNUNET_MYSQL_Context *mc, const char *query)
Prepare a statement.
Definition: mysql.c:394
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_string(const char *ptr)
Generate query parameter for a string.
struct GNUNET_MYSQL_Context * GNUNET_MYSQL_context_create(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section)
Create a mysql context.
Definition: mysql.c:321
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.FG.
void GNUNET_MYSQL_context_destroy(struct GNUNET_MYSQL_Context *mc)
Destroy a mysql context.
Definition: mysql.c:369
int GNUNET_MY_exec_prepared(struct GNUNET_MYSQL_Context *mc, struct GNUNET_MYSQL_StatementHandle *sh, struct GNUNET_MY_QueryParam *params)
Run a prepared SELECT statement.
Definition: my.c:43
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_uint32(const uint32_t *x)
Generate query parameter for an uint32_t in host byte order.
#define GNUNET_MY_result_spec_end
End of result speceter specification.
struct GNUNET_MY_ResultSpec GNUNET_MY_result_spec_uint64(uint64_t *u64)
uint64_t expected.
int GNUNET_MY_extract_result(struct GNUNET_MYSQL_StatementHandle *sh, struct GNUNET_MY_ResultSpec *specs)
Extract results from a query result according to the given specification.
Definition: my.c:137
#define GNUNET_MY_query_param_end
End of query parameter specification.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:400
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:534
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1268
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received,...
Definition: scheduler.c:1303
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:944
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1222
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:436
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:111
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:569
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
void REGEX_INTERNAL_iterate_all_edges(struct REGEX_INTERNAL_Automaton *a, REGEX_INTERNAL_KeyIterator iterator, void *iterator_cls)
Iterate over all edges starting from start state of automaton 'a'.
struct REGEX_INTERNAL_Automaton * REGEX_INTERNAL_construct_dfa(const char *regex, const size_t len, unsigned int max_path_len)
Construct DFA for the given 'regex' of length 'len'.
void REGEX_INTERNAL_automaton_destroy(struct REGEX_INTERNAL_Automaton *a)
Free the memory allocated by constructing the REGEX_INTERNAL_Automaton.
library to parse regular expressions into dfa
Definition of a command line option.
A 512-bit hashcode.
Mysql context.
Definition: mysql.c:77
Handle for a prepared statement.
Definition: mysql.c:114
Information we pass to GNUNET_MY_exec_prepared() to initialize the arguments of the prepared statemen...
Definition: gnunet_my_lib.h:93
Information we pass to GNUNET_MY_extract_result() to initialize the arguments of the prepared stateme...
Entry in list of pending tasks.
Definition: scheduler.c:136
Time for absolute times used by GNUnet, in microseconds.
Time for relative time used by GNUnet, in microseconds.
Simple struct to keep track of progress, and print a nice little percentage meter for long running ta...
int print
Should the meter be printed?
unsigned int completed
Completed number.
unsigned int dotnum
Number of dots to print.
unsigned int modnum
Interval for printing percentage.
unsigned int total
Total number of elements.
char * startup_string
String to print on startup.
Edge representation.
Automaton representation.