GNUnet  0.11.x
gnunet-regex-simulation-profiler.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2011, 2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
21 
31 #include "platform.h"
32 #include "gnunet_util_lib.h"
33 #include "regex_internal_lib.h"
34 #include "gnunet_mysql_lib.h"
35 #include "gnunet_my_lib.h"
36 #include <mysql/mysql.h>
37 
41 #define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` " \
42  "(`key`, `label`, `to_key`, `accepting`) " \
43  "VALUES (?, ?, ?, ?);"
44 
48 #define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` " \
49  "WHERE `key` = ? AND `label` = ?;"
50 
56 {
60  unsigned int total;
61 
65  unsigned int modnum;
66 
70  unsigned int dotnum;
71 
75  unsigned int completed;
76 
80  int print;
81 
86 };
87 
88 
92 static struct ProgressMeter *meter;
93 
98 
102 static int result;
103 
108 
113 
118 
122 static char *table_name;
123 
127 static char *policy_dir;
128 
132 static unsigned int num_policy_files;
133 
137 static unsigned int num_policies;
138 
142 static unsigned int max_path_compression;
143 
147 static unsigned long long num_merged_transitions;
148 
152 static unsigned long long num_merged_states;
153 
157 static char *regex_prefix;
158 
159 
170 static struct ProgressMeter *
171 create_meter (unsigned int total, char *start_string, int print)
172 {
173  struct ProgressMeter *ret;
174 
175  ret = GNUNET_new (struct ProgressMeter);
176  ret->print = print;
177  ret->total = total;
178  ret->modnum = total / 4;
179  if (ret->modnum == 0) /* Divide by zero check */
180  ret->modnum = 1;
181  ret->dotnum = (total / 50) + 1;
182  if (start_string != NULL)
183  ret->startup_string = GNUNET_strdup (start_string);
184  else
185  ret->startup_string = GNUNET_strdup ("");
186 
187  return ret;
188 }
189 
190 
199 static int
201 {
202  if (meter->print == GNUNET_YES)
203  {
204  if (meter->completed % meter->modnum == 0)
205  {
206  if (meter->completed == 0)
207  {
208  fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
209  }
210  else
211  fprintf (stdout, "%d%%",
212  (int) (((float) meter->completed / meter->total) * 100));
213  }
214  else if (meter->completed % meter->dotnum == 0)
215  fprintf (stdout, "%s", ".");
216 
217  if (meter->completed + 1 == meter->total)
218  fprintf (stdout, "%d%%]\n", 100);
219  fflush (stdout);
220  }
221  meter->completed++;
222 
223  if (meter->completed == meter->total)
224  return GNUNET_YES;
225  if (meter->completed > meter->total)
226  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n");
227  return GNUNET_NO;
228 }
229 
230 
239 static int
241 {
242  if (meter == NULL)
243  return GNUNET_SYSERR;
244 
245  meter->completed = 0;
246  return GNUNET_YES;
247 }
248 
249 
255 static void
257 {
259  GNUNET_free (meter);
260 }
261 
262 
268 static void
269 do_shutdown (void *cls)
270 {
271  if (NULL != mysql_ctx)
272  {
274  mysql_ctx = NULL;
275  }
276  if (NULL != meter)
277  {
278  free_meter (meter);
279  meter = NULL;
280  }
281 }
282 
283 
295 static void
296 do_abort (void *cls)
297 {
298  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n");
299  if (NULL != scan_task)
300  {
302  scan_task = NULL;
303  }
306 }
307 
308 
319 static void
320 regex_iterator (void *cls,
321  const struct GNUNET_HashCode *key,
322  const char *proof,
323  int accepting,
324  unsigned int num_edges,
325  const struct REGEX_BLOCK_Edge *edges)
326 {
327  unsigned int i;
328  int result;
329 
330  uint32_t iaccepting = (uint32_t) accepting;
331  uint64_t total;
332 
333  GNUNET_assert (NULL != mysql_ctx);
334 
335  for (i = 0; i < num_edges; i++)
336  {
337  struct GNUNET_MY_QueryParam params_select[] = {
339  GNUNET_MY_query_param_string (edges[i].label),
341  };
342 
343  struct GNUNET_MY_ResultSpec results_select[] = {
346  };
347 
348  result =
351  params_select);
352 
353  if (GNUNET_SYSERR == result)
354  {
356  "Error executing prepared mysql select statement\n");
358  return;
359  }
360 
361  result =
363  results_select);
364 
365  if (GNUNET_SYSERR == result)
366  {
368  "Error extracting result mysql select statement\n");
370  return;
371  }
372 
373  if ((-1 != total) && (total > 0) )
374  {
375  GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n",
376  (unsigned long long) total,
377  GNUNET_h2s (key), edges[i].label);
378  }
379 
380  struct GNUNET_MY_QueryParam params_stmt[] = {
382  GNUNET_MY_query_param_string (edges[i].label),
383  GNUNET_MY_query_param_auto_from_type (&edges[i].destination),
384  GNUNET_MY_query_param_uint32 (&iaccepting),
386  };
387 
388  result =
390  stmt_handle,
391  params_stmt);
392 
393  if (0 == result)
394  {
395  char *key_str = GNUNET_strdup (GNUNET_h2s (key));
396  char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination));
397 
398  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n",
399  key_str,
400  edges[i].label,
401  to_key_str,
402  accepting);
403 
404  GNUNET_free (key_str);
405  GNUNET_free (to_key_str);
407  }
408  else if (-1 != total)
409  {
411  }
412 
413  if ((GNUNET_SYSERR == result) || ((1 != result) && (0 != result) ))
414  {
416  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
417  result);
419  }
420  }
421 
422  if (0 == num_edges)
423  {
424  struct GNUNET_MY_QueryParam params_stmt[] = {
428  GNUNET_MY_query_param_uint32 (&iaccepting),
430  };
431 
432  result =
434  stmt_handle,
435  params_stmt);
436 
437  if ((1 != result) && (0 != result) )
438  {
440  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
441  result);
443  }
444  }
445 }
446 
447 
455 static int
456 announce_regex (const char *regex)
457 {
458  struct REGEX_INTERNAL_Automaton *dfa;
459 
460  dfa =
462  strlen (regex),
464 
465  if (NULL == dfa)
466  {
468  "Failed to create DFA for regex %s\n",
469  regex);
471  return GNUNET_SYSERR;
472  }
474  &regex_iterator, NULL);
476 
477  return GNUNET_OK;
478 }
479 
480 
489 static int
490 policy_filename_cb (void *cls, const char *filename)
491 {
492  char *regex;
493  char *data;
494  char *buf;
495  uint64_t filesize;
496  unsigned int offset;
497 
498  GNUNET_assert (NULL != filename);
499 
501  "Announcing regexes from file %s\n",
502  filename);
503 
505  {
507  "Could not find policy file %s\n",
508  filename);
509  return GNUNET_OK;
510  }
511  if (GNUNET_OK !=
512  GNUNET_DISK_file_size (filename, &filesize,
514  filesize = 0;
515  if (0 == filesize)
516  {
517  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n",
518  filename);
519  return GNUNET_OK;
520  }
521  data = GNUNET_malloc (filesize);
522  if (filesize != GNUNET_DISK_fn_read (filename, data, filesize))
523  {
524  GNUNET_free (data);
526  "Could not read policy file %s.\n",
527  filename);
528  return GNUNET_OK;
529  }
530 
532 
533  buf = data;
534  offset = 0;
535  regex = NULL;
536  while (offset < (filesize - 1))
537  {
538  offset++;
539  if (((data[offset] == '\n')) && (buf != &data[offset]))
540  {
541  data[offset] = '|';
542  num_policies++;
543  buf = &data[offset + 1];
544  }
545  else if ((data[offset] == '\n') || (data[offset] == '\0'))
546  buf = &data[offset + 1];
547  }
548  data[offset] = '\0';
549  GNUNET_asprintf (&regex, "%s(%s)", regex_prefix, data);
550  GNUNET_assert (NULL != regex);
552  "Announcing regex: %s\n", regex);
553 
554  if (GNUNET_OK != announce_regex (regex))
555  {
557  "Could not announce regex %s\n",
558  regex);
559  }
560  GNUNET_free (regex);
561  GNUNET_free (data);
562  return GNUNET_OK;
563 }
564 
565 
571 static void
572 do_directory_scan (void *cls)
573 {
576  char *stmt;
577 
578  /* Create an MySQL prepared statement for the inserts */
579  scan_task = NULL;
582  GNUNET_free (stmt);
583 
586  GNUNET_free (stmt);
587 
588  GNUNET_assert (NULL != stmt_handle);
589 
591  "Announcing policy files\n",
592  GNUNET_YES);
596  stmt_handle);
598  reset_meter (meter);
599  free_meter (meter);
600  meter = NULL;
601 
602  printf ("Announced %u files containing %u policies in %s\n"
603  "Duplicate transitions: %llu\nMerged states: %llu\n",
605  num_policies,
609  result = GNUNET_OK;
611 }
612 
613 
622 static void
623 run (void *cls,
624  char *const *args,
625  const char *cfgfile,
626  const struct GNUNET_CONFIGURATION_Handle *config)
627 {
628  if (NULL == args[0])
629  {
630  fprintf (stderr,
631  _ ("No policy directory specified on command line. Exiting.\n"));
633  return;
634  }
635  if (GNUNET_YES !=
637  {
638  fprintf (stderr,
639  _ ("Specified policies directory does not exist. Exiting.\n"));
641  return;
642  }
643  policy_dir = args[0];
644 
646  NULL, NULL);
647  meter = NULL;
648 
649  if (NULL == table_name)
650  {
652  "No table name specified, using default \"NFA\".\n");
653  table_name = "NFA";
654  }
655 
656  mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql");
657  if (NULL == mysql_ctx)
658  {
660  "Failed to create mysql context\n");
662  return;
663  }
664 
665  if (GNUNET_OK !=
667  "regex-mysql",
668  "REGEX_PREFIX",
669  &regex_prefix))
670  {
672  "regex-mysql",
673  "REGEX_PREFIX");
675  return;
676  }
677 
678  result = GNUNET_OK;
680  NULL);
682 }
683 
684 
692 int
693 main (int argc, char *const *argv)
694 {
697  "table",
698  "TABLENAME",
699  gettext_noop (
700  "name of the table to write DFAs"),
701  &table_name),
702 
704  "max-path-compression",
705  "MAX_PATH_COMPRESSION",
706  gettext_noop ("maximum path compression length"),
708 
710  };
711  int ret;
712 
713  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
714  return 2;
715 
717  ret =
718  GNUNET_PROGRAM_run (argc, argv,
719  "gnunet-regex-simulationprofiler [OPTIONS] policy-dir",
720  _ ("Profiler for regex library"), options, &run, NULL);
721  if (GNUNET_OK != ret)
722  return ret;
723  if (GNUNET_OK != result)
724  return 1;
725  return 0;
726 }
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
#define gettext_noop(String)
Definition: gettext.h:69
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static struct GNUNET_TIME_Absolute start_time
Start time of the current round; used to determine how long one iteration takes (which influences how...
struct GNUNET_HashCode key
The key used in the DHT.
static char * filename
uint32_t data
The data value.
static void regex_iterator(void *cls, const struct GNUNET_HashCode *key, const char *proof, int accepting, unsigned int num_edges, const struct REGEX_BLOCK_Edge *edges)
Iterator over all states that inserts each state into the MySQL db.
static void do_directory_scan(void *cls)
Iterate over files contained in policy_dir.
static struct ProgressMeter * meter
Handle for the progress meter.
static void free_meter(struct ProgressMeter *meter)
Release resources for meter.
static unsigned int max_path_compression
Maximal path compression length.
static char * policy_dir
Policy dir containing files that contain policies.
#define SELECT_KEY_STMT
MySQL statement to select a key count.
#define INSERT_EDGE_STMT
MySQL statement to insert an edge.
static void do_abort(void *cls)
Abort task to run on test timed out.
static unsigned int num_policies
Number of policies.
static unsigned long long num_merged_states
Number of merged states from different policies.
static struct GNUNET_MYSQL_Context * mysql_ctx
MySQL context.
static void do_shutdown(void *cls)
Shutdown task.
static struct GNUNET_SCHEDULER_Task * scan_task
Scan task identifier;.
static unsigned int num_policy_files
Number of policy files.
static int policy_filename_cb(void *cls, const char *filename)
Function called with a filename.
static int result
Global testing status.
static char * table_name
MySQL table name.
static int update_meter(struct ProgressMeter *meter)
Update progress meter (increment by one).
static int announce_regex(const char *regex)
Announce a regex by creating the DFA and iterating over each state, inserting each state into a MySQL...
static struct GNUNET_MYSQL_StatementHandle * stmt_handle
MySQL prepared statement handle.
static int reset_meter(struct ProgressMeter *meter)
Reset progress meter.
int main(int argc, char *const *argv)
Main function.
static struct GNUNET_MYSQL_StatementHandle * select_stmt_handle
MySQL prepared statement handle for key select.
static struct ProgressMeter * create_meter(unsigned int total, char *start_string, int print)
Create a meter to keep track of the progress of some task.
static char * regex_prefix
Prefix to add before every regex we're announcing.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *config)
Main function that will be run by the scheduler.
static unsigned long long num_merged_transitions
Number of merged transitions.
static struct GNUNET_TIME_Relative duration
How long do we run the test?
static uint64_t proof
Definition: gnunet-scrypt.c:48
static char buf[2048]
static const struct GNUNET_CONFIGURATION_Handle * config
#define GNUNET_log(kind,...)
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition: disk.c:482
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:222
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_test(const char *fil, int is_readable)
Test if fil is a directory and listable.
Definition: disk.c:404
ssize_t GNUNET_DISK_fn_read(const char *fn, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:664
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:814
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_uint(char shortName, const char *name, const char *argumentHelp, const char *description, unsigned int *val)
Allow user to specify an unsigned int.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_string(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a string.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
#define GNUNET_MY_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
struct GNUNET_MYSQL_StatementHandle * GNUNET_MYSQL_statement_prepare(struct GNUNET_MYSQL_Context *mc, const char *query)
Prepare a statement.
Definition: mysql.c:393
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_string(const char *ptr)
Generate query parameter for a string.
struct GNUNET_MYSQL_Context * GNUNET_MYSQL_context_create(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section)
Create a mysql context.
Definition: mysql.c:320
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.FG.
void GNUNET_MYSQL_context_destroy(struct GNUNET_MYSQL_Context *mc)
Destroy a mysql context.
Definition: mysql.c:368
int GNUNET_MY_exec_prepared(struct GNUNET_MYSQL_Context *mc, struct GNUNET_MYSQL_StatementHandle *sh, struct GNUNET_MY_QueryParam *params)
Run a prepared SELECT statement.
Definition: my.c:42
struct GNUNET_MY_QueryParam GNUNET_MY_query_param_uint32(const uint32_t *x)
Generate query parameter for an uint32_t in host byte order.
#define GNUNET_MY_result_spec_end
End of result speceter specification.
struct GNUNET_MY_ResultSpec GNUNET_MY_result_spec_uint64(uint64_t *u64)
uint64_t expected.
int GNUNET_MY_extract_result(struct GNUNET_MYSQL_StatementHandle *sh, struct GNUNET_MY_ResultSpec *specs)
Extract results from a query result according to the given specification.
Definition: my.c:136
#define GNUNET_MY_query_param_end
End of query parameter specification.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:364
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:531
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1296
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received,...
Definition: scheduler.c:1331
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:972
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1209
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:263
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:86
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:557
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
void REGEX_INTERNAL_iterate_all_edges(struct REGEX_INTERNAL_Automaton *a, REGEX_INTERNAL_KeyIterator iterator, void *iterator_cls)
Iterate over all edges starting from start state of automaton 'a'.
struct REGEX_INTERNAL_Automaton * REGEX_INTERNAL_construct_dfa(const char *regex, const size_t len, unsigned int max_path_len)
Construct DFA for the given 'regex' of length 'len'.
void REGEX_INTERNAL_automaton_destroy(struct REGEX_INTERNAL_Automaton *a)
Free the memory allocated by constructing the REGEX_INTERNAL_Automaton data structure.
library to parse regular expressions into dfa
Definition of a command line option.
A 512-bit hashcode.
Mysql context.
Definition: mysql.c:76
Handle for a prepared statement.
Definition: mysql.c:113
Information we pass to GNUNET_MY_exec_prepared() to initialize the arguments of the prepared statemen...
Definition: gnunet_my_lib.h:84
Information we pass to GNUNET_MY_extract_result() to initialize the arguments of the prepared stateme...
Entry in list of pending tasks.
Definition: scheduler.c:135
Time for absolute times used by GNUnet, in microseconds.
Time for relative time used by GNUnet, in microseconds.
Simple struct to keep track of progress, and print a nice little percentage meter for long running ta...
int print
Should the meter be printed?
unsigned int completed
Completed number.
unsigned int dotnum
Number of dots to print.
unsigned int modnum
Interval for printing percentage.
unsigned int total
Total number of elements.
char * startup_string
String to print on startup.
Edge representation.
Automaton representation.