gnunet-regex-simulation-profiler.c

Go to the documentation of this file.

/*
     This file is part of GNUnet.
     Copyright (C) 2011, 2012 GNUnet e.V.
 
     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.
 
     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.
 
     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
     SPDX-License-Identifier: AGPL3.0-or-later
 */
 
 
#include "platform.h"
#include "gnunet_util_lib.h"
#include "regex_internal_lib.h"
#include "gnunet_mysql_lib.h"
#include "gnunet_mysql_compat.h"
#include "gnunet_my_lib.h"
#include <mysql/mysql.h>
 
#define INSERT_EDGE_STMT "INSERT IGNORE INTO `%s` " \
  "(`key`, `label`, `to_key`, `accepting`) " \
  "VALUES (?, ?, ?, ?);"
 
#define SELECT_KEY_STMT "SELECT COUNT(*) FROM `%s` " \
  "WHERE `key` = ? AND `label` = ?;"
 
struct ProgressMeter
{
  unsigned int total;
 
  unsigned int modnum;
 
  unsigned int dotnum;
 
  unsigned int completed;
 
  int print;
 
  char *startup_string;
};
 
 
static struct ProgressMeter *meter;
 
static struct GNUNET_SCHEDULER_Task *scan_task;
 
static int result;
 
static struct GNUNET_MYSQL_Context *mysql_ctx;
 
static struct GNUNET_MYSQL_StatementHandle *stmt_handle;
 
static struct GNUNET_MYSQL_StatementHandle *select_stmt_handle;
 
static char *table_name;
 
static char *policy_dir;
 
static unsigned int num_policy_files;
 
static unsigned int num_policies;
 
static unsigned int max_path_compression;
 
static unsigned long long num_merged_transitions;
 
static unsigned long long num_merged_states;
 
static char *regex_prefix;
 
 
static struct ProgressMeter *
create_meter (unsigned int total, char *start_string, int print)
{
  struct ProgressMeter *ret;
 
  ret = GNUNET_new (struct ProgressMeter);
  ret->print = print;
  ret->total = total;
  ret->modnum = total / 4;
  if (ret->modnum == 0)         /* Divide by zero check */
    ret->modnum = 1;
  ret->dotnum = (total / 50) + 1;
  if (start_string != NULL)
    ret->startup_string = GNUNET_strdup (start_string);
  else
    ret->startup_string = GNUNET_strdup ("");
 
  return ret;
}
 
 
static int
update_meter (struct ProgressMeter *meter)
{
  if (meter->print == GNUNET_YES)
  {
    if (meter->completed % meter->modnum == 0)
    {
      if (meter->completed == 0)
      {
        fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
      }
      else
        fprintf (stdout, "%d%%",
                 (int) (((float) meter->completed / meter->total) * 100));
    }
    else if (meter->completed % meter->dotnum == 0)
      fprintf (stdout, "%s", ".");
 
    if (meter->completed + 1 == meter->total)
      fprintf (stdout, "%d%%]\n", 100);
    fflush (stdout);
  }
  meter->completed++;
 
  if (meter->completed == meter->total)
    return GNUNET_YES;
  if (meter->completed > meter->total)
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Progress meter overflow!!\n");
  return GNUNET_NO;
}
 
 
static int
reset_meter (struct ProgressMeter *meter)
{
  if (meter == NULL)
    return GNUNET_SYSERR;
 
  meter->completed = 0;
  return GNUNET_YES;
}
 
 
static void
free_meter (struct ProgressMeter *meter)
{
  GNUNET_free (meter->startup_string);
  GNUNET_free (meter);
}
 
 
static void
do_shutdown (void *cls)
{
  if (NULL != mysql_ctx)
  {
    GNUNET_MYSQL_context_destroy (mysql_ctx);
    mysql_ctx = NULL;
  }
  if (NULL != meter)
  {
    free_meter (meter);
    meter = NULL;
  }
}
 
 
static void
do_abort (void *cls)
{
  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting\n");
  if (NULL != scan_task)
  {
    GNUNET_SCHEDULER_cancel (scan_task);
    scan_task = NULL;
  }
  result = GNUNET_SYSERR;
  GNUNET_SCHEDULER_shutdown ();
}
 
 
static void
regex_iterator (void *cls,
                const struct GNUNET_HashCode *key,
                const char *proof,
                int accepting,
                unsigned int num_edges,
                const struct REGEX_BLOCK_Edge *edges)
{
  unsigned int i;
  int result;
 
  uint32_t iaccepting = (uint32_t) accepting;
  uint64_t total;
 
  GNUNET_assert (NULL != mysql_ctx);
 
  for (i = 0; i < num_edges; i++)
  {
    struct GNUNET_MY_QueryParam params_select[] = {
      GNUNET_MY_query_param_auto_from_type (key),
      GNUNET_MY_query_param_string (edges[i].label),
      GNUNET_MY_query_param_end
    };
 
    struct GNUNET_MY_ResultSpec results_select[] = {
      GNUNET_MY_result_spec_uint64 (&total),
      GNUNET_MY_result_spec_end
    };
 
    result =
      GNUNET_MY_exec_prepared (mysql_ctx,
                               select_stmt_handle,
                               params_select);
 
    if (GNUNET_SYSERR == result)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  "Error executing prepared mysql select statement\n");
      GNUNET_SCHEDULER_add_now (&do_abort, NULL);
      return;
    }
 
    result =
      GNUNET_MY_extract_result (select_stmt_handle,
                                results_select);
 
    if (GNUNET_SYSERR == result)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  "Error extracting result mysql select statement\n");
      GNUNET_SCHEDULER_add_now (&do_abort, NULL);
      return;
    }
 
    if ((-1 != total) && (total > 0) )
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Total: %llu (%s, %s)\n",
                  (unsigned long long) total,
                  GNUNET_h2s (key), edges[i].label);
    }
 
    struct GNUNET_MY_QueryParam params_stmt[] = {
      GNUNET_MY_query_param_auto_from_type (&key),
      GNUNET_MY_query_param_string (edges[i].label),
      GNUNET_MY_query_param_auto_from_type (&edges[i].destination),
      GNUNET_MY_query_param_uint32 (&iaccepting),
      GNUNET_MY_query_param_end
    };
 
    result =
      GNUNET_MY_exec_prepared (mysql_ctx,
                               stmt_handle,
                               params_stmt);
 
    if (0 == result)
    {
      char *key_str = GNUNET_strdup (GNUNET_h2s (key));
      char *to_key_str = GNUNET_strdup (GNUNET_h2s (&edges[i].destination));
 
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Merged (%s, %s, %s, %i)\n",
                  key_str,
                  edges[i].label,
                  to_key_str,
                  accepting);
 
      GNUNET_free (key_str);
      GNUNET_free (to_key_str);
      num_merged_transitions++;
    }
    else if (-1 != total)
    {
      num_merged_states++;
    }
 
    if ((GNUNET_SYSERR == result) || ((1 != result) && (0 != result) ))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
                  result);
      GNUNET_SCHEDULER_add_now (&do_abort, NULL);
    }
  }
 
  if (0 == num_edges)
  {
    struct GNUNET_MY_QueryParam params_stmt[] = {
      GNUNET_MY_query_param_auto_from_type (key),
      GNUNET_MY_query_param_string (""),
      GNUNET_MY_query_param_fixed_size (NULL, 0),
      GNUNET_MY_query_param_uint32 (&iaccepting),
      GNUNET_MY_query_param_end
    };
 
    result =
      GNUNET_MY_exec_prepared (mysql_ctx,
                               stmt_handle,
                               params_stmt);
 
    if ((1 != result) && (0 != result) )
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  "Error executing prepared mysql statement for edge: Affected rows: %i, expected 0 or 1!\n",
                  result);
      GNUNET_SCHEDULER_add_now (&do_abort, NULL);
    }
  }
}
 
 
static int
announce_regex (const char *regex)
{
  struct REGEX_INTERNAL_Automaton *dfa;
 
  dfa =
    REGEX_INTERNAL_construct_dfa (regex,
                                  strlen (regex),
                                  max_path_compression);
 
  if (NULL == dfa)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to create DFA for regex %s\n",
                regex);
    GNUNET_SCHEDULER_add_now (&do_abort, NULL);
    return GNUNET_SYSERR;
  }
  REGEX_INTERNAL_iterate_all_edges (dfa,
                                    &regex_iterator, NULL);
  REGEX_INTERNAL_automaton_destroy (dfa);
 
  return GNUNET_OK;
}
 
 
static int
policy_filename_cb (void *cls, const char *filename)
{
  char *regex;
  char *data;
  char *buf;
  uint64_t filesize;
  unsigned int offset;
 
  GNUNET_assert (NULL != filename);
 
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Announcing regexes from file %s\n",
              filename);
 
  if (GNUNET_YES != GNUNET_DISK_file_test (filename))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Could not find policy file %s\n",
                filename);
    return GNUNET_OK;
  }
  if (GNUNET_OK !=
      GNUNET_DISK_file_size (filename, &filesize,
                             GNUNET_YES, GNUNET_YES))
    filesize = 0;
  if (0 == filesize)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Policy file %s is empty.\n",
                filename);
    return GNUNET_OK;
  }
  data = GNUNET_malloc (filesize);
  if (filesize != GNUNET_DISK_fn_read (filename, data, filesize))
  {
    GNUNET_free (data);
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Could not read policy file %s.\n",
                filename);
    return GNUNET_OK;
  }
 
  update_meter (meter);
 
  buf = data;
  offset = 0;
  regex = NULL;
  while (offset < (filesize - 1))
  {
    offset++;
    if (((data[offset] == '\n')) && (buf != &data[offset]))
    {
      data[offset] = '|';
      num_policies++;
      buf = &data[offset + 1];
    }
    else if ((data[offset] == '\n') || (data[offset] == '\0'))
      buf = &data[offset + 1];
  }
  data[offset] = '\0';
  GNUNET_asprintf (&regex, "%s(%s)", regex_prefix, data);
  GNUNET_assert (NULL != regex);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Announcing regex: %s\n", regex);
 
  if (GNUNET_OK != announce_regex (regex))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Could not announce regex %s\n",
                regex);
  }
  GNUNET_free (regex);
  GNUNET_free (data);
  return GNUNET_OK;
}
 
 
static void
do_directory_scan (void *cls)
{
  struct GNUNET_TIME_Absolute start_time;
  struct GNUNET_TIME_Relative duration;
  char *stmt;
 
  /* Create an MySQL prepared statement for the inserts */
  scan_task = NULL;
  GNUNET_asprintf (&stmt, INSERT_EDGE_STMT, table_name);
  stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
  GNUNET_free (stmt);
 
  GNUNET_asprintf (&stmt, SELECT_KEY_STMT, table_name);
  select_stmt_handle = GNUNET_MYSQL_statement_prepare (mysql_ctx, stmt);
  GNUNET_free (stmt);
 
  GNUNET_assert (NULL != stmt_handle);
 
  meter = create_meter (num_policy_files,
                        "Announcing policy files\n",
                        GNUNET_YES);
  start_time = GNUNET_TIME_absolute_get ();
  GNUNET_DISK_directory_scan (policy_dir,
                              &policy_filename_cb,
                              stmt_handle);
  duration = GNUNET_TIME_absolute_get_duration (start_time);
  reset_meter (meter);
  free_meter (meter);
  meter = NULL;
 
  printf ("Announced %u files containing %u policies in %s\n"
          "Duplicate transitions: %llu\nMerged states: %llu\n",
          num_policy_files,
          num_policies,
          GNUNET_STRINGS_relative_time_to_string (duration, GNUNET_NO),
          num_merged_transitions,
          num_merged_states);
  result = GNUNET_OK;
  GNUNET_SCHEDULER_shutdown ();
}
 
 
static void
run (void *cls,
     char *const *args,
     const char *cfgfile,
     const struct GNUNET_CONFIGURATION_Handle *config)
{
  if (NULL == args[0])
  {
    fprintf (stderr,
             _ ("No policy directory specified on command line. Exiting.\n"));
    result = GNUNET_SYSERR;
    return;
  }
  if (GNUNET_YES !=
      GNUNET_DISK_directory_test (args[0], GNUNET_YES))
  {
    fprintf (stderr,
             _ ("Specified policies directory does not exist. Exiting.\n"));
    result = GNUNET_SYSERR;
    return;
  }
  policy_dir = args[0];
 
  num_policy_files = GNUNET_DISK_directory_scan (policy_dir,
                                                 NULL, NULL);
  meter = NULL;
 
  if (NULL == table_name)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "No table name specified, using default \"NFA\".\n");
    table_name = "NFA";
  }
 
  mysql_ctx = GNUNET_MYSQL_context_create (config, "regex-mysql");
  if (NULL == mysql_ctx)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Failed to create mysql context\n");
    result = GNUNET_SYSERR;
    return;
  }
 
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_string (config,
                                             "regex-mysql",
                                             "REGEX_PREFIX",
                                             &regex_prefix))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                               "regex-mysql",
                               "REGEX_PREFIX");
    result = GNUNET_SYSERR;
    return;
  }
 
  result = GNUNET_OK;
  GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
                                 NULL);
  scan_task = GNUNET_SCHEDULER_add_now (&do_directory_scan, NULL);
}
 
 
int
main (int argc, char *const *argv)
{
  struct GNUNET_GETOPT_CommandLineOption options[] = {
    GNUNET_GETOPT_option_string ('t',
                                 "table",
                                 "TABLENAME",
                                 gettext_noop (
                                   "name of the table to write DFAs"),
                                 &table_name),
 
    GNUNET_GETOPT_option_uint ('p',
                               "max-path-compression",
                               "MAX_PATH_COMPRESSION",
                               gettext_noop ("maximum path compression length"),
                               &max_path_compression),
 
    GNUNET_GETOPT_OPTION_END
  };
  int ret;
 
  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
    return 2;
 
  result = GNUNET_SYSERR;
  ret =
    GNUNET_PROGRAM_run (argc, argv,
                        "gnunet-regex-simulationprofiler [OPTIONS] policy-dir",
                        _ ("Profiler for regex library"), options, &run, NULL);
  if (GNUNET_OK != ret)
    return ret;
  if (GNUNET_OK != result)
    return 1;
  return 0;
}