gnunet-auto-share.c

Go to the documentation of this file.

/*
     This file is part of GNUnet.
     Copyright (C) 2001--2012 GNUnet e.V.
 
     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.
 
     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.
 
     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
     SPDX-License-Identifier: AGPL3.0-or-later
 */
#include "platform.h"
#include "gnunet_util_lib.h"
 
#define MAX_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
 
#define MIN_DELAY GNUNET_TIME_UNIT_MINUTES
 
 
struct WorkItem
{
  struct WorkItem *prev;
 
  struct WorkItem *next;
 
  char *filename;
 
  struct GNUNET_HashCode id;
};
 
 
static int ret;
 
static unsigned int verbose;
 
static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
static char *cfg_filename;
 
static int disable_extractor;
 
static int do_disable_creation_time;
 
static struct GNUNET_SCHEDULER_Task *run_task;
 
static unsigned int anonymity_level = 1;
 
static unsigned int content_priority = 365;
 
static unsigned int replication_level = 1;
 
static const char *dir_name;
 
static struct WorkItem *work_head;
 
static struct WorkItem *work_tail;
 
static struct GNUNET_CONTAINER_MultiHashMap *work_finished;
 
static int do_shutdown;
 
static struct GNUNET_TIME_Absolute start_time;
 
static struct GNUNET_DISK_PipeHandle *sigpipe;
 
static struct GNUNET_Process *publish_proc;
 
 
static char *
get_state_file ()
{
  char *retval;
 
  GNUNET_asprintf (&retval,
                   "%s%s.auto-share",
                   dir_name,
                   (DIR_SEPARATOR == dir_name[strlen (dir_name) - 1])
                   ? ""
                   : DIR_SEPARATOR_STR);
  return retval;
}
 
 
static void
load_state ()
{
  char *fn;
  struct GNUNET_BIO_ReadHandle *rh;
  uint32_t n;
  struct GNUNET_HashCode id;
  struct WorkItem *wi;
  char *emsg;
 
  emsg = NULL;
  fn = get_state_file ();
  rh = GNUNET_BIO_read_open_file (fn);
  GNUNET_free (fn);
  if (NULL == rh)
    return;
  fn = NULL;
  if (GNUNET_OK != GNUNET_BIO_read_int32 (rh, "number of files",
                                          (int32_t *) &n))
    goto error;
  while (n-- > 0)
  {
    struct GNUNET_BIO_ReadSpec rs[] = {
      GNUNET_BIO_read_spec_string ("filename", &fn, 1024),
      GNUNET_BIO_read_spec_object ("id", &id, sizeof(struct GNUNET_HashCode)),
      GNUNET_BIO_read_spec_end (),
    };
    if (GNUNET_OK != GNUNET_BIO_read_spec_commit (rh, rs))
      goto error;
    wi = GNUNET_new (struct WorkItem);
    wi->id = id;
    wi->filename = fn;
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Loaded serialization ID for `%s' is `%s'\n",
                wi->filename,
                GNUNET_h2s (&id));
    fn = NULL;
    GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &id);
    GNUNET_break (GNUNET_OK ==
                  GNUNET_CONTAINER_multihashmap_put (
                    work_finished,
                    &id,
                    wi,
                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
  }
  if (GNUNET_OK == GNUNET_BIO_read_close (rh, &emsg))
    return;
  rh = NULL;
error:
  GNUNET_free (fn);
  if (NULL != rh)
    (void) GNUNET_BIO_read_close (rh, &emsg);
  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
              _ ("Failed to load state: %s\n"),
              emsg);
  GNUNET_free (emsg);
}
 
 
static int
write_item (void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct GNUNET_BIO_WriteHandle *wh = cls;
  struct WorkItem *wi = value;
 
  struct GNUNET_BIO_WriteSpec ws[] = {
    GNUNET_BIO_write_spec_string ("auto-share-write-item-filename",
                                  wi->filename),
    GNUNET_BIO_write_spec_object ("id", &wi->id, sizeof(struct
                                                        GNUNET_HashCode)),
    GNUNET_BIO_write_spec_end (),
  };
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Saving serialization ID of file `%s' with value `%s'\n",
              wi->filename,
              GNUNET_h2s (&wi->id));
  if (GNUNET_OK != GNUNET_BIO_write_spec_commit (wh, ws))
    return GNUNET_SYSERR; /* write error, abort iteration */
  return GNUNET_OK;
}
 
 
static void
save_state ()
{
  uint32_t n;
  struct GNUNET_BIO_WriteHandle *wh;
  char *fn;
 
  n = GNUNET_CONTAINER_multihashmap_size (work_finished);
  fn = get_state_file ();
  wh = GNUNET_BIO_write_open_file (fn);
  if (NULL == wh)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Failed to save state to file %s\n"),
                fn);
    GNUNET_free (fn);
    return;
  }
  if (GNUNET_OK != GNUNET_BIO_write_int32 (wh, "size of state", n))
  {
    (void) GNUNET_BIO_write_close (wh, NULL);
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Failed to save state to file %s\n"),
                fn);
    GNUNET_free (fn);
    return;
  }
  (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished, &write_item, wh);
  if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Failed to save state to file %s\n"),
                fn);
  GNUNET_free (fn);
}
 
 
static void
do_stop_task (void *cls)
{
  do_shutdown = GNUNET_YES;
  if (NULL != publish_proc)
  {
    GNUNET_break (GNUNET_OK ==
                  GNUNET_process_kill (publish_proc,
                                       SIGKILL));
    return;
  }
  if (NULL != run_task)
  {
    GNUNET_SCHEDULER_cancel (run_task);
    run_task = NULL;
  }
}
 
 
static void
schedule_next_task (void);
 
 
static void
maint_child_death (void *cls)
{
  struct WorkItem *wi = cls;
  struct GNUNET_HashCode key;
  enum GNUNET_OS_ProcessStatusType type;
  unsigned long code;
  int retval;
  char c;
  const struct GNUNET_DISK_FileHandle *pr;
  const struct GNUNET_SCHEDULER_TaskContext *tc;
 
  run_task = NULL;
  pr = GNUNET_DISK_pipe_handle (sigpipe, GNUNET_DISK_PIPE_END_READ);
  tc = GNUNET_SCHEDULER_get_task_context ();
  if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
  {
    /* shutdown scheduled us, someone else will kill child,
       we should just try again */
    run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
                                               pr,
                                               &maint_child_death,
                                               wi);
    return;
  }
  /* consume the signal */
  GNUNET_break (0 < GNUNET_DISK_file_read (pr, &c, sizeof(c)));
 
  retval = GNUNET_process_wait (publish_proc,
                                false,
                                &type,
                                &code);
  GNUNET_assert (GNUNET_SYSERR != retval);
  if (GNUNET_NO == retval)
  {
    /* process still running? Then where did the SIGCHLD come from?
       Well, let's declare it spurious (kernel bug?) and keep rolling.
     */
    GNUNET_break (0);
    run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
                                               pr,
                                               &maint_child_death,
                                               wi);
    return;
  }
  GNUNET_assert (GNUNET_OK == retval);
 
  GNUNET_process_destroy (publish_proc);
  publish_proc = NULL;
 
  if (GNUNET_YES == do_shutdown)
  {
    GNUNET_free (wi->filename);
    GNUNET_free (wi);
    return;
  }
  if ((GNUNET_OS_PROCESS_EXITED == type) && (0 == code))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Publication of `%s' done\n"),
                wi->filename);
    GNUNET_CRYPTO_hash (wi->filename, strlen (wi->filename), &key);
    GNUNET_break (GNUNET_OK ==
                  GNUNET_CONTAINER_multihashmap_put (
                    work_finished,
                    &key,
                    wi,
                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
  }
  else
  {
    GNUNET_CONTAINER_DLL_insert_tail (work_head, work_tail, wi);
  }
  save_state ();
  schedule_next_task ();
}
 
 
static void
sighandler_child_death ()
{
  static char c;
  int old_errno = errno; /* back-up errno */
 
  GNUNET_break (
    1 ==
    GNUNET_DISK_file_write (GNUNET_DISK_pipe_handle (sigpipe,
                                                     GNUNET_DISK_PIPE_END_WRITE)
                            ,
                            &c,
                            sizeof(c)));
  errno = old_errno; /* restore errno */
}
 
 
static void
work (void *cls)
{
  static const char *argv[14];
  static char anon_level[20];
  static char content_prio[20];
  static char repl_level[20];
  struct WorkItem *wi;
  const struct GNUNET_DISK_FileHandle *pr;
  int argc;
 
  run_task = NULL;
  wi = work_head;
  GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi);
  argc = 0;
  argv[argc++] = "gnunet-publish";
  if (verbose)
    argv[argc++] = "-V";
  if (disable_extractor)
    argv[argc++] = "-D";
  if (do_disable_creation_time)
    argv[argc++] = "-d";
  argv[argc++] = "-c";
  argv[argc++] = cfg_filename;
  GNUNET_snprintf (anon_level, sizeof(anon_level), "%u", anonymity_level);
  argv[argc++] = "-a";
  argv[argc++] = anon_level;
  GNUNET_snprintf (content_prio, sizeof(content_prio), "%u", content_priority);
  argv[argc++] = "-p";
  argv[argc++] = content_prio;
  GNUNET_snprintf (repl_level, sizeof(repl_level), "%u", replication_level);
  argv[argc++] = "-r";
  argv[argc++] = repl_level;
  argv[argc++] = wi->filename;
  argv[argc] = NULL;
  GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Publishing `%s'\n"), wi->filename);
  GNUNET_assert (NULL == publish_proc);
  publish_proc = GNUNET_process_create (GNUNET_OS_USE_PIPE_CONTROL);
  if (GNUNET_OK !=
      GNUNET_process_run_command_argv (publish_proc,
                                       "gnunet-publish",
                                       (const char **) argv))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("Failed to run `%s'\n"),
                "gnunet-publish");
    GNUNET_CONTAINER_DLL_insert (work_head,
                                 work_tail,
                                 wi);
    GNUNET_process_destroy (publish_proc);
    publish_proc = NULL;
    run_task =
      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
                                    &work,
                                    NULL);
    return;
  }
  pr = GNUNET_DISK_pipe_handle (sigpipe,
                                GNUNET_DISK_PIPE_END_READ);
  run_task = GNUNET_SCHEDULER_add_read_file (GNUNET_TIME_UNIT_FOREVER_REL,
                                             pr,
                                             &maint_child_death,
                                             wi);
}
 
 
static int
determine_id (void *cls, const char *filename)
{
  struct GNUNET_HashCode *id = cls;
  struct stat sbuf;
  struct GNUNET_HashCode fx[2];
  struct GNUNET_HashCode ft;
 
  if (0 != stat (filename, &sbuf))
  {
    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "stat", filename);
    return GNUNET_OK;
  }
  GNUNET_CRYPTO_hash (filename, strlen (filename), &fx[0]);
  if (! S_ISDIR (sbuf.st_mode))
  {
    uint64_t fattr[2];
 
    fattr[0] = GNUNET_htonll (sbuf.st_size);
    fattr[0] = GNUNET_htonll (sbuf.st_mtime);
 
    GNUNET_CRYPTO_hash (fattr, sizeof(fattr), &fx[1]);
  }
  else
  {
    memset (&fx[1], 1, sizeof(struct GNUNET_HashCode));
    GNUNET_DISK_directory_scan (filename, &determine_id, &fx[1]);
  }
  /* use hash here to make hierarchical structure distinct from
     all files on the same level */
  GNUNET_CRYPTO_hash (fx, sizeof(fx), &ft);
  /* use XOR here so that order of the files in the directory
     does not matter! */
  GNUNET_CRYPTO_hash_xor (&ft, id, id);
  return GNUNET_OK;
}
 
 
static int
add_file (void *cls, const char *filename)
{
  struct WorkItem *wi;
  struct GNUNET_HashCode key;
  struct GNUNET_HashCode id;
 
  if (GNUNET_YES == do_shutdown)
    return GNUNET_SYSERR;
  if ((NULL != strstr (filename, "/.auto-share")) ||
      (NULL != strstr (filename, "\\.auto-share")))
    return GNUNET_OK; /* skip internal file */
  GNUNET_CRYPTO_hash (filename, strlen (filename), &key);
  wi = GNUNET_CONTAINER_multihashmap_get (work_finished, &key);
  memset (&id, 0, sizeof(struct GNUNET_HashCode));
  determine_id (&id, filename);
  if (NULL != wi)
  {
    if (0 == memcmp (&id, &wi->id, sizeof(struct GNUNET_HashCode)))
      return GNUNET_OK;   /* skip: we did this one already */
    /* contents changed, need to re-do the directory... */
    GNUNET_assert (
      GNUNET_YES ==
      GNUNET_CONTAINER_multihashmap_remove (work_finished, &key, wi));
  }
  else
  {
    wi = GNUNET_new (struct WorkItem);
    wi->filename = GNUNET_strdup (filename);
  }
  wi->id = id;
  GNUNET_CONTAINER_DLL_insert (work_head, work_tail, wi);
  if (GNUNET_YES == do_shutdown)
    return GNUNET_SYSERR;
  return GNUNET_OK;
}
 
 
static void
scan (void *cls)
{
  run_task = NULL;
  start_time = GNUNET_TIME_absolute_get ();
  (void) GNUNET_DISK_directory_scan (dir_name, &add_file, NULL);
  schedule_next_task ();
}
 
 
static void
schedule_next_task ()
{
  struct GNUNET_TIME_Relative delay;
 
  if (GNUNET_YES == do_shutdown)
    return;
  GNUNET_assert (NULL == run_task);
  if (NULL == work_head)
  {
    /* delay by at most 4h, at least 1s, and otherwise in between depending
       on how long it took to scan */
    delay = GNUNET_TIME_absolute_get_duration (start_time);
    delay = GNUNET_TIME_relative_saturating_multiply (delay, 100);
    delay = GNUNET_TIME_relative_min (delay, MAX_DELAY);
    delay = GNUNET_TIME_relative_max (delay, MIN_DELAY);
    run_task = GNUNET_SCHEDULER_add_delayed (delay, &scan, NULL);
  }
  else
  {
    run_task = GNUNET_SCHEDULER_add_now (&work, NULL);
  }
}
 
 
static void
run (void *cls,
     char *const *args,
     const char *cfgfile,
     const struct GNUNET_CONFIGURATION_Handle *c)
{
  /* check arguments */
  if ((NULL == args[0]) || (NULL != args[1]) ||
      (GNUNET_YES != GNUNET_DISK_directory_test (args[0], GNUNET_YES)))
  {
    printf (_ (
              "You must specify one and only one directory name for automatic publication.\n"));
    ret = -1;
    return;
  }
  cfg_filename = GNUNET_strdup (cfgfile);
  cfg = c;
  dir_name = args[0];
  work_finished = GNUNET_CONTAINER_multihashmap_create (1024, GNUNET_NO);
  load_state ();
  run_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                 &scan,
                                                 NULL);
  GNUNET_SCHEDULER_add_shutdown (&do_stop_task, NULL);
}
 
 
static int
free_item (void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct WorkItem *wi = value;
 
  GNUNET_free (wi->filename);
  GNUNET_free (wi);
  return GNUNET_OK;
}
 
 
int
main (int argc, char *const *argv)
{
  struct GNUNET_GETOPT_CommandLineOption options[] = {
    GNUNET_GETOPT_option_uint ('a',
                               "anonymity",
                               "LEVEL",
                               gettext_noop (
                                 "set the desired LEVEL of sender-anonymity"),
                               &anonymity_level),
 
    GNUNET_GETOPT_option_flag (
      'd',
      "disable-creation-time",
      gettext_noop (
        "disable adding the creation time to the metadata of the uploaded file")
      ,
      &do_disable_creation_time),
 
    GNUNET_GETOPT_option_flag (
      'D',
      "disable-extractor",
      gettext_noop ("do not use libextractor to add keywords or metadata"),
      &disable_extractor),
 
    GNUNET_GETOPT_option_uint ('p',
                               "priority",
                               "PRIORITY",
                               gettext_noop (
                                 "specify the priority of the content"),
                               &content_priority),
 
    GNUNET_GETOPT_option_uint ('r',
                               "replication",
                               "LEVEL",
                               gettext_noop (
                                 "set the desired replication LEVEL"),
                               &replication_level),
 
    GNUNET_GETOPT_option_verbose (&verbose),
 
    GNUNET_GETOPT_OPTION_END
  };
  struct WorkItem *wi;
  int ok;
  struct GNUNET_SIGNAL_Context *shc_chld;
 
  sigpipe = GNUNET_DISK_pipe (GNUNET_DISK_PF_NONE);
  GNUNET_assert (NULL != sigpipe);
  shc_chld =
    GNUNET_SIGNAL_handler_install (GNUNET_SIGCHLD, &sighandler_child_death);
  ok =
    (GNUNET_OK ==
     GNUNET_PROGRAM_run (
       GNUNET_OS_project_data_gnunet (),
       argc,
       argv,
       "gnunet-auto-share [OPTIONS] FILENAME",
       gettext_noop ("Automatically publish files from a directory on GNUnet"),
       options,
       &run,
       NULL))
    ? ret
    : 1;
  if (NULL != work_finished)
  {
    (void) GNUNET_CONTAINER_multihashmap_iterate (work_finished,
                                                  &free_item,
                                                  NULL);
    GNUNET_CONTAINER_multihashmap_destroy (work_finished);
  }
  while (NULL != (wi = work_head))
  {
    GNUNET_CONTAINER_DLL_remove (work_head, work_tail, wi);
    GNUNET_free (wi->filename);
    GNUNET_free (wi);
  }
  GNUNET_SIGNAL_handler_uninstall (shc_chld);
  shc_chld = NULL;
  GNUNET_DISK_pipe_close (sigpipe);
  sigpipe = NULL;
  GNUNET_free (cfg_filename);
  cfg_filename = NULL;
  return ok;
}
 
 
/* end of gnunet-auto-share.c */