gnunet-auto-share.c

Go to the documentation of this file.

/*
     This file is part of GNUnet.
     Copyright (C) 2001--2012 GNUnet e.V.

     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.

     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.

     SPDX-License-Identifier: AGPL3.0-or-later
 */
#include "platform.h"
#include "gnunet_util_lib.h"

#define MAX_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_HOURS, 4)

#define MIN_DELAY GNUNET_TIME_UNIT_MINUTES


struct WorkItem {
  struct WorkItem *prev;

  struct WorkItem *next;

  char *filename;

  struct GNUNET_HashCode id;
};


static int ret;

static unsigned int verbose;

static const struct GNUNET_CONFIGURATION_Handle *cfg;

static char *cfg_filename;

static int disable_extractor;

static int do_disable_creation_time;

static struct GNUNET_SCHEDULER_Task *run_task;

static unsigned int anonymity_level = 1;

static unsigned int content_priority = 365;

static unsigned int replication_level = 1;

static const char *dir_name;

static struct WorkItem *work_head;

static struct WorkItem *work_tail;

static struct GNUNET_CONTAINER_MultiHashMap *work_finished;

static int do_shutdown;

static struct GNUNET_TIME_Absolute start_time;

static struct GNUNET_DISK_PipeHandle *sigpipe;

static struct GNUNET_OS_Process *publish_proc;


static char *
get_state_file()
{
  char *ret;

  GNUNET_asprintf(&ret,
                  "%s%s.auto-share",
                  dir_name,
                  (DIR_SEPARATOR == dir_name[strlen(dir_name) - 1])
                  ? ""
                  : DIR_SEPARATOR_STR);
  return ret;
}


static void
load_state()
{
  char *fn;
  struct GNUNET_BIO_ReadHandle *rh;
  uint32_t n;
  struct GNUNET_HashCode id;
  struct WorkItem *wi;
  char *emsg;

  emsg = NULL;
  fn = get_state_file();
  rh = GNUNET_BIO_read_open(fn);
  GNUNET_free(fn);
  if (NULL == rh)
    return;
  fn = NULL;
  if (GNUNET_OK != GNUNET_BIO_read_int32(rh, &n))
    goto error;
  while (n-- > 0)
    {
      if ((GNUNET_OK != GNUNET_BIO_read_string(rh, "filename", &fn, 1024)) ||
          (GNUNET_OK !=
           GNUNET_BIO_read(rh, "id", &id, sizeof(struct GNUNET_HashCode))))
        goto error;
      wi = GNUNET_new(struct WorkItem);
      wi->id = id;
      wi->filename = fn;
      GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
                 "Loaded serialization ID for `%s' is `%s'\n",
                 wi->filename,
                 GNUNET_h2s(&id));
      fn = NULL;
      GNUNET_CRYPTO_hash(wi->filename, strlen(wi->filename), &id);
      GNUNET_break(GNUNET_OK ==
                   GNUNET_CONTAINER_multihashmap_put(
                     work_finished,
                     &id,
                     wi,
                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
    }
  if (GNUNET_OK == GNUNET_BIO_read_close(rh, &emsg))
    return;
  rh = NULL;
error:
  GNUNET_free_non_null(fn);
  if (NULL != rh)
    (void)GNUNET_BIO_read_close(rh, &emsg);
  GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
             _("Failed to load state: %s\n"),
             emsg);
  GNUNET_free_non_null(emsg);
}


static int
write_item(void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct GNUNET_BIO_WriteHandle *wh = cls;
  struct WorkItem *wi = value;

  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG,
             "Saving serialization ID of file `%s' with value `%s'\n",
             wi->filename,
             GNUNET_h2s(&wi->id));
  if ((GNUNET_OK != GNUNET_BIO_write_string(wh, wi->filename)) ||
      (GNUNET_OK !=
       GNUNET_BIO_write(wh, &wi->id, sizeof(struct GNUNET_HashCode))))
    return GNUNET_SYSERR; /* write error, abort iteration */
  return GNUNET_OK;
}


static void
save_state()
{
  uint32_t n;
  struct GNUNET_BIO_WriteHandle *wh;
  char *fn;

  n = GNUNET_CONTAINER_multihashmap_size(work_finished);
  fn = get_state_file();
  wh = GNUNET_BIO_write_open(fn);
  if (NULL == wh)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
                 _("Failed to save state to file %s\n"),
                 fn);
      GNUNET_free(fn);
      return;
    }
  if (GNUNET_OK != GNUNET_BIO_write_int32(wh, n))
    {
      (void)GNUNET_BIO_write_close(wh);
      GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
                 _("Failed to save state to file %s\n"),
                 fn);
      GNUNET_free(fn);
      return;
    }
  (void)GNUNET_CONTAINER_multihashmap_iterate(work_finished, &write_item, wh);
  if (GNUNET_OK != GNUNET_BIO_write_close(wh))
    GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
               _("Failed to save state to file %s\n"),
               fn);
  GNUNET_free(fn);
}


static void
do_stop_task(void *cls)
{
  do_shutdown = GNUNET_YES;
  if (NULL != publish_proc)
    {
      GNUNET_OS_process_kill(publish_proc, SIGKILL);
      return;
    }
  if (NULL != run_task)
    {
      GNUNET_SCHEDULER_cancel(run_task);
      run_task = NULL;
    }
}


static void
schedule_next_task(void);


static void
maint_child_death(void *cls)
{
  struct WorkItem *wi = cls;
  struct GNUNET_HashCode key;
  enum GNUNET_OS_ProcessStatusType type;
  unsigned long code;
  int ret;
  char c;
  const struct GNUNET_DISK_FileHandle *pr;
  const struct GNUNET_SCHEDULER_TaskContext *tc;

  run_task = NULL;
  pr = GNUNET_DISK_pipe_handle(sigpipe, GNUNET_DISK_PIPE_END_READ);
  tc = GNUNET_SCHEDULER_get_task_context();
  if (0 == (tc->reason & GNUNET_SCHEDULER_REASON_READ_READY))
    {
      /* shutdown scheduled us, someone else will kill child,
         we should just try again */
      run_task = GNUNET_SCHEDULER_add_read_file(GNUNET_TIME_UNIT_FOREVER_REL,
                                                pr,
                                                &maint_child_death,
                                                wi);
      return;
    }
  /* consume the signal */
  GNUNET_break(0 < GNUNET_DISK_file_read(pr, &c, sizeof(c)));

  ret = GNUNET_OS_process_status(publish_proc, &type, &code);
  GNUNET_assert(GNUNET_SYSERR != ret);
  if (GNUNET_NO == ret)
    {
      /* process still running? Then where did the SIGCHLD come from?
         Well, let's declare it spurious (kernel bug?) and keep rolling.
       */
      GNUNET_break(0);
      run_task = GNUNET_SCHEDULER_add_read_file(GNUNET_TIME_UNIT_FOREVER_REL,
                                                pr,
                                                &maint_child_death,
                                                wi);
      return;
    }
  GNUNET_assert(GNUNET_OK == ret);

  GNUNET_OS_process_destroy(publish_proc);
  publish_proc = NULL;

  if (GNUNET_YES == do_shutdown)
    {
      GNUNET_free(wi->filename);
      GNUNET_free(wi);
      return;
    }
  if ((GNUNET_OS_PROCESS_EXITED == type) && (0 == code))
    {
      GNUNET_log(GNUNET_ERROR_TYPE_INFO,
                 _("Publication of `%s' done\n"),
                 wi->filename);
      GNUNET_CRYPTO_hash(wi->filename, strlen(wi->filename), &key);
      GNUNET_break(GNUNET_OK ==
                   GNUNET_CONTAINER_multihashmap_put(
                     work_finished,
                     &key,
                     wi,
                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
    }
  else
    {
      GNUNET_CONTAINER_DLL_insert_tail(work_head, work_tail, wi);
    }
  save_state();
  schedule_next_task();
}


static void
sighandler_child_death()
{
  static char c;
  int old_errno = errno; /* back-up errno */

  GNUNET_break(
    1 ==
    GNUNET_DISK_file_write(GNUNET_DISK_pipe_handle(sigpipe,
                                                   GNUNET_DISK_PIPE_END_WRITE),
                           &c,
                           sizeof(c)));
  errno = old_errno; /* restore errno */
}


static void
work(void *cls)
{
  static char *argv[14];
  static char anon_level[20];
  static char content_prio[20];
  static char repl_level[20];
  struct WorkItem *wi;
  const struct GNUNET_DISK_FileHandle *pr;
  int argc;

  run_task = NULL;
  wi = work_head;
  GNUNET_CONTAINER_DLL_remove(work_head, work_tail, wi);
  argc = 0;
  argv[argc++] = "gnunet-publish";
  if (verbose)
    argv[argc++] = "-V";
  if (disable_extractor)
    argv[argc++] = "-D";
  if (do_disable_creation_time)
    argv[argc++] = "-d";
  argv[argc++] = "-c";
  argv[argc++] = cfg_filename;
  GNUNET_snprintf(anon_level, sizeof(anon_level), "%u", anonymity_level);
  argv[argc++] = "-a";
  argv[argc++] = anon_level;
  GNUNET_snprintf(content_prio, sizeof(content_prio), "%u", content_priority);
  argv[argc++] = "-p";
  argv[argc++] = content_prio;
  GNUNET_snprintf(repl_level, sizeof(repl_level), "%u", replication_level);
  argv[argc++] = "-r";
  argv[argc++] = repl_level;
  argv[argc++] = wi->filename;
  argv[argc] = NULL;
  GNUNET_log(GNUNET_ERROR_TYPE_INFO, _("Publishing `%s'\n"), wi->filename);
  GNUNET_assert(NULL == publish_proc);
  publish_proc = GNUNET_OS_start_process_vap(GNUNET_YES,
                                             0,
                                             NULL,
                                             NULL,
                                             NULL,
                                             "gnunet-publish",
                                             argv);
  if (NULL == publish_proc)
    {
      GNUNET_log(GNUNET_ERROR_TYPE_ERROR,
                 _("Failed to run `%s'\n"),
                 "gnunet-publish");
      GNUNET_CONTAINER_DLL_insert(work_head, work_tail, wi);
      run_task =
        GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_UNIT_MINUTES, &work, NULL);
      return;
    }
  pr = GNUNET_DISK_pipe_handle(sigpipe, GNUNET_DISK_PIPE_END_READ);
  run_task = GNUNET_SCHEDULER_add_read_file(GNUNET_TIME_UNIT_FOREVER_REL,
                                            pr,
                                            &maint_child_death,
                                            wi);
}


static int
determine_id(void *cls, const char *filename)
{
  struct GNUNET_HashCode *id = cls;
  struct stat sbuf;
  struct GNUNET_HashCode fx[2];
  struct GNUNET_HashCode ft;

  if (0 != stat(filename, &sbuf))
    {
      GNUNET_log_strerror_file(GNUNET_ERROR_TYPE_WARNING, "stat", filename);
      return GNUNET_OK;
    }
  GNUNET_CRYPTO_hash(filename, strlen(filename), &fx[0]);
  if (!S_ISDIR(sbuf.st_mode))
    {
      uint64_t fattr[2];

      fattr[0] = GNUNET_htonll(sbuf.st_size);
      fattr[0] = GNUNET_htonll(sbuf.st_mtime);

      GNUNET_CRYPTO_hash(fattr, sizeof(fattr), &fx[1]);
    }
  else
    {
      memset(&fx[1], 1, sizeof(struct GNUNET_HashCode));
      GNUNET_DISK_directory_scan(filename, &determine_id, &fx[1]);
    }
  /* use hash here to make hierarchical structure distinct from
     all files on the same level */
  GNUNET_CRYPTO_hash(fx, sizeof(fx), &ft);
  /* use XOR here so that order of the files in the directory
     does not matter! */
  GNUNET_CRYPTO_hash_xor(&ft, id, id);
  return GNUNET_OK;
}


static int
add_file(void *cls, const char *filename)
{
  struct WorkItem *wi;
  struct GNUNET_HashCode key;
  struct GNUNET_HashCode id;

  if (GNUNET_YES == do_shutdown)
    return GNUNET_SYSERR;
  if ((NULL != strstr(filename, "/.auto-share")) ||
      (NULL != strstr(filename, "\\.auto-share")))
    return GNUNET_OK; /* skip internal file */
  GNUNET_CRYPTO_hash(filename, strlen(filename), &key);
  wi = GNUNET_CONTAINER_multihashmap_get(work_finished, &key);
  memset(&id, 0, sizeof(struct GNUNET_HashCode));
  determine_id(&id, filename);
  if (NULL != wi)
    {
      if (0 == memcmp(&id, &wi->id, sizeof(struct GNUNET_HashCode)))
        return GNUNET_OK; /* skip: we did this one already */
      /* contents changed, need to re-do the directory... */
      GNUNET_assert(
        GNUNET_YES ==
        GNUNET_CONTAINER_multihashmap_remove(work_finished, &key, wi));
    }
  else
    {
      wi = GNUNET_new(struct WorkItem);
      wi->filename = GNUNET_strdup(filename);
    }
  wi->id = id;
  GNUNET_CONTAINER_DLL_insert(work_head, work_tail, wi);
  if (GNUNET_YES == do_shutdown)
    return GNUNET_SYSERR;
  return GNUNET_OK;
}


static void
scan(void *cls)
{
  run_task = NULL;
  start_time = GNUNET_TIME_absolute_get();
  (void)GNUNET_DISK_directory_scan(dir_name, &add_file, NULL);
  schedule_next_task();
}


static void
schedule_next_task()
{
  struct GNUNET_TIME_Relative delay;

  if (GNUNET_YES == do_shutdown)
    return;
  GNUNET_assert(NULL == run_task);
  if (NULL == work_head)
    {
      /* delay by at most 4h, at least 1s, and otherwise in between depending
         on how long it took to scan */
      delay = GNUNET_TIME_absolute_get_duration(start_time);
      delay = GNUNET_TIME_relative_saturating_multiply(delay, 100);
      delay = GNUNET_TIME_relative_min(delay, MAX_DELAY);
      delay = GNUNET_TIME_relative_max(delay, MIN_DELAY);
      run_task = GNUNET_SCHEDULER_add_delayed(delay, &scan, NULL);
    }
  else
    {
      run_task = GNUNET_SCHEDULER_add_now(&work, NULL);
    }
}


static void
run(void *cls,
    char *const *args,
    const char *cfgfile,
    const struct GNUNET_CONFIGURATION_Handle *c)
{
  /* check arguments */
  if ((NULL == args[0]) || (NULL != args[1]) ||
      (GNUNET_YES != GNUNET_DISK_directory_test(args[0], GNUNET_YES)))
    {
      printf(_(
               "You must specify one and only one directory name for automatic publication.\n"));
      ret = -1;
      return;
    }
  cfg_filename = GNUNET_strdup(cfgfile);
  cfg = c;
  dir_name = args[0];
  work_finished = GNUNET_CONTAINER_multihashmap_create(1024, GNUNET_NO);
  load_state();
  run_task = GNUNET_SCHEDULER_add_with_priority(GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                &scan,
                                                NULL);
  GNUNET_SCHEDULER_add_shutdown(&do_stop_task, NULL);
}


static int
free_item(void *cls, const struct GNUNET_HashCode *key, void *value)
{
  struct WorkItem *wi = value;

  GNUNET_free(wi->filename);
  GNUNET_free(wi);
  return GNUNET_OK;
}


int
main(int argc, char *const *argv)
{
  struct GNUNET_GETOPT_CommandLineOption options[] = {
    GNUNET_GETOPT_option_uint('a',
                              "anonymity",
                              "LEVEL",
                              gettext_noop(
                                "set the desired LEVEL of sender-anonymity"),
                              &anonymity_level),

    GNUNET_GETOPT_option_flag(
      'd',
      "disable-creation-time",
      gettext_noop(
        "disable adding the creation time to the metadata of the uploaded file"),
      &do_disable_creation_time),

    GNUNET_GETOPT_option_flag(
      'D',
      "disable-extractor",
      gettext_noop("do not use libextractor to add keywords or metadata"),
      &disable_extractor),

    GNUNET_GETOPT_option_uint('p',
                              "priority",
                              "PRIORITY",
                              gettext_noop(
                                "specify the priority of the content"),
                              &content_priority),

    GNUNET_GETOPT_option_uint('r',
                              "replication",
                              "LEVEL",
                              gettext_noop(
                                "set the desired replication LEVEL"),
                              &replication_level),

    GNUNET_GETOPT_option_verbose(&verbose),

    GNUNET_GETOPT_OPTION_END
  };
  struct WorkItem *wi;
  int ok;
  struct GNUNET_SIGNAL_Context *shc_chld;

  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args(argc, argv, &argc, &argv))
    return 2;
  sigpipe = GNUNET_DISK_pipe(GNUNET_NO, GNUNET_NO, GNUNET_NO, GNUNET_NO);
  GNUNET_assert(NULL != sigpipe);
  shc_chld =
    GNUNET_SIGNAL_handler_install(GNUNET_SIGCHLD, &sighandler_child_death);
  ok =
    (GNUNET_OK ==
     GNUNET_PROGRAM_run(
       argc,
       argv,
       "gnunet-auto-share [OPTIONS] FILENAME",
       gettext_noop("Automatically publish files from a directory on GNUnet"),
       options,
       &run,
       NULL))
    ? ret
    : 1;
  if (NULL != work_finished)
    {
      (void)GNUNET_CONTAINER_multihashmap_iterate(work_finished,
                                                  &free_item,
                                                  NULL);
      GNUNET_CONTAINER_multihashmap_destroy(work_finished);
    }
  while (NULL != (wi = work_head))
    {
      GNUNET_CONTAINER_DLL_remove(work_head, work_tail, wi);
      GNUNET_free(wi->filename);
      GNUNET_free(wi);
    }
  GNUNET_SIGNAL_handler_uninstall(shc_chld);
  shc_chld = NULL;
  GNUNET_DISK_pipe_close(sigpipe);
  sigpipe = NULL;
  GNUNET_free_non_null(cfg_filename);
  cfg_filename = NULL;
  GNUNET_free((void *)argv);
  return ok;
}

/* end of gnunet-auto-share.c */