gnunet-helper-fs-publish.c

Go to the documentation of this file.

/*
     This file is part of GNUnet.
     Copyright (C) 2012 GNUnet e.V.
 
     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.
 
     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.
 
     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
     SPDX-License-Identifier: AGPL3.0-or-later
 */
 
#include "platform.h"
 
#include "gnunet_fs_service.h"
 
 
struct ScanTreeNode
{
  struct ScanTreeNode *next;
 
  struct ScanTreeNode *prev;
 
  struct ScanTreeNode *parent;
 
  struct ScanTreeNode *children_head;
 
  struct ScanTreeNode *children_tail;
 
  char *filename;
 
  uint64_t file_size;
 
  int is_directory;
};
 
 
#if HAVE_LIBEXTRACTOR
static struct EXTRACTOR_PluginList *plugins;
#endif
 
static int output_stream;
 
 
#if HAVE_LIBEXTRACTOR
static int
add_to_md (void *cls,
           const char *plugin_name,
           enum EXTRACTOR_MetaType type,
           enum EXTRACTOR_MetaFormat format,
           const char *data_mime_type,
           const char *data,
           size_t data_len)
{
  struct GNUNET_FS_MetaData *md = cls;
 
  if (((EXTRACTOR_METAFORMAT_UTF8 == format) ||
       (EXTRACTOR_METAFORMAT_C_STRING == format)) &&
      ('\0' != data[data_len - 1]))
  {
    char zdata[data_len + 1];
    GNUNET_memcpy (zdata, data, data_len);
    zdata[data_len] = '\0';
    (void) GNUNET_FS_meta_data_insert (md,
                                       plugin_name,
                                       type,
                                       format,
                                       data_mime_type,
                                       zdata,
                                       data_len + 1);
  }
  else
  {
    (void) GNUNET_FS_meta_data_insert (md,
                                       plugin_name,
                                       type,
                                       format,
                                       data_mime_type,
                                       data,
                                       data_len);
  }
  return 0;
}
 
 
#endif
 
 
static void
free_tree (struct ScanTreeNode *tree)
{
  struct ScanTreeNode *pos;
 
  while (NULL != (pos = tree->children_head))
    free_tree (pos);
  if (NULL != tree->parent)
    GNUNET_CONTAINER_DLL_remove (tree->parent->children_head,
                                 tree->parent->children_tail,
                                 tree);
  GNUNET_free (tree->filename);
  GNUNET_free (tree);
}
 
 
static int
write_all (const void *buf, size_t size)
{
  const char *cbuf = buf;
  size_t total;
  ssize_t wr;
 
  total = 0;
  do
  {
    wr = write (output_stream, &cbuf[total], size - total);
    if (wr > 0)
      total += wr;
  }
  while ((wr > 0) && (total < size));
  if (wr <= 0)
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Failed to write to stdout: %s\n",
                strerror (errno));
  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
}
 
 
static int
write_message (uint16_t message_type, const char *data, size_t data_length)
{
  struct GNUNET_MessageHeader hdr;
 
#if 0
  fprintf (stderr,
           "Helper sends %u-byte message of type %u\n",
           (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
           (unsigned int) message_type);
#endif
  hdr.type = htons (message_type);
  hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
  if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
      (GNUNET_OK != write_all (data, data_length)))
    return GNUNET_SYSERR;
  return GNUNET_OK;
}
 
 
static int
preprocess_file (const char *filename, struct ScanTreeNode **dst);
 
 
struct RecursionContext
{
  struct ScanTreeNode *parent;
 
  int stop;
};
 
 
static int
scan_callback (void *cls, const char *filename)
{
  struct RecursionContext *rc = cls;
  struct ScanTreeNode *chld;
 
  if (GNUNET_OK != preprocess_file (filename, &chld))
  {
    rc->stop = GNUNET_YES;
    return GNUNET_SYSERR;
  }
  if (NULL == chld)
    return GNUNET_OK;
  chld->parent = rc->parent;
  GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
                               rc->parent->children_tail,
                               chld);
  return GNUNET_OK;
}
 
 
static int
preprocess_file (const char *filename, struct ScanTreeNode **dst)
{
  struct ScanTreeNode *item;
  struct stat sbuf;
  uint64_t fsize = 0;
 
  if ((0 != stat (filename, &sbuf)) ||
      ((! S_ISDIR (sbuf.st_mode)) &&
       (GNUNET_OK !=
        GNUNET_DISK_file_size (filename, &fsize, GNUNET_NO, GNUNET_YES))))
  {
    /* If the file doesn't exist (or is not stat-able for any other reason)
       skip it (but report it), but do continue. */
    if (GNUNET_OK !=
        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE,
                       filename,
                       strlen (filename) + 1))
      return GNUNET_SYSERR;
    /* recoverable error, store 'NULL' in *dst */
    *dst = NULL;
    return GNUNET_OK;
  }
 
  /* Report the progress */
  if (
    GNUNET_OK !=
    write_message (S_ISDIR (sbuf.st_mode)
                   ? GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
                   : GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE,
                   filename,
                   strlen (filename) + 1))
    return GNUNET_SYSERR;
  item = GNUNET_new (struct ScanTreeNode);
  item->filename = GNUNET_strdup (filename);
  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
  item->file_size = fsize;
  if (GNUNET_YES == item->is_directory)
  {
    struct RecursionContext rc;
 
    rc.parent = item;
    rc.stop = GNUNET_NO;
    GNUNET_DISK_directory_scan (filename, &scan_callback, &rc);
    if (
      (GNUNET_YES == rc.stop) ||
      (GNUNET_OK !=
       write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY,
                      "..",
                      3)))
    {
      free_tree (item);
      return GNUNET_SYSERR;
    }
  }
  *dst = item;
  return GNUNET_OK;
}
 
 
static int
extract_files (struct ScanTreeNode *item)
{
  struct GNUNET_FS_MetaData *meta;
  ssize_t size;
  size_t slen;
 
  if (GNUNET_YES == item->is_directory)
  {
    /* for directories, we simply only descent, no extraction, no
       progress reporting */
    struct ScanTreeNode *pos;
 
    for (pos = item->children_head; NULL != pos; pos = pos->next)
      if (GNUNET_OK != extract_files (pos))
        return GNUNET_SYSERR;
    return GNUNET_OK;
  }
 
  /* this is the expensive operation, *afterwards* we'll check for aborts */
  meta = GNUNET_FS_meta_data_create ();
#if HAVE_LIBEXTRACTOR
  EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
#endif
  slen = strlen (item->filename) + 1;
  size = GNUNET_FS_meta_data_get_serialized_size (meta);
  if (-1 == size)
  {
    /* no meta data */
    GNUNET_FS_meta_data_destroy (meta);
    if (GNUNET_OK !=
        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
                       item->filename,
                       slen))
      return GNUNET_SYSERR;
    return GNUNET_OK;
  }
  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
  {
    /* We can't transfer more than 64k bytes in one message. */
    size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
  }
  {
    char buf[size + slen];
    char *dst = &buf[slen];
 
    GNUNET_memcpy (buf, item->filename, slen);
    size = GNUNET_FS_meta_data_serialize (
      meta,
      &dst,
      size,
      GNUNET_FS_META_DATA_SERIALIZE_PART);
    if (size < 0)
    {
      GNUNET_break (0);
      size = 0;
    }
    GNUNET_FS_meta_data_destroy (meta);
    if (GNUNET_OK !=
        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
                       buf,
                       slen + size))
      return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}
 
 
static void
ignore_sigpipe ()
{
  struct sigaction oldsig;
  struct sigaction sig;
 
  memset (&sig, 0, sizeof(struct sigaction));
  sig.sa_handler = SIG_IGN;
  sigemptyset (&sig.sa_mask);
#ifdef SA_INTERRUPT
  sig.sa_flags = SA_INTERRUPT; /* SunOS */
#else
  sig.sa_flags = SA_RESTART;
#endif
  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
    fprintf (stderr,
             "Failed to install SIGPIPE handler: %s\n",
             strerror (errno));
}
 
 
static void
make_dev_zero (int fd, int flags)
{
  int z;
 
  GNUNET_assert (0 == close (fd));
  z = open ("/dev/null", flags);
  GNUNET_assert (-1 != z);
  if (z == fd)
    return;
  GNUNET_break (fd == dup2 (z, fd));
  GNUNET_assert (0 == close (z));
}
 
 
int
main (int argc, char *const *argv)
{
  const char *filename_expanded;
  const char *ex;
  struct ScanTreeNode *root;
 
  ignore_sigpipe ();
  /* move stdout to some other FD for IPC, bind
     stdout/stderr to /dev/null */
  output_stream = dup (1);
  make_dev_zero (1, O_WRONLY);
  make_dev_zero (2, O_WRONLY);
 
  /* parse command line */
  if ((3 != argc) && (2 != argc))
  {
    fprintf (stderr,
             "%s",
             "gnunet-helper-fs-publish needs exactly one or two arguments\n");
    return 1;
  }
  filename_expanded = argv[1];
  ex = argv[2];
  if ((NULL == ex) || (0 != strcmp (ex, "-")))
  {
#if HAVE_LIBEXTRACTOR
    plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
    if (NULL != ex)
      plugins = EXTRACTOR_plugin_add_config (plugins,
                                             ex,
                                             EXTRACTOR_OPTION_DEFAULT_POLICY);
#endif
  }
 
  /* scan tree to find out how much work there is to be done */
  if (GNUNET_OK != preprocess_file (filename_expanded, &root))
  {
    (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0);
#if HAVE_LIBEXTRACTOR
    EXTRACTOR_plugin_remove_all (plugins);
#endif
    return 2;
  }
  /* signal that we're done counting files, so that a percentage of
     progress can now be calculated */
  if (GNUNET_OK !=
      write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE,
                     NULL,
                     0))
  {
#if HAVE_LIBEXTRACTOR
    EXTRACTOR_plugin_remove_all (plugins);
#endif
    return 3;
  }
  if (NULL != root)
  {
    if (GNUNET_OK != extract_files (root))
    {
      (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR,
                            NULL,
                            0);
      free_tree (root);
#if HAVE_LIBEXTRACTOR
      EXTRACTOR_plugin_remove_all (plugins);
#endif
      return 4;
    }
    free_tree (root);
  }
  /* enable "clean" shutdown by telling parent that we are done */
  (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED,
                        NULL,
                        0);
#if HAVE_LIBEXTRACTOR
  EXTRACTOR_plugin_remove_all (plugins);
#endif
  return 0;
}
 
 
/* end of gnunet-helper-fs-publish.c */