gnunet-helper-fs-publish.c File Reference

Tool to help extract meta data asynchronously. More...

#include "platform.h"
#include "gnunet_fs_service.h"

Include dependency graph for gnunet-helper-fs-publish.c:

Go to the source code of this file.

Data Structures
struct	ScanTreeNode
	A node of a directory tree. More...
struct	RecursionContext
	Closure for the 'scan_callback'. More...

Functions
static void	free_tree (struct ScanTreeNode *tree)
	Free memory of the tree structure. More...
static int	write_all (const void *buf, size_t size)
	Write size bytes from buf into the output_stream. More...
static int	write_message (uint16_t message_type, const char *data, size_t data_length)
	Write message to the master process. More...
static int	preprocess_file (const char *filename, struct ScanTreeNode **dst)
	Function called to (recursively) add all of the files in the directory to the tree. More...
static int	scan_callback (void *cls, const char *filename)
	Function called by the directory iterator to (recursively) add all of the files in the directory to the tree. More...
static int	extract_files (struct ScanTreeNode *item)
	Extract metadata from files. More...
static void	ignore_sigpipe ()
	Install a signal handler to ignore SIGPIPE. More...
static void	make_dev_zero (int fd, int flags)
	Turn the given file descriptor in to '/dev/null'. More...
int	main (int argc, char *const *argv)
	Main function of the helper process to extract meta data. More...

Variables
static int	output_stream
	File descriptor we use for IPC with the parent. More...

Detailed Description

Tool to help extract meta data asynchronously.

Author

Christian Grothoff

This program will scan a directory for files with meta data and report the results to stdout.

Definition in file gnunet-helper-fs-publish.c.

Function Documentation

free_tree()

static void free_tree ( struct ScanTreeNode *  tree )

static

Free memory of the tree structure.

Parameters

tree	tree to free

Definition at line 164 of file gnunet-helper-fs-publish.c.

{
  struct ScanTreeNode *pos;
 
  while (NULL != (pos = tree->children_head))
    free_tree (pos);
  if (NULL != tree->parent)
    GNUNET_CONTAINER_DLL_remove (tree->parent->children_head,
                                 tree->parent->children_tail,
                                 tree);
  GNUNET_free (tree->filename);
  GNUNET_free (tree);
}

References ScanTreeNode::children_head, ScanTreeNode::children_tail, ScanTreeNode::filename, GNUNET_CONTAINER_DLL_remove, GNUNET_free, and ScanTreeNode::parent.

Referenced by main(), and preprocess_file().

Here is the caller graph for this function:

write_all()

static int write_all ( const void *  buf, size_t  size  )

static

Write size bytes from buf into the output_stream.

Parameters

buf	buffer with data to write
size	number of bytes to write

Returns

GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 187 of file gnunet-helper-fs-publish.c.

{
  const char *cbuf = buf;
  size_t total;
  ssize_t wr;
 
  total = 0;
  do
  {
    wr = write (output_stream, &cbuf[total], size - total);
    if (wr > 0)
      total += wr;
  }
  while ((wr > 0) && (total < size));
  if (wr <= 0)
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Failed to write to stdout: %s\n",
                strerror (errno));
  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
}

References buf, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_OK, GNUNET_SYSERR, output_stream, and size.

Referenced by write_message().

Here is the caller graph for this function:

write_message()

static int write_message ( uint16_t  message_type, const char *  data, size_t  data_length  )

static

Write message to the master process.

Parameters

message_type	message type to use
data	data to append, NULL for none
data_length	number of bytes in data

Returns

GNUNET_SYSERR to stop scanning (the pipe was broken somehow)

Definition at line 218 of file gnunet-helper-fs-publish.c.

{
  struct GNUNET_MessageHeader hdr;
 
#if 0
  fprintf (stderr,
           "Helper sends %u-byte message of type %u\n",
           (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
           (unsigned int) message_type);
#endif
  hdr.type = htons (message_type);
  hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
  if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
      (GNUNET_OK != write_all (data, data_length)))
    return GNUNET_SYSERR;
  return GNUNET_OK;
}

References data, GNUNET_OK, GNUNET_SYSERR, GNUNET_MessageHeader::size, GNUNET_MessageHeader::type, and write_all().

Referenced by extract_files(), GNUNET_TESTING_send_barrier_attach(), main(), and preprocess_file().

Here is the call graph for this function:

Here is the caller graph for this function:

preprocess_file()

static int preprocess_file ( const char *  filename, struct ScanTreeNode **  dst  )

static

Function called to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters

filename	file or directory to scan
dst	where to store the resulting share tree item; NULL is stored in dst upon recoverable errors (GNUNET_OK is returned)

Returns

GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 309 of file gnunet-helper-fs-publish.c.

{
  struct ScanTreeNode *item;
  struct stat sbuf;
  uint64_t fsize = 0;
 
  if ((0 != stat (filename, &sbuf)) ||
      ((! S_ISDIR (sbuf.st_mode)) &&
       (GNUNET_OK !=
        GNUNET_DISK_file_size (filename, &fsize, GNUNET_NO, GNUNET_YES))))
  {
    /* If the file doesn't exist (or is not stat-able for any other reason)
       skip it (but report it), but do continue. */
    if (GNUNET_OK !=
        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE,
                       filename,
                       strlen (filename) + 1))
      return GNUNET_SYSERR;
    /* recoverable error, store 'NULL' in *dst */
    *dst = NULL;
    return GNUNET_OK;
  }
 
  /* Report the progress */
  if (
    GNUNET_OK !=
    write_message (S_ISDIR (sbuf.st_mode)
                   ? GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
                   : GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE,
                   filename,
                   strlen (filename) + 1))
    return GNUNET_SYSERR;
  item = GNUNET_new (struct ScanTreeNode);
  item->filename = GNUNET_strdup (filename);
  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
  item->file_size = fsize;
  if (GNUNET_YES == item->is_directory)
  {
    struct RecursionContext rc;
 
    rc.parent = item;
    rc.stop = GNUNET_NO;
    GNUNET_DISK_directory_scan (filename, &scan_callback, &rc);
    if (
      (GNUNET_YES == rc.stop) ||
      (GNUNET_OK !=
       write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY,
                      "..",
                      3)))
    {
      free_tree (item);
      return GNUNET_SYSERR;
    }
  }
  *dst = item;
  return GNUNET_OK;
}

References ScanTreeNode::file_size, filename, ScanTreeNode::filename, free_tree(), GNUNET_DISK_directory_scan(), GNUNET_DISK_file_size(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_new, GNUNET_NO, GNUNET_OK, GNUNET_strdup, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, RecursionContext::parent, scan_callback(), RecursionContext::stop, and write_message().

Referenced by main(), and scan_callback().

Here is the call graph for this function:

Here is the caller graph for this function:

scan_callback()

static int scan_callback ( void *  cls, const char *  filename  )

static

Function called by the directory iterator to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters

cls	the struct RecursionContext
filename	file or directory to scan

Returns

GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 278 of file gnunet-helper-fs-publish.c.

{
  struct RecursionContext *rc = cls;
  struct ScanTreeNode *chld;
 
  if (GNUNET_OK != preprocess_file (filename, &chld))
  {
    rc->stop = GNUNET_YES;
    return GNUNET_SYSERR;
  }
  if (NULL == chld)
    return GNUNET_OK;
  chld->parent = rc->parent;
  GNUNET_CONTAINER_DLL_insert (rc->parent->children_head,
                               rc->parent->children_tail,
                               chld);
  return GNUNET_OK;
}

References ScanTreeNode::children_head, ScanTreeNode::children_tail, filename, GNUNET_CONTAINER_DLL_insert, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::parent, RecursionContext::parent, preprocess_file(), and RecursionContext::stop.

Referenced by preprocess_file().

Here is the call graph for this function:

Here is the caller graph for this function:

extract_files()

static int extract_files ( struct ScanTreeNode *  item )

static

Extract metadata from files.

Parameters

item	entry we are processing

Returns

GNUNET_OK on success, GNUNET_SYSERR on fatal errors

Definition at line 375 of file gnunet-helper-fs-publish.c.

{
  struct GNUNET_FS_MetaData *meta;
  ssize_t size;
  size_t slen;
 
  if (GNUNET_YES == item->is_directory)
  {
    /* for directories, we simply only descent, no extraction, no
       progress reporting */
    struct ScanTreeNode *pos;
 
    for (pos = item->children_head; NULL != pos; pos = pos->next)
      if (GNUNET_OK != extract_files (pos))
        return GNUNET_SYSERR;
    return GNUNET_OK;
  }
 
  /* this is the expensive operation, *afterwards* we'll check for aborts */
  meta = GNUNET_FS_meta_data_create ();
#if HAVE_LIBEXTRACTOR
  EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
#endif
  slen = strlen (item->filename) + 1;
  size = GNUNET_FS_meta_data_get_serialized_size (meta);
  if (-1 == size)
  {
    /* no meta data */
    GNUNET_FS_meta_data_destroy (meta);
    if (GNUNET_OK !=
        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
                       item->filename,
                       slen))
      return GNUNET_SYSERR;
    return GNUNET_OK;
  }
  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
  {
    /* We can't transfer more than 64k bytes in one message. */
    size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
  }
  {
    char buf[size + slen];
    char *dst = &buf[slen];
 
    GNUNET_memcpy (buf, item->filename, slen);
    size = GNUNET_FS_meta_data_serialize (
      meta,
      &dst,
      size,
      GNUNET_FS_META_DATA_SERIALIZE_PART);
    if (size < 0)
    {
      GNUNET_break (0);
      size = 0;
    }
    GNUNET_FS_meta_data_destroy (meta);
    if (GNUNET_OK !=
        write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA,
                       buf,
                       slen + size))
      return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}

References buf, ScanTreeNode::children_head, ScanTreeNode::filename, GNUNET_break, GNUNET_FS_meta_data_create(), GNUNET_FS_meta_data_destroy(), GNUNET_FS_meta_data_get_serialized_size(), GNUNET_FS_meta_data_serialize(), GNUNET_FS_META_DATA_SERIALIZE_PART, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, meta, ScanTreeNode::next, plugins, size, and write_message().

Referenced by main().

Here is the call graph for this function:

Here is the caller graph for this function:

ignore_sigpipe()

static void ignore_sigpipe ( )

static

Install a signal handler to ignore SIGPIPE.

Definition at line 446 of file gnunet-helper-fs-publish.c.

{
  struct sigaction oldsig;
  struct sigaction sig;
 
  memset (&sig, 0, sizeof(struct sigaction));
  sig.sa_handler = SIG_IGN;
  sigemptyset (&sig.sa_mask);
#ifdef SA_INTERRUPT
  sig.sa_flags = SA_INTERRUPT; /* SunOS */
#else
  sig.sa_flags = SA_RESTART;
#endif
  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
    fprintf (stderr,
             "Failed to install SIGPIPE handler: %s\n",
             strerror (errno));
}

Referenced by main().

Here is the caller graph for this function:

make_dev_zero()

static void make_dev_zero ( int  fd, int  flags  )

static

Turn the given file descriptor in to '/dev/null'.

Parameters

fd	fd to bind to /dev/null
flags	flags to use (O_RDONLY or O_WRONLY)

Definition at line 473 of file gnunet-helper-fs-publish.c.

{
  int z;
 
  GNUNET_assert (0 == close (fd));
  z = open ("/dev/null", flags);
  GNUNET_assert (-1 != z);
  if (z == fd)
    return;
  GNUNET_break (fd == dup2 (z, fd));
  GNUNET_assert (0 == close (z));
}

References GNUNET_assert, and GNUNET_break.

Referenced by main().

Here is the caller graph for this function:

main()

int main	(	int	argc,
		char *const *	argv
	)

Main function of the helper process to extract meta data.

Parameters

argc	should be 3
argv	[0] our binary name [1] name of the file or directory to process [2] "-" to disable extraction, NULL for defaults, otherwise custom plugins to load from LE

Returns

0 on success

Definition at line 498 of file gnunet-helper-fs-publish.c.

{
  const char *filename_expanded;
  const char *ex;
  struct ScanTreeNode *root;
 
  ignore_sigpipe ();
  /* move stdout to some other FD for IPC, bind
     stdout/stderr to /dev/null */
  output_stream = dup (1);
  make_dev_zero (1, O_WRONLY);
  make_dev_zero (2, O_WRONLY);
 
  /* parse command line */
  if ((3 != argc) && (2 != argc))
  {
    fprintf (stderr,
             "%s",
             "gnunet-helper-fs-publish needs exactly one or two arguments\n");
    return 1;
  }
  filename_expanded = argv[1];
  ex = argv[2];
  if ((NULL == ex) || (0 != strcmp (ex, "-")))
  {
#if HAVE_LIBEXTRACTOR
    plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
    if (NULL != ex)
      plugins = EXTRACTOR_plugin_add_config (plugins,
                                             ex,
                                             EXTRACTOR_OPTION_DEFAULT_POLICY);
#endif
  }
 
  /* scan tree to find out how much work there is to be done */
  if (GNUNET_OK != preprocess_file (filename_expanded, &root))
  {
    (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, NULL, 0);
#if HAVE_LIBEXTRACTOR
    EXTRACTOR_plugin_remove_all (plugins);
#endif
    return 2;
  }
  /* signal that we're done counting files, so that a percentage of
     progress can now be calculated */
  if (GNUNET_OK !=
      write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE,
                     NULL,
                     0))
  {
#if HAVE_LIBEXTRACTOR
    EXTRACTOR_plugin_remove_all (plugins);
#endif
    return 3;
  }
  if (NULL != root)
  {
    if (GNUNET_OK != extract_files (root))
    {
      (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR,
                            NULL,
                            0);
      free_tree (root);
#if HAVE_LIBEXTRACTOR
      EXTRACTOR_plugin_remove_all (plugins);
#endif
      return 4;
    }
    free_tree (root);
  }
  /* enable "clean" shutdown by telling parent that we are done */
  (void) write_message (GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED,
                        NULL,
                        0);
#if HAVE_LIBEXTRACTOR
  EXTRACTOR_plugin_remove_all (plugins);
#endif
  return 0;
}

References extract_files(), free_tree(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_OK, ignore_sigpipe(), make_dev_zero(), output_stream, plugins, preprocess_file(), and write_message().

Here is the call graph for this function:

Variable Documentation

output_stream

int output_stream

static

File descriptor we use for IPC with the parent.

Definition at line 94 of file gnunet-helper-fs-publish.c.

Referenced by main(), and write_all().