fs_dirmetascan.c File Reference

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'. More...

#include "platform.h"
#include "gnunet_fs_service.h"
#include "gnunet_scheduler_lib.h"
#include <pthread.h>

Include dependency graph for fs_dirmetascan.c:

Go to the source code of this file.

Data Structures
struct	GNUNET_FS_DirScanner
	An opaque structure a pointer to which is returned to the caller to be used to control the scanner. More...

Functions
void	GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
	Abort the scan. More...
struct GNUNET_FS_ShareTreeItem *	GNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
	Obtain the result of the scan after the scan has signalled completion. More...
static struct GNUNET_FS_ShareTreeItem *	advance (struct GNUNET_FS_ShareTreeItem *pos)
	Move in the directory from the given position to the next file in DFS traversal. More...
static struct GNUNET_FS_ShareTreeItem *	expand_tree (struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
	Add another child node to the tree. More...
static void	finish_scan (void *cls)
	Task run last to shut everything down. More...
static int	process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
	Called every time there is data to read from the scanner. More...
static void	helper_died_cb (void *cls)
	Function called if our helper process died. More...
struct GNUNET_FS_DirScanner *	GNUNET_FS_directory_scan_start (const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
	Start a directory scanner thread. More...

Detailed Description

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.

Author

LRN

Christian Grothoff

Definition in file fs_dirmetascan.c.

Function Documentation

advance()

static struct GNUNET_FS_ShareTreeItem * advance ( struct GNUNET_FS_ShareTreeItem *  pos )

static

Move in the directory from the given position to the next file in DFS traversal.

Parameters

pos	current position

Returns

next file, NULL for none

Definition at line 139 of file fs_dirmetascan.c.

{
  int moved;
 
  GNUNET_assert (NULL != pos);
  moved = 0; /* must not terminate, even on file, otherwise "normal" */
  while ((pos->is_directory == GNUNET_YES) || (0 == moved))
  {
    if ((moved != -1) && (NULL != pos->children_head))
    {
      pos = pos->children_head;
      moved = 1;     /* can terminate if file */
      continue;
    }
    if (NULL != pos->next)
    {
      pos = pos->next;
      moved = 1;     /* can terminate if file */
      continue;
    }
    if (NULL != pos->parent)
    {
      pos = pos->parent;
      moved = -1;     /* force move to 'next' or 'parent' */
      continue;
    }
    /* no more options, end of traversal */
    return NULL;
  }
  return pos;
}

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_assert, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::next, and GNUNET_FS_ShareTreeItem::parent.

Referenced by process_helper_msgs().

Here is the caller graph for this function:

expand_tree()

static struct GNUNET_FS_ShareTreeItem * expand_tree ( struct GNUNET_FS_ShareTreeItem *  parent, const char *  filename, int  is_directory  )

static

Add another child node to the tree.

Parameters

parent	parent of the child, NULL for top level
filename	name of the file or directory
is_directory	GNUNET_YES for directories

Returns

new entry that was just created

Definition at line 181 of file fs_dirmetascan.c.

{
  struct GNUNET_FS_ShareTreeItem *chld;
  size_t slen;
 
  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
  chld->parent = parent;
  chld->filename = GNUNET_strdup (filename);
  GNUNET_asprintf (&chld->short_filename,
                   "%s%s",
                   GNUNET_STRINGS_get_short_name (filename),
                   is_directory == GNUNET_YES ? "/" : "");
  /* make sure we do not end with '//' */
  slen = strlen (chld->short_filename);
  if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
      (chld->short_filename[slen - 2] == '/'))
    chld->short_filename[slen - 1] = '\0';
  chld->is_directory = is_directory;
  if (NULL != parent)
    GNUNET_CONTAINER_DLL_insert (parent->children_head,
                                 parent->children_tail,
                                 chld);
  return chld;
}

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_FS_ShareTreeItem::children_tail, filename, GNUNET_FS_ShareTreeItem::filename, GNUNET_asprintf(), GNUNET_CONTAINER_DLL_insert, GNUNET_new, GNUNET_strdup, GNUNET_STRINGS_get_short_name(), GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::parent, and GNUNET_FS_ShareTreeItem::short_filename.

Referenced by process_helper_msgs().

Here is the call graph for this function:

Here is the caller graph for this function:

finish_scan()

static void finish_scan ( void *  cls )

static

Task run last to shut everything down.

Parameters

cls	the 'struct GNUNET_FS_DirScanner'

Definition at line 215 of file fs_dirmetascan.c.

{
  struct GNUNET_FS_DirScanner *ds = cls;
 
  ds->stop_task = NULL;
  if (NULL != ds->helper)
  {
    GNUNET_HELPER_stop (ds->helper, GNUNET_NO);
    ds->helper = NULL;
  }
  ds->progress_callback (ds->progress_callback_cls,
                         NULL,
                         GNUNET_SYSERR,
                         GNUNET_FS_DIRSCANNER_FINISHED);
}

References ds, GNUNET_FS_DIRSCANNER_FINISHED, GNUNET_HELPER_stop(), GNUNET_NO, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by process_helper_msgs().

Here is the call graph for this function:

Here is the caller graph for this function:

process_helper_msgs()

static int process_helper_msgs ( void *  cls, const struct GNUNET_MessageHeader *  msg  )

static

Called every time there is data to read from the scanner.

Calls the scanner progress handler.

Parameters

cls	the closure (directory scanner object)
msg	message from the helper process

Returns

GNUNET_OK on success, GNUNET_NO to stop further processing (no error) GNUNET_SYSERR to stop further processing with error

Definition at line 243 of file fs_dirmetascan.c.

{
  struct GNUNET_FS_DirScanner *ds = cls;
  const char *filename;
  size_t left;
 
#if 0
  fprintf (stderr,
           "DMS parses %u-byte message of type %u\n",
           (unsigned int) ntohs (msg->size),
           (unsigned int) ntohs (msg->type));
#endif
  left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
  filename = (const char *) &msg[1];
  switch (ntohs (msg->type))
  {
  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE:
    if (filename[left - 1] != '\0')
    {
      GNUNET_break (0);
      break;
    }
    ds->progress_callback (ds->progress_callback_cls,
                           filename,
                           GNUNET_NO,
                           GNUNET_FS_DIRSCANNER_FILE_START);
    if (NULL == ds->toplevel)
    {
      ds->toplevel = expand_tree (ds->pos, filename, GNUNET_NO);
    }
    else
    {
      GNUNET_assert (NULL != ds->pos);
      (void) expand_tree (ds->pos, filename, GNUNET_NO);
    }
    return GNUNET_OK;
 
  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY:
    if (filename[left - 1] != '\0')
    {
      GNUNET_break (0);
      break;
    }
    if (0 == strcmp ("..", filename))
    {
      if (NULL == ds->pos)
      {
        GNUNET_break (0);
        break;
      }
      ds->pos = ds->pos->parent;
      return GNUNET_OK;
    }
    ds->progress_callback (ds->progress_callback_cls,
                           filename,
                           GNUNET_YES,
                           GNUNET_FS_DIRSCANNER_FILE_START);
    ds->pos = expand_tree (ds->pos, filename, GNUNET_YES);
    if (NULL == ds->toplevel)
      ds->toplevel = ds->pos;
    return GNUNET_OK;
 
  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR:
    break;
 
  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE:
    if ('\0' != filename[left - 1])
      break;
    ds->progress_callback (ds->progress_callback_cls,
                           filename,
                           GNUNET_SYSERR,
                           GNUNET_FS_DIRSCANNER_FILE_IGNORED);
    return GNUNET_OK;
 
  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE:
    if (0 != left)
    {
      GNUNET_break (0);
      break;
    }
    if (NULL == ds->toplevel)
      break;
    ds->progress_callback (ds->progress_callback_cls,
                           NULL,
                           GNUNET_SYSERR,
                           GNUNET_FS_DIRSCANNER_ALL_COUNTED);
    ds->pos = ds->toplevel;
    if (GNUNET_YES == ds->pos->is_directory)
      ds->pos = advance (ds->pos);
    return GNUNET_OK;
 
  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA: {
      size_t nlen;
      const char *end;
 
      if (NULL == ds->pos)
      {
        GNUNET_break (0);
        break;
      }
      end = memchr (filename, 0, left);
      if (NULL == end)
      {
        GNUNET_break (0);
        break;
      }
      end++;
      nlen = end - filename;
      left -= nlen;
      if (0 != strcmp (filename, ds->pos->filename))
      {
        GNUNET_break (0);
        break;
      }
      ds->progress_callback (ds->progress_callback_cls,
                             filename,
                             GNUNET_YES,
                             GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED);
      if (0 < left)
      {
        ds->pos->meta = GNUNET_FS_meta_data_deserialize (end, left);
        if (NULL == ds->pos->meta)
        {
          GNUNET_break (0);
          break;
        }
        /* having full filenames is too dangerous; always make sure we clean them up */
        GNUNET_FS_meta_data_delete (ds->pos->meta,
                                    EXTRACTOR_METATYPE_FILENAME,
                                    NULL,
                                    0);
        /* instead, put in our 'safer' original filename */
        GNUNET_FS_meta_data_insert (ds->pos->meta,
                                    "<libgnunetfs>",
                                    EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME,
                                    EXTRACTOR_METAFORMAT_UTF8,
                                    "text/plain",
                                    ds->pos->short_filename,
                                    strlen (ds->pos->short_filename)
                                    + 1);
      }
      ds->pos->ksk_uri = GNUNET_FS_uri_ksk_create_from_meta_data (
        ds->pos->meta);
      ds->pos = advance (ds->pos);
      return GNUNET_OK;
    }
 
  case GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED:
    if (NULL != ds->pos)
    {
      GNUNET_break (0);
      break;
    }
    if (0 != left)
    {
      GNUNET_break (0);
      break;
    }
    if (NULL == ds->toplevel)
      break;
    ds->stop_task = GNUNET_SCHEDULER_add_now (&finish_scan, ds);
    return GNUNET_OK;
 
  default:
    GNUNET_break (0);
    break;
  }
  ds->progress_callback (ds->progress_callback_cls,
                         NULL,
                         GNUNET_SYSERR,
                         GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
  return GNUNET_OK;
}

References advance(), ds, end, expand_tree(), EXTRACTOR_METAFORMAT_UTF8, EXTRACTOR_METATYPE_FILENAME, EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, filename, GNUNET_FS_ShareTreeItem::filename, finish_scan(), GNUNET_assert, GNUNET_break, GNUNET_FS_DIRSCANNER_ALL_COUNTED, GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED, GNUNET_FS_DIRSCANNER_FILE_IGNORED, GNUNET_FS_DIRSCANNER_FILE_START, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_FS_meta_data_delete(), GNUNET_FS_meta_data_deserialize(), GNUNET_FS_meta_data_insert(), GNUNET_FS_uri_ksk_create_from_meta_data(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_NO, GNUNET_OK, GNUNET_SCHEDULER_add_now(), GNUNET_SYSERR, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::ksk_uri, GNUNET_FS_ShareTreeItem::meta, msg, GNUNET_FS_ShareTreeItem::parent, GNUNET_FS_DirScanner::pos, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, GNUNET_FS_ShareTreeItem::short_filename, GNUNET_MessageHeader::size, GNUNET_FS_DirScanner::stop_task, GNUNET_FS_DirScanner::toplevel, and GNUNET_MessageHeader::type.

Referenced by GNUNET_FS_directory_scan_start().

Here is the call graph for this function:

Here is the caller graph for this function:

helper_died_cb()

static void helper_died_cb ( void *  cls )

static

Function called if our helper process died.

Parameters

cls	the 'struct GNUNET_FS_DirScanner' callback.

Definition at line 424 of file fs_dirmetascan.c.

{
  struct GNUNET_FS_DirScanner *ds = cls;
 
  ds->helper = NULL;
  if (NULL != ds->stop_task)
    return; /* normal death, was finished */
  ds->progress_callback (ds->progress_callback_cls,
                         NULL,
                         GNUNET_SYSERR,
                         GNUNET_FS_DIRSCANNER_INTERNAL_ERROR);
}

References ds, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by GNUNET_FS_directory_scan_start().

Here is the caller graph for this function: