GNUnet  0.10.x
Data Structures | Functions
fs_dirmetascan.c File Reference

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'. More...

#include "platform.h"
#include "gnunet_fs_service.h"
#include "gnunet_scheduler_lib.h"
#include <pthread.h>
Include dependency graph for fs_dirmetascan.c:

Go to the source code of this file.

Data Structures

struct  GNUNET_FS_DirScanner
 An opaque structure a pointer to which is returned to the caller to be used to control the scanner. More...
 

Functions

void GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
 Abort the scan. More...
 
struct GNUNET_FS_ShareTreeItemGNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
 Obtain the result of the scan after the scan has signalled completion. More...
 
static struct GNUNET_FS_ShareTreeItemadvance (struct GNUNET_FS_ShareTreeItem *pos)
 Move in the directory from the given position to the next file in DFS traversal. More...
 
static struct GNUNET_FS_ShareTreeItemexpand_tree (struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
 Add another child node to the tree. More...
 
static void finish_scan (void *cls)
 Task run last to shut everything down. More...
 
static int process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
 Called every time there is data to read from the scanner. More...
 
static void helper_died_cb (void *cls)
 Function called if our helper process died. More...
 
struct GNUNET_FS_DirScannerGNUNET_FS_directory_scan_start (const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
 Start a directory scanner thread. More...
 

Detailed Description

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.

Author
LRN
Christian Grothoff

Definition in file fs_dirmetascan.c.

Function Documentation

◆ advance()

static struct GNUNET_FS_ShareTreeItem* advance ( struct GNUNET_FS_ShareTreeItem pos)
static

Move in the directory from the given position to the next file in DFS traversal.

Parameters
poscurrent position
Returns
next file, NULL for none

Definition at line 148 of file fs_dirmetascan.c.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_assert, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::next, GNUNET_FS_ShareTreeItem::parent, and GNUNET_FS_DirScanner::pos.

Referenced by process_helper_msgs().

149 {
150  int moved;
151 
152  GNUNET_assert (NULL != pos);
153  moved = 0; /* must not terminate, even on file, otherwise "normal" */
154  while ( (pos->is_directory == GNUNET_YES) ||
155  (0 == moved) )
156  {
157  if ( (moved != -1) &&
158  (NULL != pos->children_head) )
159  {
160  pos = pos->children_head;
161  moved = 1; /* can terminate if file */
162  continue;
163  }
164  if (NULL != pos->next)
165  {
166  pos = pos->next;
167  moved = 1; /* can terminate if file */
168  continue;
169  }
170  if (NULL != pos->parent)
171  {
172  pos = pos->parent;
173  moved = -1; /* force move to 'next' or 'parent' */
174  continue;
175  }
176  /* no more options, end of traversal */
177  return NULL;
178  }
179  return pos;
180 }
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_ShareTreeItem * next
This is a doubly-linked list.
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.
int is_directory
GNUNET_YES if this is a directory
#define GNUNET_YES
Definition: gnunet_common.h:80
Here is the caller graph for this function:

◆ expand_tree()

static struct GNUNET_FS_ShareTreeItem* expand_tree ( struct GNUNET_FS_ShareTreeItem parent,
const char *  filename,
int  is_directory 
)
static

Add another child node to the tree.

Parameters
parentparent of the child, NULL for top level
filenamename of the file or directory
is_directoryGNUNET_YES for directories
Returns
new entry that was just created

Definition at line 192 of file fs_dirmetascan.c.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_FS_ShareTreeItem::children_tail, GNUNET_FS_ShareTreeItem::filename, GNUNET_asprintf(), GNUNET_CONTAINER_DLL_insert, GNUNET_new, GNUNET_strdup, GNUNET_STRINGS_get_short_name(), GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::parent, and GNUNET_FS_ShareTreeItem::short_filename.

Referenced by process_helper_msgs().

195 {
196  struct GNUNET_FS_ShareTreeItem *chld;
197  size_t slen;
198 
199  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
200  chld->parent = parent;
201  chld->filename = GNUNET_strdup (filename);
203  "%s%s",
205  is_directory == GNUNET_YES ? "/" : "");
206  /* make sure we do not end with '//' */
207  slen = strlen (chld->short_filename);
208  if ( (slen >= 2) &&
209  (chld->short_filename[slen-1] == '/') &&
210  (chld->short_filename[slen-2] == '/') )
211  chld->short_filename[slen-1] = '\0';
212  chld->is_directory = is_directory;
213  if (NULL != parent)
215  parent->children_tail,
216  chld);
217  return chld;
218 }
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_ShareTreeItem * children_tail
This is a doubly-linked tree NULL for files and empty directories.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
int GNUNET_asprintf(char **buf, const char *format,...)
Like asprintf, just portable.
A node of a directory tree (produced by dirscanner)
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char * filename
char * short_filename
Base name of the file/directory.
const char * GNUNET_STRINGS_get_short_name(const char *filename)
"man basename" Returns a pointer to a part of filename (allocates nothing)!
Definition: strings.c:845
int is_directory
GNUNET_YES if this is a directory
char * filename
Name of the file/directory.
#define GNUNET_YES
Definition: gnunet_common.h:80
Here is the call graph for this function:
Here is the caller graph for this function:

◆ finish_scan()

static void finish_scan ( void *  cls)
static

Task run last to shut everything down.

Parameters
clsthe 'struct GNUNET_FS_DirScanner'

Definition at line 227 of file fs_dirmetascan.c.

References ds, GNUNET_FS_DIRSCANNER_FINISHED, GNUNET_HELPER_stop(), GNUNET_NO, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by process_helper_msgs().

228 {
229  struct GNUNET_FS_DirScanner *ds = cls;
230 
231  ds->stop_task = NULL;
232  if (NULL != ds->helper)
233  {
235  ds->helper = NULL;
236  }
238  NULL, GNUNET_SYSERR,
240 }
struct GNUNET_HELPER_Handle * helper
Helper process.
void * progress_callback_cls
A closure for progress_callback.
void GNUNET_HELPER_stop(struct GNUNET_HELPER_Handle *h, int soft_kill)
Kills the helper, closes the pipe, frees the handle and calls wait() on the helper process...
Definition: helper.c:569
#define GNUNET_NO
Definition: gnunet_common.h:81
An opaque structure a pointer to which is returned to the caller to be used to control the scanner...
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there&#39;s a progress message.
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
Last call to the progress function: we have finished scanning the directory.
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
Here is the call graph for this function:
Here is the caller graph for this function:

◆ process_helper_msgs()

static int process_helper_msgs ( void *  cls,
const struct GNUNET_MessageHeader msg 
)
static

Called every time there is data to read from the scanner.

Calls the scanner progress handler.

Parameters
clsthe closure (directory scanner object)
msgmessage from the helper process
Returns
GNUNET_OK on success, GNUNET_NO to stop further processing (no error) GNUNET_SYSERR to stop further processing with error

Definition at line 254 of file fs_dirmetascan.c.

References advance(), ds, end, expand_tree(), EXTRACTOR_METAFORMAT_UTF8, EXTRACTOR_METATYPE_FILENAME, EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, filename, GNUNET_FS_ShareTreeItem::filename, finish_scan(), GNUNET_assert, GNUNET_break, GNUNET_CONTAINER_meta_data_delete(), GNUNET_CONTAINER_meta_data_deserialize(), GNUNET_CONTAINER_meta_data_insert(), GNUNET_FS_DIRSCANNER_ALL_COUNTED, GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED, GNUNET_FS_DIRSCANNER_FILE_IGNORED, GNUNET_FS_DIRSCANNER_FILE_START, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_FS_uri_ksk_create_from_meta_data(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_NO, GNUNET_OK, GNUNET_SCHEDULER_add_now(), GNUNET_SYSERR, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::ksk_uri, GNUNET_FS_ShareTreeItem::meta, GNUNET_FS_ShareTreeItem::parent, GNUNET_FS_DirScanner::pos, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, GNUNET_FS_ShareTreeItem::short_filename, GNUNET_MessageHeader::size, GNUNET_FS_DirScanner::stop_task, GNUNET_FS_DirScanner::toplevel, and GNUNET_MessageHeader::type.

Referenced by GNUNET_FS_directory_scan_start().

256 {
257  struct GNUNET_FS_DirScanner *ds = cls;
258  const char *filename;
259  size_t left;
260 
261 #if 0
262  fprintf (stderr,
263  "DMS parses %u-byte message of type %u\n",
264  (unsigned int) ntohs (msg->size),
265  (unsigned int) ntohs (msg->type));
266 #endif
267  left = ntohs (msg->size) - sizeof (struct GNUNET_MessageHeader);
268  filename = (const char*) &msg[1];
269  switch (ntohs (msg->type))
270  {
272  if (filename[left-1] != '\0')
273  {
274  GNUNET_break (0);
275  break;
276  }
278  filename, GNUNET_NO,
280  if (NULL == ds->toplevel)
281  {
282  ds->toplevel = expand_tree (ds->pos,
283  filename,
284  GNUNET_NO);
285  }
286  else
287  {
288  GNUNET_assert (NULL != ds->pos);
289  (void) expand_tree (ds->pos,
290  filename,
291  GNUNET_NO);
292  }
293  return GNUNET_OK;
295  if (filename[left-1] != '\0')
296  {
297  GNUNET_break (0);
298  break;
299  }
300  if (0 == strcmp ("..", filename))
301  {
302  if (NULL == ds->pos)
303  {
304  GNUNET_break (0);
305  break;
306  }
307  ds->pos = ds->pos->parent;
308  return GNUNET_OK;
309  }
311  filename, GNUNET_YES,
313  ds->pos = expand_tree (ds->pos,
314  filename,
315  GNUNET_YES);
316  if (NULL == ds->toplevel)
317  ds->toplevel = ds->pos;
318  return GNUNET_OK;
320  break;
322  if ('\0' != filename[left-1])
323  break;
325  filename, GNUNET_SYSERR,
327  return GNUNET_OK;
329  if (0 != left)
330  {
331  GNUNET_break (0);
332  break;
333  }
334  if (NULL == ds->toplevel)
335  break;
337  NULL, GNUNET_SYSERR,
339  ds->pos = ds->toplevel;
340  if (GNUNET_YES == ds->pos->is_directory)
341  ds->pos = advance (ds->pos);
342  return GNUNET_OK;
344  {
345  size_t nlen;
346  const char *end;
347 
348  if (NULL == ds->pos)
349  {
350  GNUNET_break (0);
351  break;
352  }
353  end = memchr (filename, 0, left);
354  if (NULL == end)
355  {
356  GNUNET_break (0);
357  break;
358  }
359  end++;
360  nlen = end - filename;
361  left -= nlen;
362  if (0 != strcmp (filename,
363  ds->pos->filename))
364  {
365  GNUNET_break (0);
366  break;
367  }
369  filename,
370  GNUNET_YES,
372  if (0 < left)
373  {
375  left);
376  if (NULL == ds->pos->meta)
377  {
378  GNUNET_break (0);
379  break;
380  }
381  /* having full filenames is too dangerous; always make sure we clean them up */
384  NULL, 0);
385  /* instead, put in our 'safer' original filename */
386  GNUNET_CONTAINER_meta_data_insert (ds->pos->meta, "<libgnunetfs>",
388  EXTRACTOR_METAFORMAT_UTF8, "text/plain",
389  ds->pos->short_filename,
390  strlen (ds->pos->short_filename) + 1);
391  }
393  ds->pos = advance (ds->pos);
394  return GNUNET_OK;
395  }
397  if (NULL != ds->pos)
398  {
399  GNUNET_break (0);
400  break;
401  }
402  if (0 != left)
403  {
404  GNUNET_break (0);
405  break;
406  }
407  if (NULL == ds->toplevel)
408  break;
410  ds);
411  return GNUNET_OK;
412  default:
413  GNUNET_break (0);
414  break;
415  }
417  NULL, GNUNET_SYSERR,
419  return GNUNET_OK;
420 }
static void finish_scan(void *cls)
Task run last to shut everything down.
static int end
Set if we are to shutdown all services (including ARM).
Definition: gnunet-arm.c:34
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
static struct GNUNET_FS_ShareTreeItem * expand_tree(struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
Add another child node to the tree.
void * progress_callback_cls
A closure for progress_callback.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_Uri * ksk_uri
Keywords for this file or directory (derived from metadata).
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:81
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:78
An opaque structure a pointer to which is returned to the caller to be used to control the scanner...
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format...
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
We&#39;ve found all files (in the pre-pass).
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there&#39;s a progress message.
struct GNUNET_FS_ShareTreeItem * pos
Current position during processing.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1273
static char * filename
There was an internal error.
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
int GNUNET_CONTAINER_meta_data_delete(struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type, const char *data, size_t data_size)
Remove an item.
char * short_filename
Base name of the file/directory.
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
int is_directory
GNUNET_YES if this is a directory
char * filename
Name of the file/directory.
static struct GNUNET_FS_ShareTreeItem * advance(struct GNUNET_FS_ShareTreeItem *pos)
Move in the directory from the given position to the next file in DFS traversal.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
struct GNUNET_CONTAINER_MetaData * meta
Metadata for this file or directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
We&#39;ve started processing a file or directory.
struct GNUNET_FS_Uri * GNUNET_FS_uri_ksk_create_from_meta_data(const struct GNUNET_CONTAINER_MetaData *md)
Construct a keyword-URI from meta-data (take all entries in the meta-data and construct one large key...
Definition: fs_uri.c:1758
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
We&#39;ve finished extracting meta data from a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
struct GNUNET_FS_ShareTreeItem * toplevel
After the scan is finished, it will contain a pointer to the top-level directory entry in the directo...
Header for all communications.
#define GNUNET_YES
Definition: gnunet_common.h:80
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
We&#39;re having trouble accessing a file (soft-error); it will be ignored.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ helper_died_cb()

static void helper_died_cb ( void *  cls)
static

Function called if our helper process died.

Parameters
clsthe 'struct GNUNET_FS_DirScanner' callback.

Definition at line 429 of file fs_dirmetascan.c.

References ds, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by GNUNET_FS_directory_scan_start().

430 {
431  struct GNUNET_FS_DirScanner *ds = cls;
432 
433  ds->helper = NULL;
434  if (NULL != ds->stop_task)
435  return; /* normal death, was finished */
437  NULL, GNUNET_SYSERR,
439 }
struct GNUNET_HELPER_Handle * helper
Helper process.
void * progress_callback_cls
A closure for progress_callback.
An opaque structure a pointer to which is returned to the caller to be used to control the scanner...
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there&#39;s a progress message.
There was an internal error.
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
Here is the caller graph for this function: