GNUnet  0.11.x
Data Structures | Functions
fs_dirmetascan.c File Reference

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'. More...

#include "platform.h"
#include "gnunet_fs_service.h"
#include "gnunet_scheduler_lib.h"
#include <pthread.h>
Include dependency graph for fs_dirmetascan.c:

Go to the source code of this file.

Data Structures

struct  GNUNET_FS_DirScanner
 An opaque structure a pointer to which is returned to the caller to be used to control the scanner. More...
 

Functions

void GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
 Abort the scan. More...
 
struct GNUNET_FS_ShareTreeItemGNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
 Obtain the result of the scan after the scan has signalled completion. More...
 
static struct GNUNET_FS_ShareTreeItemadvance (struct GNUNET_FS_ShareTreeItem *pos)
 Move in the directory from the given position to the next file in DFS traversal. More...
 
static struct GNUNET_FS_ShareTreeItemexpand_tree (struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
 Add another child node to the tree. More...
 
static void finish_scan (void *cls)
 Task run last to shut everything down. More...
 
static int process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
 Called every time there is data to read from the scanner. More...
 
static void helper_died_cb (void *cls)
 Function called if our helper process died. More...
 
struct GNUNET_FS_DirScannerGNUNET_FS_directory_scan_start (const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
 Start a directory scanner thread. More...
 

Detailed Description

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.

Author
LRN
Christian Grothoff

Definition in file fs_dirmetascan.c.

Function Documentation

◆ advance()

static struct GNUNET_FS_ShareTreeItem* advance ( struct GNUNET_FS_ShareTreeItem pos)
static

Move in the directory from the given position to the next file in DFS traversal.

Parameters
poscurrent position
Returns
next file, NULL for none

Definition at line 146 of file fs_dirmetascan.c.

147 {
148  int moved;
149 
150  GNUNET_assert (NULL != pos);
151  moved = 0; /* must not terminate, even on file, otherwise "normal" */
152  while ((pos->is_directory == GNUNET_YES) || (0 == moved))
153  {
154  if ((moved != -1) && (NULL != pos->children_head))
155  {
156  pos = pos->children_head;
157  moved = 1; /* can terminate if file */
158  continue;
159  }
160  if (NULL != pos->next)
161  {
162  pos = pos->next;
163  moved = 1; /* can terminate if file */
164  continue;
165  }
166  if (NULL != pos->parent)
167  {
168  pos = pos->parent;
169  moved = -1; /* force move to 'next' or 'parent' */
170  continue;
171  }
172  /* no more options, end of traversal */
173  return NULL;
174  }
175  return pos;
176 }
@ GNUNET_YES
Definition: gnunet_common.h:97
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int is_directory
GNUNET_YES if this is a directory
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_ShareTreeItem * next
This is a doubly-linked list.
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_assert, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::next, and GNUNET_FS_ShareTreeItem::parent.

Referenced by process_helper_msgs().

Here is the caller graph for this function:

◆ expand_tree()

static struct GNUNET_FS_ShareTreeItem* expand_tree ( struct GNUNET_FS_ShareTreeItem parent,
const char *  filename,
int  is_directory 
)
static

Add another child node to the tree.

Parameters
parentparent of the child, NULL for top level
filenamename of the file or directory
is_directoryGNUNET_YES for directories
Returns
new entry that was just created

Definition at line 188 of file fs_dirmetascan.c.

191 {
192  struct GNUNET_FS_ShareTreeItem *chld;
193  size_t slen;
194 
195  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
196  chld->parent = parent;
197  chld->filename = GNUNET_strdup (filename);
199  "%s%s",
201  is_directory == GNUNET_YES ? "/" : "");
202  /* make sure we do not end with '//' */
203  slen = strlen (chld->short_filename);
204  if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
205  (chld->short_filename[slen - 2] == '/'))
206  chld->short_filename[slen - 1] = '\0';
207  chld->is_directory = is_directory;
208  if (NULL != parent)
211  chld);
212  return chld;
213 }
static char * filename
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
const char * GNUNET_STRINGS_get_short_name(const char *filename)
"man basename" Returns a pointer to a part of filename (allocates nothing)!
Definition: strings.c:626
A node of a directory tree (produced by dirscanner)
char * short_filename
Base name of the file/directory.
char * filename
Name of the file/directory.
struct GNUNET_FS_ShareTreeItem * children_tail
This is a doubly-linked tree NULL for files and empty directories.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_FS_ShareTreeItem::children_tail, filename, GNUNET_FS_ShareTreeItem::filename, GNUNET_asprintf(), GNUNET_CONTAINER_DLL_insert, GNUNET_new, GNUNET_strdup, GNUNET_STRINGS_get_short_name(), GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::parent, and GNUNET_FS_ShareTreeItem::short_filename.

Referenced by process_helper_msgs().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ finish_scan()

static void finish_scan ( void *  cls)
static

Task run last to shut everything down.

Parameters
clsthe 'struct GNUNET_FS_DirScanner'

Definition at line 222 of file fs_dirmetascan.c.

223 {
224  struct GNUNET_FS_DirScanner *ds = cls;
225 
226  ds->stop_task = NULL;
227  if (NULL != ds->helper)
228  {
230  ds->helper = NULL;
231  }
233  NULL,
236 }
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
@ GNUNET_FS_DIRSCANNER_FINISHED
Last call to the progress function: we have finished scanning the directory.
void GNUNET_HELPER_stop(struct GNUNET_HELPER_Handle *h, int soft_kill)
Kills the helper, closes the pipe, frees the handle and calls wait() on the helper process.
Definition: helper.c:567
An opaque structure a pointer to which is returned to the caller to be used to control the scanner.
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there's a progress message.
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
void * progress_callback_cls
A closure for progress_callback.
struct GNUNET_HELPER_Handle * helper
Helper process.

References ds, GNUNET_FS_DIRSCANNER_FINISHED, GNUNET_HELPER_stop(), GNUNET_NO, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by process_helper_msgs().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ process_helper_msgs()

static int process_helper_msgs ( void *  cls,
const struct GNUNET_MessageHeader msg 
)
static

Called every time there is data to read from the scanner.

Calls the scanner progress handler.

Parameters
clsthe closure (directory scanner object)
msgmessage from the helper process
Returns
GNUNET_OK on success, GNUNET_NO to stop further processing (no error) GNUNET_SYSERR to stop further processing with error

Definition at line 250 of file fs_dirmetascan.c.

251 {
252  struct GNUNET_FS_DirScanner *ds = cls;
253  const char *filename;
254  size_t left;
255 
256 #if 0
257  fprintf (stderr,
258  "DMS parses %u-byte message of type %u\n",
259  (unsigned int) ntohs (msg->size),
260  (unsigned int) ntohs (msg->type));
261 #endif
262  left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
263  filename = (const char *) &msg[1];
264  switch (ntohs (msg->type))
265  {
267  if (filename[left - 1] != '\0')
268  {
269  GNUNET_break (0);
270  break;
271  }
273  filename,
274  GNUNET_NO,
276  if (NULL == ds->toplevel)
277  {
279  }
280  else
281  {
282  GNUNET_assert (NULL != ds->pos);
283  (void) expand_tree (ds->pos, filename, GNUNET_NO);
284  }
285  return GNUNET_OK;
286 
288  if (filename[left - 1] != '\0')
289  {
290  GNUNET_break (0);
291  break;
292  }
293  if (0 == strcmp ("..", filename))
294  {
295  if (NULL == ds->pos)
296  {
297  GNUNET_break (0);
298  break;
299  }
300  ds->pos = ds->pos->parent;
301  return GNUNET_OK;
302  }
304  filename,
305  GNUNET_YES,
308  if (NULL == ds->toplevel)
309  ds->toplevel = ds->pos;
310  return GNUNET_OK;
311 
313  break;
314 
316  if ('\0' != filename[left - 1])
317  break;
319  filename,
322  return GNUNET_OK;
323 
325  if (0 != left)
326  {
327  GNUNET_break (0);
328  break;
329  }
330  if (NULL == ds->toplevel)
331  break;
333  NULL,
336  ds->pos = ds->toplevel;
337  if (GNUNET_YES == ds->pos->is_directory)
338  ds->pos = advance (ds->pos);
339  return GNUNET_OK;
340 
342  size_t nlen;
343  const char *end;
344 
345  if (NULL == ds->pos)
346  {
347  GNUNET_break (0);
348  break;
349  }
350  end = memchr (filename, 0, left);
351  if (NULL == end)
352  {
353  GNUNET_break (0);
354  break;
355  }
356  end++;
357  nlen = end - filename;
358  left -= nlen;
359  if (0 != strcmp (filename, ds->pos->filename))
360  {
361  GNUNET_break (0);
362  break;
363  }
365  filename,
366  GNUNET_YES,
368  if (0 < left)
369  {
371  if (NULL == ds->pos->meta)
372  {
373  GNUNET_break (0);
374  break;
375  }
376  /* having full filenames is too dangerous; always make sure we clean them up */
379  NULL,
380  0);
381  /* instead, put in our 'safer' original filename */
383  "<libgnunetfs>",
386  "text/plain",
388  strlen (ds->pos->short_filename)
389  + 1);
390  }
392  ds->pos->meta);
393  ds->pos = advance (ds->pos);
394  return GNUNET_OK;
395  }
396 
398  if (NULL != ds->pos)
399  {
400  GNUNET_break (0);
401  break;
402  }
403  if (0 != left)
404  {
405  GNUNET_break (0);
406  break;
407  }
408  if (NULL == ds->toplevel)
409  break;
411  return GNUNET_OK;
412 
413  default:
414  GNUNET_break (0);
415  break;
416  }
418  NULL,
421  return GNUNET_OK;
422 }
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
static struct GNUNET_FS_ShareTreeItem * advance(struct GNUNET_FS_ShareTreeItem *pos)
Move in the directory from the given position to the next file in DFS traversal.
static struct GNUNET_FS_ShareTreeItem * expand_tree(struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
Add another child node to the tree.
static void finish_scan(void *cls)
Task run last to shut everything down.
static int end
Set if we are to shutdown all services (including ARM).
Definition: gnunet-arm.c:34
@ GNUNET_OK
Definition: gnunet_common.h:95
@ EXTRACTOR_METAFORMAT_UTF8
0-terminated, UTF-8 encoded string.
@ EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
@ EXTRACTOR_METATYPE_FILENAME
struct GNUNET_FS_Uri * GNUNET_FS_uri_ksk_create_from_meta_data(const struct GNUNET_CONTAINER_MetaData *md)
Construct a keyword-URI from meta-data (take all entries in the meta-data and construct one large key...
Definition: fs_uri.c:1781
@ GNUNET_FS_DIRSCANNER_INTERNAL_ERROR
There was an internal error.
@ GNUNET_FS_DIRSCANNER_FILE_IGNORED
We're having trouble accessing a file (soft-error); it will be ignored.
@ GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED
We've finished extracting meta data from a file.
@ GNUNET_FS_DIRSCANNER_ALL_COUNTED
We've found all files (in the pre-pass).
@ GNUNET_FS_DIRSCANNER_FILE_START
We've started processing a file or directory.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
int GNUNET_CONTAINER_meta_data_delete(struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type, const char *data, size_t data_size)
Remove an item.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1296
struct GNUNET_FS_ShareTreeItem * toplevel
After the scan is finished, it will contain a pointer to the top-level directory entry in the directo...
struct GNUNET_FS_ShareTreeItem * pos
Current position during processing.
struct GNUNET_FS_Uri * ksk_uri
Keywords for this file or directory (derived from metadata).
struct GNUNET_CONTAINER_MetaData * meta
Metadata for this file or directory.
Header for all communications.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format.

References advance(), ds, end, expand_tree(), EXTRACTOR_METAFORMAT_UTF8, EXTRACTOR_METATYPE_FILENAME, EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, filename, GNUNET_FS_ShareTreeItem::filename, finish_scan(), GNUNET_assert, GNUNET_break, GNUNET_CONTAINER_meta_data_delete(), GNUNET_CONTAINER_meta_data_deserialize(), GNUNET_CONTAINER_meta_data_insert(), GNUNET_FS_DIRSCANNER_ALL_COUNTED, GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED, GNUNET_FS_DIRSCANNER_FILE_IGNORED, GNUNET_FS_DIRSCANNER_FILE_START, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_FS_uri_ksk_create_from_meta_data(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_NO, GNUNET_OK, GNUNET_SCHEDULER_add_now(), GNUNET_SYSERR, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::ksk_uri, GNUNET_FS_ShareTreeItem::meta, msg, GNUNET_FS_ShareTreeItem::parent, GNUNET_FS_DirScanner::pos, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, GNUNET_FS_ShareTreeItem::short_filename, GNUNET_MessageHeader::size, GNUNET_FS_DirScanner::stop_task, GNUNET_FS_DirScanner::toplevel, and GNUNET_MessageHeader::type.

Referenced by GNUNET_FS_directory_scan_start().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ helper_died_cb()

static void helper_died_cb ( void *  cls)
static

Function called if our helper process died.

Parameters
clsthe 'struct GNUNET_FS_DirScanner' callback.

Definition at line 431 of file fs_dirmetascan.c.

432 {
433  struct GNUNET_FS_DirScanner *ds = cls;
434 
435  ds->helper = NULL;
436  if (NULL != ds->stop_task)
437  return; /* normal death, was finished */
439  NULL,
442 }

References ds, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by GNUNET_FS_directory_scan_start().

Here is the caller graph for this function: