GNUnet  0.20.0
fs_dirmetascan.c File Reference

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'. More...

#include "platform.h"
#include "gnunet_fs_service.h"
#include "gnunet_scheduler_lib.h"
#include <pthread.h>
Include dependency graph for fs_dirmetascan.c:

Go to the source code of this file.

Data Structures

struct  GNUNET_FS_DirScanner
 An opaque structure a pointer to which is returned to the caller to be used to control the scanner. More...
 

Functions

void GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
 Abort the scan. More...
 
struct GNUNET_FS_ShareTreeItemGNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
 Obtain the result of the scan after the scan has signalled completion. More...
 
static struct GNUNET_FS_ShareTreeItemadvance (struct GNUNET_FS_ShareTreeItem *pos)
 Move in the directory from the given position to the next file in DFS traversal. More...
 
static struct GNUNET_FS_ShareTreeItemexpand_tree (struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
 Add another child node to the tree. More...
 
static void finish_scan (void *cls)
 Task run last to shut everything down. More...
 
static int process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
 Called every time there is data to read from the scanner. More...
 
static void helper_died_cb (void *cls)
 Function called if our helper process died. More...
 
struct GNUNET_FS_DirScannerGNUNET_FS_directory_scan_start (const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
 Start a directory scanner thread. More...
 

Detailed Description

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.

Author
LRN
Christian Grothoff

Definition in file fs_dirmetascan.c.

Function Documentation

◆ advance()

static struct GNUNET_FS_ShareTreeItem* advance ( struct GNUNET_FS_ShareTreeItem pos)
static

Move in the directory from the given position to the next file in DFS traversal.

Parameters
poscurrent position
Returns
next file, NULL for none

Definition at line 139 of file fs_dirmetascan.c.

140 {
141  int moved;
142 
143  GNUNET_assert (NULL != pos);
144  moved = 0; /* must not terminate, even on file, otherwise "normal" */
145  while ((pos->is_directory == GNUNET_YES) || (0 == moved))
146  {
147  if ((moved != -1) && (NULL != pos->children_head))
148  {
149  pos = pos->children_head;
150  moved = 1; /* can terminate if file */
151  continue;
152  }
153  if (NULL != pos->next)
154  {
155  pos = pos->next;
156  moved = 1; /* can terminate if file */
157  continue;
158  }
159  if (NULL != pos->parent)
160  {
161  pos = pos->parent;
162  moved = -1; /* force move to 'next' or 'parent' */
163  continue;
164  }
165  /* no more options, end of traversal */
166  return NULL;
167  }
168  return pos;
169 }
@ GNUNET_YES
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int is_directory
GNUNET_YES if this is a directory
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_ShareTreeItem * next
This is a doubly-linked list.
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_assert, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::next, and GNUNET_FS_ShareTreeItem::parent.

Referenced by process_helper_msgs().

Here is the caller graph for this function:

◆ expand_tree()

static struct GNUNET_FS_ShareTreeItem* expand_tree ( struct GNUNET_FS_ShareTreeItem parent,
const char *  filename,
int  is_directory 
)
static

Add another child node to the tree.

Parameters
parentparent of the child, NULL for top level
filenamename of the file or directory
is_directoryGNUNET_YES for directories
Returns
new entry that was just created

Definition at line 181 of file fs_dirmetascan.c.

184 {
185  struct GNUNET_FS_ShareTreeItem *chld;
186  size_t slen;
187 
188  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
189  chld->parent = parent;
190  chld->filename = GNUNET_strdup (filename);
192  "%s%s",
194  is_directory == GNUNET_YES ? "/" : "");
195  /* make sure we do not end with '//' */
196  slen = strlen (chld->short_filename);
197  if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
198  (chld->short_filename[slen - 2] == '/'))
199  chld->short_filename[slen - 1] = '\0';
200  chld->is_directory = is_directory;
201  if (NULL != parent)
204  chld);
205  return chld;
206 }
static char * filename
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
const char * GNUNET_STRINGS_get_short_name(const char *filename)
"man basename" Returns a pointer to a part of filename (allocates nothing)!
Definition: strings.c:639
A node of a directory tree (produced by dirscanner)
char * short_filename
Base name of the file/directory.
char * filename
Name of the file/directory.
struct GNUNET_FS_ShareTreeItem * children_tail
This is a doubly-linked tree NULL for files and empty directories.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_FS_ShareTreeItem::children_tail, filename, GNUNET_FS_ShareTreeItem::filename, GNUNET_asprintf(), GNUNET_CONTAINER_DLL_insert, GNUNET_new, GNUNET_strdup, GNUNET_STRINGS_get_short_name(), GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::parent, and GNUNET_FS_ShareTreeItem::short_filename.

Referenced by process_helper_msgs().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ finish_scan()

static void finish_scan ( void *  cls)
static

Task run last to shut everything down.

Parameters
clsthe 'struct GNUNET_FS_DirScanner'

Definition at line 215 of file fs_dirmetascan.c.

216 {
217  struct GNUNET_FS_DirScanner *ds = cls;
218 
219  ds->stop_task = NULL;
220  if (NULL != ds->helper)
221  {
223  ds->helper = NULL;
224  }
226  NULL,
229 }
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
@ GNUNET_FS_DIRSCANNER_FINISHED
Last call to the progress function: we have finished scanning the directory.
void GNUNET_HELPER_stop(struct GNUNET_HELPER_Handle *h, int soft_kill)
Kills the helper, closes the pipe, frees the handle and calls wait() on the helper process.
Definition: helper.c:538
@ GNUNET_NO
@ GNUNET_SYSERR
An opaque structure a pointer to which is returned to the caller to be used to control the scanner.
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there's a progress message.
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
void * progress_callback_cls
A closure for progress_callback.
struct GNUNET_HELPER_Handle * helper
Helper process.

References ds, GNUNET_FS_DIRSCANNER_FINISHED, GNUNET_HELPER_stop(), GNUNET_NO, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by process_helper_msgs().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ process_helper_msgs()

static int process_helper_msgs ( void *  cls,
const struct GNUNET_MessageHeader msg 
)
static

Called every time there is data to read from the scanner.

Calls the scanner progress handler.

Parameters
clsthe closure (directory scanner object)
msgmessage from the helper process
Returns
GNUNET_OK on success, GNUNET_NO to stop further processing (no error) GNUNET_SYSERR to stop further processing with error

Definition at line 243 of file fs_dirmetascan.c.

244 {
245  struct GNUNET_FS_DirScanner *ds = cls;
246  const char *filename;
247  size_t left;
248 
249 #if 0
250  fprintf (stderr,
251  "DMS parses %u-byte message of type %u\n",
252  (unsigned int) ntohs (msg->size),
253  (unsigned int) ntohs (msg->type));
254 #endif
255  left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
256  filename = (const char *) &msg[1];
257  switch (ntohs (msg->type))
258  {
260  if (filename[left - 1] != '\0')
261  {
262  GNUNET_break (0);
263  break;
264  }
266  filename,
267  GNUNET_NO,
269  if (NULL == ds->toplevel)
270  {
272  }
273  else
274  {
275  GNUNET_assert (NULL != ds->pos);
276  (void) expand_tree (ds->pos, filename, GNUNET_NO);
277  }
278  return GNUNET_OK;
279 
281  if (filename[left - 1] != '\0')
282  {
283  GNUNET_break (0);
284  break;
285  }
286  if (0 == strcmp ("..", filename))
287  {
288  if (NULL == ds->pos)
289  {
290  GNUNET_break (0);
291  break;
292  }
293  ds->pos = ds->pos->parent;
294  return GNUNET_OK;
295  }
297  filename,
298  GNUNET_YES,
301  if (NULL == ds->toplevel)
302  ds->toplevel = ds->pos;
303  return GNUNET_OK;
304 
306  break;
307 
309  if ('\0' != filename[left - 1])
310  break;
312  filename,
315  return GNUNET_OK;
316 
318  if (0 != left)
319  {
320  GNUNET_break (0);
321  break;
322  }
323  if (NULL == ds->toplevel)
324  break;
326  NULL,
329  ds->pos = ds->toplevel;
330  if (GNUNET_YES == ds->pos->is_directory)
331  ds->pos = advance (ds->pos);
332  return GNUNET_OK;
333 
335  size_t nlen;
336  const char *end;
337 
338  if (NULL == ds->pos)
339  {
340  GNUNET_break (0);
341  break;
342  }
343  end = memchr (filename, 0, left);
344  if (NULL == end)
345  {
346  GNUNET_break (0);
347  break;
348  }
349  end++;
350  nlen = end - filename;
351  left -= nlen;
352  if (0 != strcmp (filename, ds->pos->filename))
353  {
354  GNUNET_break (0);
355  break;
356  }
358  filename,
359  GNUNET_YES,
361  if (0 < left)
362  {
364  if (NULL == ds->pos->meta)
365  {
366  GNUNET_break (0);
367  break;
368  }
369  /* having full filenames is too dangerous; always make sure we clean them up */
371  EXTRACTOR_METATYPE_FILENAME,
372  NULL,
373  0);
374  /* instead, put in our 'safer' original filename */
376  "<libgnunetfs>",
378  EXTRACTOR_METAFORMAT_UTF8,
379  "text/plain",
381  strlen (ds->pos->short_filename)
382  + 1);
383  }
385  ds->pos->meta);
386  ds->pos = advance (ds->pos);
387  return GNUNET_OK;
388  }
389 
391  if (NULL != ds->pos)
392  {
393  GNUNET_break (0);
394  break;
395  }
396  if (0 != left)
397  {
398  GNUNET_break (0);
399  break;
400  }
401  if (NULL == ds->toplevel)
402  break;
404  return GNUNET_OK;
405 
406  default:
407  GNUNET_break (0);
408  break;
409  }
411  NULL,
414  return GNUNET_OK;
415 }
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
static struct GNUNET_FS_ShareTreeItem * advance(struct GNUNET_FS_ShareTreeItem *pos)
Move in the directory from the given position to the next file in DFS traversal.
static struct GNUNET_FS_ShareTreeItem * expand_tree(struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
Add another child node to the tree.
static void finish_scan(void *cls)
Task run last to shut everything down.
static int end
Set if we are to shutdown all services (including ARM).
Definition: gnunet-arm.c:34
#define EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
struct GNUNET_FS_Uri * GNUNET_FS_uri_ksk_create_from_meta_data(const struct GNUNET_FS_MetaData *md)
Construct a keyword-URI from meta-data (take all entries in the meta-data and construct one large key...
Definition: fs_uri.c:1771
@ GNUNET_FS_DIRSCANNER_INTERNAL_ERROR
There was an internal error.
@ GNUNET_FS_DIRSCANNER_FILE_IGNORED
We're having trouble accessing a file (soft-error); it will be ignored.
@ GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED
We've finished extracting meta data from a file.
@ GNUNET_FS_DIRSCANNER_ALL_COUNTED
We've found all files (in the pre-pass).
@ GNUNET_FS_DIRSCANNER_FILE_START
We've started processing a file or directory.
@ GNUNET_OK
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
int GNUNET_FS_meta_data_insert(struct GNUNET_FS_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
Definition: meta_data.c:259
int GNUNET_FS_meta_data_delete(struct GNUNET_FS_MetaData *md, enum EXTRACTOR_MetaType type, const char *data, size_t data_size)
Remove an item.
Definition: meta_data.c:369
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
Definition: meta_data.c:890
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1299
struct GNUNET_FS_ShareTreeItem * toplevel
After the scan is finished, it will contain a pointer to the top-level directory entry in the directo...
struct GNUNET_FS_ShareTreeItem * pos
Current position during processing.
struct GNUNET_FS_Uri * ksk_uri
Keywords for this file or directory (derived from metadata).
struct GNUNET_FS_MetaData * meta
Metadata for this file or directory.
Header for all communications.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format.

References advance(), ds, end, expand_tree(), EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, filename, GNUNET_FS_ShareTreeItem::filename, finish_scan(), GNUNET_assert, GNUNET_break, GNUNET_FS_DIRSCANNER_ALL_COUNTED, GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED, GNUNET_FS_DIRSCANNER_FILE_IGNORED, GNUNET_FS_DIRSCANNER_FILE_START, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_FS_meta_data_delete(), GNUNET_FS_meta_data_deserialize(), GNUNET_FS_meta_data_insert(), GNUNET_FS_uri_ksk_create_from_meta_data(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_NO, GNUNET_OK, GNUNET_SCHEDULER_add_now(), GNUNET_SYSERR, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::ksk_uri, GNUNET_FS_ShareTreeItem::meta, msg, GNUNET_FS_ShareTreeItem::parent, GNUNET_FS_DirScanner::pos, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, GNUNET_FS_ShareTreeItem::short_filename, GNUNET_MessageHeader::size, GNUNET_FS_DirScanner::stop_task, GNUNET_FS_DirScanner::toplevel, and GNUNET_MessageHeader::type.

Referenced by GNUNET_FS_directory_scan_start().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ helper_died_cb()

static void helper_died_cb ( void *  cls)
static

Function called if our helper process died.

Parameters
clsthe 'struct GNUNET_FS_DirScanner' callback.

Definition at line 424 of file fs_dirmetascan.c.

425 {
426  struct GNUNET_FS_DirScanner *ds = cls;
427 
428  ds->helper = NULL;
429  if (NULL != ds->stop_task)
430  return; /* normal death, was finished */
432  NULL,
435 }

References ds, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by GNUNET_FS_directory_scan_start().

Here is the caller graph for this function: