GNUnet  0.11.x
Data Structures | Functions | Variables
gnunet-helper-fs-publish.c File Reference

Tool to help extract meta data asynchronously. More...

#include "platform.h"
#include "gnunet_fs_service.h"
Include dependency graph for gnunet-helper-fs-publish.c:

Go to the source code of this file.

Data Structures

struct  ScanTreeNode
 A node of a directory tree. More...
 
struct  RecursionContext
 Closure for the 'scan_callback'. More...
 

Functions

static void free_tree (struct ScanTreeNode *tree)
 Free memory of the tree structure. More...
 
static int write_all (const void *buf, size_t size)
 Write size bytes from buf into the output_stream. More...
 
static int write_message (uint16_t message_type, const char *data, size_t data_length)
 Write message to the master process. More...
 
static int preprocess_file (const char *filename, struct ScanTreeNode **dst)
 Function called to (recursively) add all of the files in the directory to the tree. More...
 
static int scan_callback (void *cls, const char *filename)
 Function called by the directory iterator to (recursively) add all of the files in the directory to the tree. More...
 
static int extract_files (struct ScanTreeNode *item)
 Extract metadata from files. More...
 
static void ignore_sigpipe ()
 Install a signal handler to ignore SIGPIPE. More...
 
static void make_dev_zero (int fd, int flags)
 Turn the given file descriptor in to '/dev/null'. More...
 
int main (int argc, char *const *argv)
 Main function of the helper process to extract meta data. More...
 

Variables

static int output_stream
 File descriptor we use for IPC with the parent. More...
 

Detailed Description

Tool to help extract meta data asynchronously.

Author
Christian Grothoff

This program will scan a directory for files with meta data and report the results to stdout.

Definition in file gnunet-helper-fs-publish.c.

Function Documentation

◆ free_tree()

static void free_tree ( struct ScanTreeNode tree)
static

Free memory of the tree structure.

Parameters
treetree to free

Definition at line 163 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, ScanTreeNode::filename, GNUNET_CONTAINER_DLL_remove, GNUNET_free, and ScanTreeNode::parent.

Referenced by main(), and preprocess_file().

164 {
165  struct ScanTreeNode *pos;
166 
167  while (NULL != (pos = tree->children_head))
168  free_tree (pos);
169  if (NULL != tree->parent)
171  tree->parent->children_tail,
172  tree);
173  GNUNET_free (tree->filename);
174  GNUNET_free (tree);
175 }
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
A node of a directory tree.
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ write_all()

static int write_all ( const void *  buf,
size_t  size 
)
static

Write size bytes from buf into the output_stream.

Parameters
bufbuffer with data to write
sizenumber of bytes to write
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 186 of file gnunet-helper-fs-publish.c.

References buf, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_OK, GNUNET_SYSERR, and output_stream.

Referenced by write_message().

187 {
188  const char *cbuf = buf;
189  size_t total;
190  ssize_t wr;
191 
192  total = 0;
193  do
194  {
195  wr = write (output_stream, &cbuf[total], size - total);
196  if (wr > 0)
197  total += wr;
198  }
199  while ((wr > 0) && (total < size));
200  if (wr <= 0)
202  "Failed to write to stdout: %s\n",
203  strerror (errno));
204  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
205 }
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
static char buf[2048]
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:67
static int output_stream
File descriptor we use for IPC with the parent.
#define GNUNET_log(kind,...)
Here is the caller graph for this function:

◆ write_message()

static int write_message ( uint16_t  message_type,
const char *  data,
size_t  data_length 
)
static

Write message to the master process.

Parameters
message_typemessage type to use
datadata to append, NULL for none
data_lengthnumber of bytes in data
Returns
GNUNET_SYSERR to stop scanning (the pipe was broken somehow)

Definition at line 217 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::filename, GNUNET_OK, GNUNET_SYSERR, preprocess_file(), GNUNET_MessageHeader::size, GNUNET_MessageHeader::type, and write_all().

Referenced by extract_files(), main(), and preprocess_file().

218 {
219  struct GNUNET_MessageHeader hdr;
220 
221 #if 0
222  fprintf (stderr,
223  "Helper sends %u-byte message of type %u\n",
224  (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
225  (unsigned int) message_type);
226 #endif
227  hdr.type = htons (message_type);
228  hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
229  if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
230  (GNUNET_OK != write_all (data, data_length)))
231  return GNUNET_SYSERR;
232  return GNUNET_OK;
233 }
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
Header for all communications.
uint32_t data
The data value.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ preprocess_file()

static int preprocess_file ( const char *  filename,
struct ScanTreeNode **  dst 
)
static

Function called to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
filenamefile or directory to scan
dstwhere to store the resulting share tree item; NULL is stored in dst upon recoverable errors (GNUNET_OK is returned)
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 308 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::file_size, ScanTreeNode::filename, free_tree(), GNUNET_DISK_directory_scan(), GNUNET_DISK_file_size(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_new, GNUNET_NO, GNUNET_OK, GNUNET_strdup, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, RecursionContext::parent, scan_callback(), RecursionContext::stop, and write_message().

Referenced by main(), scan_callback(), and write_message().

309 {
310  struct ScanTreeNode *item;
311  struct stat sbuf;
312  uint64_t fsize = 0;
313 
314  if ((0 != stat (filename, &sbuf)) ||
315  ((! S_ISDIR (sbuf.st_mode)) &&
316  (GNUNET_OK !=
318  {
319  /* If the file doesn't exist (or is not stat-able for any other reason)
320  skip it (but report it), but do continue. */
321  if (GNUNET_OK !=
323  filename,
324  strlen (filename) + 1))
325  return GNUNET_SYSERR;
326  /* recoverable error, store 'NULL' in *dst */
327  *dst = NULL;
328  return GNUNET_OK;
329  }
330 
331  /* Report the progress */
332  if (
333  GNUNET_OK !=
334  write_message (S_ISDIR (sbuf.st_mode)
337  filename,
338  strlen (filename) + 1))
339  return GNUNET_SYSERR;
340  item = GNUNET_new (struct ScanTreeNode);
341  item->filename = GNUNET_strdup (filename);
342  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
343  item->file_size = fsize;
344  if (GNUNET_YES == item->is_directory)
345  {
346  struct RecursionContext rc;
347 
348  rc.parent = item;
349  rc.stop = GNUNET_NO;
351  if (
352  (GNUNET_YES == rc.stop) ||
353  (GNUNET_OK !=
355  "..",
356  3)))
357  {
358  free_tree (item);
359  return GNUNET_SYSERR;
360  }
361  }
362  *dst = item;
363  return GNUNET_OK;
364 }
struct ScanTreeNode * parent
Parent to add the files to.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_new(type)
Allocate a struct or union of the given type.
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:912
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
uint64_t file_size
Size of the file (if it is a file), in bytes.
static char * filename
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
int GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:257
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
char * filename
Name of the file/directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
A node of a directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_YES
Definition: gnunet_common.h:77
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
int is_directory
GNUNET_YES if this is a directory
Closure for the &#39;scan_callback&#39;.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_callback()

static int scan_callback ( void *  cls,
const char *  filename 
)
static

Function called by the directory iterator to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
clsthe struct RecursionContext
filenamefile or directory to scan
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 277 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, GNUNET_CONTAINER_DLL_insert, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::parent, RecursionContext::parent, preprocess_file(), and RecursionContext::stop.

Referenced by preprocess_file().

278 {
279  struct RecursionContext *rc = cls;
280  struct ScanTreeNode *chld;
281 
282  if (GNUNET_OK != preprocess_file (filename, &chld))
283  {
284  rc->stop = GNUNET_YES;
285  return GNUNET_SYSERR;
286  }
287  if (NULL == chld)
288  return GNUNET_OK;
289  chld->parent = rc->parent;
291  rc->parent->children_tail,
292  chld);
293  return GNUNET_OK;
294 }
struct ScanTreeNode * parent
Parent to add the files to.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char * filename
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
int stop
Flag to set to GNUNET_YES on serious errors.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
A node of a directory tree.
#define GNUNET_YES
Definition: gnunet_common.h:77
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
Closure for the &#39;scan_callback&#39;.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ extract_files()

static int extract_files ( struct ScanTreeNode item)
static

Extract metadata from files.

Parameters
itementry we are processing
Returns
GNUNET_OK on success, GNUNET_SYSERR on fatal errors

Definition at line 374 of file gnunet-helper-fs-publish.c.

References buf, ScanTreeNode::children_head, ScanTreeNode::filename, GNUNET_break, GNUNET_CONTAINER_meta_data_create(), GNUNET_CONTAINER_meta_data_destroy(), GNUNET_CONTAINER_meta_data_get_serialized_size(), GNUNET_CONTAINER_meta_data_serialize(), GNUNET_CONTAINER_META_DATA_SERIALIZE_PART, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, meta, ScanTreeNode::next, plugins, size, and write_message().

Referenced by main().

375 {
377  ssize_t size;
378  size_t slen;
379 
380  if (GNUNET_YES == item->is_directory)
381  {
382  /* for directories, we simply only descent, no extraction, no
383  progress reporting */
384  struct ScanTreeNode *pos;
385 
386  for (pos = item->children_head; NULL != pos; pos = pos->next)
387  if (GNUNET_OK != extract_files (pos))
388  return GNUNET_SYSERR;
389  return GNUNET_OK;
390  }
391 
392  /* this is the expensive operation, *afterwards* we'll check for aborts */
394 #if HAVE_LIBEXTRACTOR
395  EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
396 #endif
397  slen = strlen (item->filename) + 1;
399  if (-1 == size)
400  {
401  /* no meta data */
403  if (GNUNET_OK !=
405  item->filename,
406  slen))
407  return GNUNET_SYSERR;
408  return GNUNET_OK;
409  }
410  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
411  {
412  /* We can't transfer more than 64k bytes in one message. */
413  size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
414  }
415  {
416  char buf[size + slen];
417  char *dst = &buf[slen];
418 
419  GNUNET_memcpy (buf, item->filename, slen);
421  meta,
422  &dst,
423  size,
425  if (size < 0)
426  {
427  GNUNET_break (0);
428  size = 0;
429  }
431  if (GNUNET_OK !=
433  buf,
434  slen + size))
435  return GNUNET_SYSERR;
436  }
437  return GNUNET_OK;
438 }
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
Meta data to associate with a file, directory or namespace.
struct ScanTreeNode * next
This is a doubly-linked list.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char buf[2048]
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
char * filename
Name of the file/directory.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:68
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
A node of a directory tree.
Header for all communications.
#define GNUNET_YES
Definition: gnunet_common.h:77
int is_directory
GNUNET_YES if this is a directory
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ignore_sigpipe()

static void ignore_sigpipe ( )
static

Install a signal handler to ignore SIGPIPE.

Definition at line 445 of file gnunet-helper-fs-publish.c.

Referenced by main().

446 {
447  struct sigaction oldsig;
448  struct sigaction sig;
449 
450  memset (&sig, 0, sizeof(struct sigaction));
451  sig.sa_handler = SIG_IGN;
452  sigemptyset (&sig.sa_mask);
453 #ifdef SA_INTERRUPT
454  sig.sa_flags = SA_INTERRUPT; /* SunOS */
455 #else
456  sig.sa_flags = SA_RESTART;
457 #endif
458  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
459  fprintf (stderr,
460  "Failed to install SIGPIPE handler: %s\n",
461  strerror (errno));
462 }
Here is the caller graph for this function:

◆ make_dev_zero()

static void make_dev_zero ( int  fd,
int  flags 
)
static

Turn the given file descriptor in to '/dev/null'.

Parameters
fdfd to bind to /dev/null
flagsflags to use (O_RDONLY or O_WRONLY)

Definition at line 472 of file gnunet-helper-fs-publish.c.

References GNUNET_assert, and GNUNET_break.

Referenced by main().

473 {
474  int z;
475 
476  GNUNET_assert (0 == close (fd));
477  z = open ("/dev/null", flags);
478  GNUNET_assert (-1 != z);
479  if (z == fd)
480  return;
481  GNUNET_break (fd == dup2 (z, fd));
482  GNUNET_assert (0 == close (z));
483 }
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
Here is the caller graph for this function:

◆ main()

int main ( int  argc,
char *const *  argv 
)

Main function of the helper process to extract meta data.

Parameters
argcshould be 3
argv[0] our binary name [1] name of the file or directory to process [2] "-" to disable extraction, NULL for defaults, otherwise custom plugins to load from LE
Returns
0 on success

Definition at line 497 of file gnunet-helper-fs-publish.c.

References extract_files(), free_tree(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_OK, ignore_sigpipe(), make_dev_zero(), output_stream, plugins, preprocess_file(), and write_message().

498 {
499  const char *filename_expanded;
500  const char *ex;
501  struct ScanTreeNode *root;
502 
503  ignore_sigpipe ();
504  /* move stdout to some other FD for IPC, bind
505  stdout/stderr to /dev/null */
506  output_stream = dup (1);
507  make_dev_zero (1, O_WRONLY);
508  make_dev_zero (2, O_WRONLY);
509 
510  /* parse command line */
511  if ((3 != argc) && (2 != argc))
512  {
513  fprintf (stderr,
514  "%s",
515  "gnunet-helper-fs-publish needs exactly one or two arguments\n");
516  return 1;
517  }
518  filename_expanded = argv[1];
519  ex = argv[2];
520  if ((NULL == ex) || (0 != strcmp (ex, "-")))
521  {
522 #if HAVE_LIBEXTRACTOR
523  plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
524  if (NULL != ex)
525  plugins = EXTRACTOR_plugin_add_config (plugins,
526  ex,
527  EXTRACTOR_OPTION_DEFAULT_POLICY);
528 #endif
529  }
530 
531  /* scan tree to find out how much work there is to be done */
532  if (GNUNET_OK != preprocess_file (filename_expanded, &root))
533  {
535 #if HAVE_LIBEXTRACTOR
536  EXTRACTOR_plugin_remove_all (plugins);
537 #endif
538  return 2;
539  }
540  /* signal that we're done counting files, so that a percentage of
541  progress can now be calculated */
542  if (GNUNET_OK !=
544  NULL,
545  0))
546  {
547 #if HAVE_LIBEXTRACTOR
548  EXTRACTOR_plugin_remove_all (plugins);
549 #endif
550  return 3;
551  }
552  if (NULL != root)
553  {
554  if (GNUNET_OK != extract_files (root))
555  {
557  NULL,
558  0);
559  free_tree (root);
560 #if HAVE_LIBEXTRACTOR
561  EXTRACTOR_plugin_remove_all (plugins);
562 #endif
563  return 4;
564  }
565  free_tree (root);
566  }
567  /* enable "clean" shutdown by telling parent that we are done */
569  NULL,
570  0);
571 #if HAVE_LIBEXTRACTOR
572  EXTRACTOR_plugin_remove_all (plugins);
573 #endif
574  return 0;
575 }
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to &#39;/dev/null&#39;.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
static int output_stream
File descriptor we use for IPC with the parent.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:68
A node of a directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
Here is the call graph for this function:

Variable Documentation

◆ output_stream

int output_stream
static

File descriptor we use for IPC with the parent.

Definition at line 93 of file gnunet-helper-fs-publish.c.

Referenced by main(), and write_all().