GNUnet  0.10.x
Data Structures | Functions | Variables
gnunet-helper-fs-publish.c File Reference

Tool to help extract meta data asynchronously. More...

#include "platform.h"
#include "gnunet_fs_service.h"
Include dependency graph for gnunet-helper-fs-publish.c:

Go to the source code of this file.

Data Structures

struct  ScanTreeNode
 A node of a directory tree. More...
 
struct  RecursionContext
 Closure for the 'scan_callback'. More...
 

Functions

static void free_tree (struct ScanTreeNode *tree)
 Free memory of the tree structure. More...
 
static int write_all (const void *buf, size_t size)
 Write size bytes from buf into the output_stream. More...
 
static int write_message (uint16_t message_type, const char *data, size_t data_length)
 Write message to the master process. More...
 
static int preprocess_file (const char *filename, struct ScanTreeNode **dst)
 Function called to (recursively) add all of the files in the directory to the tree. More...
 
static int scan_callback (void *cls, const char *filename)
 Function called by the directory iterator to (recursively) add all of the files in the directory to the tree. More...
 
static int extract_files (struct ScanTreeNode *item)
 Extract metadata from files. More...
 
static void ignore_sigpipe ()
 Install a signal handler to ignore SIGPIPE. More...
 
static void make_dev_zero (int fd, int flags)
 Turn the given file descriptor in to '/dev/null'. More...
 
int main (int argc, char *const *argv)
 Main function of the helper process to extract meta data. More...
 

Variables

static int output_stream
 File descriptor we use for IPC with the parent. More...
 

Detailed Description

Tool to help extract meta data asynchronously.

Author
Christian Grothoff

This program will scan a directory for files with meta data and report the results to stdout.

Definition in file gnunet-helper-fs-publish.c.

Function Documentation

◆ free_tree()

static void free_tree ( struct ScanTreeNode tree)
static

Free memory of the tree structure.

Parameters
treetree to free

Definition at line 160 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, ScanTreeNode::filename, GNUNET_CONTAINER_DLL_remove, GNUNET_free, and ScanTreeNode::parent.

Referenced by main(), and preprocess_file().

161 {
162  struct ScanTreeNode *pos;
163 
164  while (NULL != (pos = tree->children_head))
165  free_tree(pos);
166  if (NULL != tree->parent)
168  tree->parent->children_tail,
169  tree);
170  GNUNET_free(tree->filename);
171  GNUNET_free(tree);
172 }
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
A node of a directory tree.
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
#define GNUNET_free(ptr)
Wrapper around free.
Here is the caller graph for this function:

◆ write_all()

static int write_all ( const void *  buf,
size_t  size 
)
static

Write size bytes from buf into the output_stream.

Parameters
bufbuffer with data to write
sizenumber of bytes to write
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 183 of file gnunet-helper-fs-publish.c.

References buf, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_OK, GNUNET_SYSERR, and output_stream.

Referenced by write_message().

184 {
185  const char *cbuf = buf;
186  size_t total;
187  ssize_t wr;
188 
189  total = 0;
190  do
191  {
192  wr = write(output_stream, &cbuf[total], size - total);
193  if (wr > 0)
194  total += wr;
195  }
196  while ((wr > 0) && (total < size));
197  if (wr <= 0)
199  "Failed to write to stdout: %s\n",
200  strerror(errno));
201  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
202 }
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
static char buf[2048]
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:66
static int output_stream
File descriptor we use for IPC with the parent.
#define GNUNET_log(kind,...)
Here is the caller graph for this function:

◆ write_message()

static int write_message ( uint16_t  message_type,
const char *  data,
size_t  data_length 
)
static

Write message to the master process.

Parameters
message_typemessage type to use
datadata to append, NULL for none
data_lengthnumber of bytes in data
Returns
GNUNET_SYSERR to stop scanning (the pipe was broken somehow)

Definition at line 214 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::filename, GNUNET_OK, GNUNET_SYSERR, preprocess_file(), GNUNET_MessageHeader::size, GNUNET_MessageHeader::type, and write_all().

Referenced by extract_files(), main(), and preprocess_file().

215 {
216  struct GNUNET_MessageHeader hdr;
217 
218 #if 0
219  fprintf(stderr,
220  "Helper sends %u-byte message of type %u\n",
221  (unsigned int)(sizeof(struct GNUNET_MessageHeader) + data_length),
222  (unsigned int)message_type);
223 #endif
224  hdr.type = htons(message_type);
225  hdr.size = htons(sizeof(struct GNUNET_MessageHeader) + data_length);
226  if ((GNUNET_OK != write_all(&hdr, sizeof(hdr))) ||
227  (GNUNET_OK != write_all(data, data_length)))
228  return GNUNET_SYSERR;
229  return GNUNET_OK;
230 }
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
Header for all communications.
uint32_t data
The data value.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ preprocess_file()

static int preprocess_file ( const char *  filename,
struct ScanTreeNode **  dst 
)
static

Function called to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
filenamefile or directory to scan
dstwhere to store the resulting share tree item; NULL is stored in dst upon recoverable errors (GNUNET_OK is returned)
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 304 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::file_size, ScanTreeNode::filename, free_tree(), GNUNET_DISK_directory_scan(), GNUNET_DISK_file_size(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_new, GNUNET_NO, GNUNET_OK, GNUNET_strdup, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, RecursionContext::parent, scan_callback(), RecursionContext::stop, and write_message().

Referenced by main(), scan_callback(), and write_message().

305 {
306  struct ScanTreeNode *item;
307  struct stat sbuf;
308  uint64_t fsize = 0;
309 
310  if ((0 != stat(filename, &sbuf)) ||
311  ((!S_ISDIR(sbuf.st_mode)) &&
312  (GNUNET_OK !=
314  {
315  /* If the file doesn't exist (or is not stat-able for any other reason)
316  skip it (but report it), but do continue. */
317  if (GNUNET_OK !=
319  filename,
320  strlen(filename) + 1))
321  return GNUNET_SYSERR;
322  /* recoverable error, store 'NULL' in *dst */
323  *dst = NULL;
324  return GNUNET_OK;
325  }
326 
327  /* Report the progress */
328  if (
329  GNUNET_OK !=
330  write_message(S_ISDIR(sbuf.st_mode)
333  filename,
334  strlen(filename) + 1))
335  return GNUNET_SYSERR;
336  item = GNUNET_new(struct ScanTreeNode);
338  item->is_directory = (S_ISDIR(sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
339  item->file_size = fsize;
340  if (GNUNET_YES == item->is_directory)
341  {
342  struct RecursionContext rc;
343 
344  rc.parent = item;
345  rc.stop = GNUNET_NO;
347  if (
348  (GNUNET_YES == rc.stop) ||
349  (GNUNET_OK !=
351  "..",
352  3)))
353  {
354  free_tree(item);
355  return GNUNET_SYSERR;
356  }
357  }
358  *dst = item;
359  return GNUNET_OK;
360 }
struct ScanTreeNode * parent
Parent to add the files to.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_new(type)
Allocate a struct or union of the given type.
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:909
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
uint64_t file_size
Size of the file (if it is a file), in bytes.
static char * filename
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
int GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:254
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
char * filename
Name of the file/directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
A node of a directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_YES
Definition: gnunet_common.h:77
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
int is_directory
GNUNET_YES if this is a directory
Closure for the &#39;scan_callback&#39;.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_callback()

static int scan_callback ( void *  cls,
const char *  filename 
)
static

Function called by the directory iterator to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
clsthe struct RecursionContext
filenamefile or directory to scan
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 273 of file gnunet-helper-fs-publish.c.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, GNUNET_CONTAINER_DLL_insert, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::parent, RecursionContext::parent, preprocess_file(), and RecursionContext::stop.

Referenced by preprocess_file().

274 {
275  struct RecursionContext *rc = cls;
276  struct ScanTreeNode *chld;
277 
278  if (GNUNET_OK != preprocess_file(filename, &chld))
279  {
280  rc->stop = GNUNET_YES;
281  return GNUNET_SYSERR;
282  }
283  if (NULL == chld)
284  return GNUNET_OK;
285  chld->parent = rc->parent;
287  rc->parent->children_tail,
288  chld);
289  return GNUNET_OK;
290 }
struct ScanTreeNode * parent
Parent to add the files to.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char * filename
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
int stop
Flag to set to GNUNET_YES on serious errors.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
A node of a directory tree.
#define GNUNET_YES
Definition: gnunet_common.h:77
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
Closure for the &#39;scan_callback&#39;.
Here is the call graph for this function:
Here is the caller graph for this function:

◆ extract_files()

static int extract_files ( struct ScanTreeNode item)
static

Extract metadata from files.

Parameters
itementry we are processing
Returns
GNUNET_OK on success, GNUNET_SYSERR on fatal errors

Definition at line 370 of file gnunet-helper-fs-publish.c.

References buf, ScanTreeNode::children_head, ScanTreeNode::filename, GNUNET_break, GNUNET_CONTAINER_meta_data_create(), GNUNET_CONTAINER_meta_data_destroy(), GNUNET_CONTAINER_meta_data_get_serialized_size(), GNUNET_CONTAINER_meta_data_serialize(), GNUNET_CONTAINER_META_DATA_SERIALIZE_PART, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, meta, ScanTreeNode::next, plugins, size, and write_message().

Referenced by main().

371 {
373  ssize_t size;
374  size_t slen;
375 
376  if (GNUNET_YES == item->is_directory)
377  {
378  /* for directories, we simply only descent, no extraction, no
379  progress reporting */
380  struct ScanTreeNode *pos;
381 
382  for (pos = item->children_head; NULL != pos; pos = pos->next)
383  if (GNUNET_OK != extract_files(pos))
384  return GNUNET_SYSERR;
385  return GNUNET_OK;
386  }
387 
388  /* this is the expensive operation, *afterwards* we'll check for aborts */
390 #if HAVE_LIBEXTRACTOR
391  EXTRACTOR_extract(plugins, item->filename, NULL, 0, &add_to_md, meta);
392 #endif
393  slen = strlen(item->filename) + 1;
395  if (-1 == size)
396  {
397  /* no meta data */
399  if (GNUNET_OK !=
401  item->filename,
402  slen))
403  return GNUNET_SYSERR;
404  return GNUNET_OK;
405  }
406  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
407  {
408  /* We can't transfer more than 64k bytes in one message. */
409  size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
410  }
411  {
412  char buf[size + slen];
413  char *dst = &buf[slen];
414 
415  GNUNET_memcpy(buf, item->filename, slen);
417  meta,
418  &dst,
419  size,
421  if (size < 0)
422  {
423  GNUNET_break(0);
424  size = 0;
425  }
427  if (GNUNET_OK !=
429  buf,
430  slen + size))
431  return GNUNET_SYSERR;
432  }
433  return GNUNET_OK;
434 }
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
Meta data to associate with a file, directory or namespace.
struct ScanTreeNode * next
This is a doubly-linked list.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char buf[2048]
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:66
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
char * filename
Name of the file/directory.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:67
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
A node of a directory tree.
Header for all communications.
#define GNUNET_YES
Definition: gnunet_common.h:77
int is_directory
GNUNET_YES if this is a directory
Here is the call graph for this function:
Here is the caller graph for this function:

◆ ignore_sigpipe()

static void ignore_sigpipe ( )
static

Install a signal handler to ignore SIGPIPE.

Definition at line 441 of file gnunet-helper-fs-publish.c.

Referenced by main().

442 {
443  struct sigaction oldsig;
444  struct sigaction sig;
445 
446  memset(&sig, 0, sizeof(struct sigaction));
447  sig.sa_handler = SIG_IGN;
448  sigemptyset(&sig.sa_mask);
449 #ifdef SA_INTERRUPT
450  sig.sa_flags = SA_INTERRUPT; /* SunOS */
451 #else
452  sig.sa_flags = SA_RESTART;
453 #endif
454  if (0 != sigaction(SIGPIPE, &sig, &oldsig))
455  fprintf(stderr,
456  "Failed to install SIGPIPE handler: %s\n",
457  strerror(errno));
458 }
Here is the caller graph for this function:

◆ make_dev_zero()

static void make_dev_zero ( int  fd,
int  flags 
)
static

Turn the given file descriptor in to '/dev/null'.

Parameters
fdfd to bind to /dev/null
flagsflags to use (O_RDONLY or O_WRONLY)

Definition at line 468 of file gnunet-helper-fs-publish.c.

References GNUNET_assert, and GNUNET_break.

Referenced by main().

469 {
470  int z;
471 
472  GNUNET_assert(0 == close(fd));
473  z = open("/dev/null", flags);
474  GNUNET_assert(-1 != z);
475  if (z == fd)
476  return;
477  GNUNET_break(fd == dup2(z, fd));
478  GNUNET_assert(0 == close(z));
479 }
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
Here is the caller graph for this function:

◆ main()

int main ( int  argc,
char *const *  argv 
)

Main function of the helper process to extract meta data.

Parameters
argcshould be 3
argv[0] our binary name [1] name of the file or directory to process [2] "-" to disable extraction, NULL for defaults, otherwise custom plugins to load from LE
Returns
0 on success

Definition at line 493 of file gnunet-helper-fs-publish.c.

References extract_files(), free_tree(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_OK, ignore_sigpipe(), make_dev_zero(), output_stream, plugins, preprocess_file(), and write_message().

494 {
495  const char *filename_expanded;
496  const char *ex;
497  struct ScanTreeNode *root;
498 
499  ignore_sigpipe();
500  /* move stdout to some other FD for IPC, bind
501  stdout/stderr to /dev/null */
502  output_stream = dup(1);
503  make_dev_zero(1, O_WRONLY);
504  make_dev_zero(2, O_WRONLY);
505 
506  /* parse command line */
507  if ((3 != argc) && (2 != argc))
508  {
509  fprintf(stderr,
510  "%s",
511  "gnunet-helper-fs-publish needs exactly one or two arguments\n");
512  return 1;
513  }
514  filename_expanded = argv[1];
515  ex = argv[2];
516  if ((NULL == ex) || (0 != strcmp(ex, "-")))
517  {
518 #if HAVE_LIBEXTRACTOR
519  plugins = EXTRACTOR_plugin_add_defaults(EXTRACTOR_OPTION_DEFAULT_POLICY);
520  if (NULL != ex)
521  plugins = EXTRACTOR_plugin_add_config(plugins,
522  ex,
523  EXTRACTOR_OPTION_DEFAULT_POLICY);
524 #endif
525  }
526 
527  /* scan tree to find out how much work there is to be done */
528  if (GNUNET_OK != preprocess_file(filename_expanded, &root))
529  {
531 #if HAVE_LIBEXTRACTOR
532  EXTRACTOR_plugin_remove_all(plugins);
533 #endif
534  return 2;
535  }
536  /* signal that we're done counting files, so that a percentage of
537  progress can now be calculated */
538  if (GNUNET_OK !=
540  NULL,
541  0))
542  {
543 #if HAVE_LIBEXTRACTOR
544  EXTRACTOR_plugin_remove_all(plugins);
545 #endif
546  return 3;
547  }
548  if (NULL != root)
549  {
550  if (GNUNET_OK != extract_files(root))
551  {
553  NULL,
554  0);
555  free_tree(root);
556 #if HAVE_LIBEXTRACTOR
557  EXTRACTOR_plugin_remove_all(plugins);
558 #endif
559  return 4;
560  }
561  free_tree(root);
562  }
563  /* enable "clean" shutdown by telling parent that we are done */
565  NULL,
566  0);
567 #if HAVE_LIBEXTRACTOR
568  EXTRACTOR_plugin_remove_all(plugins);
569 #endif
570  return 0;
571 }
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to &#39;/dev/null&#39;.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
static int output_stream
File descriptor we use for IPC with the parent.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:67
A node of a directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
Here is the call graph for this function:

Variable Documentation

◆ output_stream

int output_stream
static

File descriptor we use for IPC with the parent.

Definition at line 92 of file gnunet-helper-fs-publish.c.

Referenced by main(), and write_all().