GNUnet  0.20.0
gnunet-helper-fs-publish.c File Reference

Tool to help extract meta data asynchronously. More...

#include "platform.h"
#include "gnunet_fs_service.h"
Include dependency graph for gnunet-helper-fs-publish.c:

Go to the source code of this file.

Data Structures

struct  ScanTreeNode
 A node of a directory tree. More...
 
struct  RecursionContext
 Closure for the 'scan_callback'. More...
 

Functions

static void free_tree (struct ScanTreeNode *tree)
 Free memory of the tree structure. More...
 
static int write_all (const void *buf, size_t size)
 Write size bytes from buf into the output_stream. More...
 
static int write_message (uint16_t message_type, const char *data, size_t data_length)
 Write message to the master process. More...
 
static int preprocess_file (const char *filename, struct ScanTreeNode **dst)
 Function called to (recursively) add all of the files in the directory to the tree. More...
 
static int scan_callback (void *cls, const char *filename)
 Function called by the directory iterator to (recursively) add all of the files in the directory to the tree. More...
 
static int extract_files (struct ScanTreeNode *item)
 Extract metadata from files. More...
 
static void ignore_sigpipe ()
 Install a signal handler to ignore SIGPIPE. More...
 
static void make_dev_zero (int fd, int flags)
 Turn the given file descriptor in to '/dev/null'. More...
 
int main (int argc, char *const *argv)
 Main function of the helper process to extract meta data. More...
 

Variables

static int output_stream
 File descriptor we use for IPC with the parent. More...
 

Detailed Description

Tool to help extract meta data asynchronously.

Author
Christian Grothoff

This program will scan a directory for files with meta data and report the results to stdout.

Definition in file gnunet-helper-fs-publish.c.

Function Documentation

◆ free_tree()

static void free_tree ( struct ScanTreeNode tree)
static

Free memory of the tree structure.

Parameters
treetree to free

Definition at line 164 of file gnunet-helper-fs-publish.c.

165 {
166  struct ScanTreeNode *pos;
167 
168  while (NULL != (pos = tree->children_head))
169  free_tree (pos);
170  if (NULL != tree->parent)
172  tree->parent->children_tail,
173  tree);
174  GNUNET_free (tree->filename);
175  GNUNET_free (tree);
176 }
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
#define GNUNET_free(ptr)
Wrapper around free.
A node of a directory tree.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, ScanTreeNode::filename, GNUNET_CONTAINER_DLL_remove, GNUNET_free, and ScanTreeNode::parent.

Referenced by main(), and preprocess_file().

Here is the caller graph for this function:

◆ write_all()

static int write_all ( const void *  buf,
size_t  size 
)
static

Write size bytes from buf into the output_stream.

Parameters
bufbuffer with data to write
sizenumber of bytes to write
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 187 of file gnunet-helper-fs-publish.c.

188 {
189  const char *cbuf = buf;
190  size_t total;
191  ssize_t wr;
192 
193  total = 0;
194  do
195  {
196  wr = write (output_stream, &cbuf[total], size - total);
197  if (wr > 0)
198  total += wr;
199  }
200  while ((wr > 0) && (total < size));
201  if (wr <= 0)
203  "Failed to write to stdout: %s\n",
204  strerror (errno));
205  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
206 }
static int output_stream
File descriptor we use for IPC with the parent.
static char buf[2048]
#define GNUNET_log(kind,...)
@ GNUNET_OK
@ GNUNET_SYSERR
@ GNUNET_ERROR_TYPE_DEBUG
static unsigned int size
Size of the "table".
Definition: peer.c:68

References buf, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_OK, GNUNET_SYSERR, output_stream, and size.

Referenced by write_message().

Here is the caller graph for this function:

◆ write_message()

static int write_message ( uint16_t  message_type,
const char *  data,
size_t  data_length 
)
static

Write message to the master process.

Parameters
message_typemessage type to use
datadata to append, NULL for none
data_lengthnumber of bytes in data
Returns
GNUNET_SYSERR to stop scanning (the pipe was broken somehow)

Definition at line 218 of file gnunet-helper-fs-publish.c.

219 {
220  struct GNUNET_MessageHeader hdr;
221 
222 #if 0
223  fprintf (stderr,
224  "Helper sends %u-byte message of type %u\n",
225  (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
226  (unsigned int) message_type);
227 #endif
228  hdr.type = htons (message_type);
229  hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
230  if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
231  (GNUNET_OK != write_all (data, data_length)))
232  return GNUNET_SYSERR;
233  return GNUNET_OK;
234 }
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
uint32_t data
The data value.
Header for all communications.

References data, GNUNET_OK, GNUNET_SYSERR, GNUNET_MessageHeader::size, GNUNET_MessageHeader::type, and write_all().

Referenced by extract_files(), GNUNET_TESTING_send_barrier_attach(), main(), and preprocess_file().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ preprocess_file()

static int preprocess_file ( const char *  filename,
struct ScanTreeNode **  dst 
)
static

Function called to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
filenamefile or directory to scan
dstwhere to store the resulting share tree item; NULL is stored in dst upon recoverable errors (GNUNET_OK is returned)
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 309 of file gnunet-helper-fs-publish.c.

310 {
311  struct ScanTreeNode *item;
312  struct stat sbuf;
313  uint64_t fsize = 0;
314 
315  if ((0 != stat (filename, &sbuf)) ||
316  ((! S_ISDIR (sbuf.st_mode)) &&
317  (GNUNET_OK !=
319  {
320  /* If the file doesn't exist (or is not stat-able for any other reason)
321  skip it (but report it), but do continue. */
322  if (GNUNET_OK !=
324  filename,
325  strlen (filename) + 1))
326  return GNUNET_SYSERR;
327  /* recoverable error, store 'NULL' in *dst */
328  *dst = NULL;
329  return GNUNET_OK;
330  }
331 
332  /* Report the progress */
333  if (
334  GNUNET_OK !=
335  write_message (S_ISDIR (sbuf.st_mode)
338  filename,
339  strlen (filename) + 1))
340  return GNUNET_SYSERR;
341  item = GNUNET_new (struct ScanTreeNode);
342  item->filename = GNUNET_strdup (filename);
343  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
344  item->file_size = fsize;
345  if (GNUNET_YES == item->is_directory)
346  {
347  struct RecursionContext rc;
348 
349  rc.parent = item;
350  rc.stop = GNUNET_NO;
352  if (
353  (GNUNET_YES == rc.stop) ||
354  (GNUNET_OK !=
356  "..",
357  3)))
358  {
359  free_tree (item);
360  return GNUNET_SYSERR;
361  }
362  }
363  *dst = item;
364  return GNUNET_OK;
365 }
static char * filename
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:221
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:814
@ GNUNET_YES
@ GNUNET_NO
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
Closure for the 'scan_callback'.
struct ScanTreeNode * parent
Parent to add the files to.
uint64_t file_size
Size of the file (if it is a file), in bytes.
int is_directory
GNUNET_YES if this is a directory

References ScanTreeNode::file_size, filename, ScanTreeNode::filename, free_tree(), GNUNET_DISK_directory_scan(), GNUNET_DISK_file_size(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_new, GNUNET_NO, GNUNET_OK, GNUNET_strdup, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, RecursionContext::parent, scan_callback(), RecursionContext::stop, and write_message().

Referenced by main(), and scan_callback().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_callback()

static int scan_callback ( void *  cls,
const char *  filename 
)
static

Function called by the directory iterator to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
clsthe struct RecursionContext
filenamefile or directory to scan
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 278 of file gnunet-helper-fs-publish.c.

279 {
280  struct RecursionContext *rc = cls;
281  struct ScanTreeNode *chld;
282 
283  if (GNUNET_OK != preprocess_file (filename, &chld))
284  {
285  rc->stop = GNUNET_YES;
286  return GNUNET_SYSERR;
287  }
288  if (NULL == chld)
289  return GNUNET_OK;
290  chld->parent = rc->parent;
292  rc->parent->children_tail,
293  chld);
294  return GNUNET_OK;
295 }
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
int stop
Flag to set to GNUNET_YES on serious errors.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, filename, GNUNET_CONTAINER_DLL_insert, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::parent, RecursionContext::parent, preprocess_file(), and RecursionContext::stop.

Referenced by preprocess_file().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ extract_files()

static int extract_files ( struct ScanTreeNode item)
static

Extract metadata from files.

Parameters
itementry we are processing
Returns
GNUNET_OK on success, GNUNET_SYSERR on fatal errors

Definition at line 375 of file gnunet-helper-fs-publish.c.

376 {
377  struct GNUNET_FS_MetaData *meta;
378  ssize_t size;
379  size_t slen;
380 
381  if (GNUNET_YES == item->is_directory)
382  {
383  /* for directories, we simply only descent, no extraction, no
384  progress reporting */
385  struct ScanTreeNode *pos;
386 
387  for (pos = item->children_head; NULL != pos; pos = pos->next)
388  if (GNUNET_OK != extract_files (pos))
389  return GNUNET_SYSERR;
390  return GNUNET_OK;
391  }
392 
393  /* this is the expensive operation, *afterwards* we'll check for aborts */
395 #if HAVE_LIBEXTRACTOR
396  EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
397 #endif
398  slen = strlen (item->filename) + 1;
400  if (-1 == size)
401  {
402  /* no meta data */
404  if (GNUNET_OK !=
406  item->filename,
407  slen))
408  return GNUNET_SYSERR;
409  return GNUNET_OK;
410  }
411  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
412  {
413  /* We can't transfer more than 64k bytes in one message. */
414  size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
415  }
416  {
417  char buf[size + slen];
418  char *dst = &buf[slen];
419 
420  GNUNET_memcpy (buf, item->filename, slen);
422  meta,
423  &dst,
424  size,
426  if (size < 0)
427  {
428  GNUNET_break (0);
429  size = 0;
430  }
432  if (GNUNET_OK !=
434  buf,
435  slen + size))
436  return GNUNET_SYSERR;
437  }
438  return GNUNET_OK;
439 }
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
static struct GNUNET_FS_MetaData * meta
Meta-data provided via command-line option.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
ssize_t GNUNET_FS_meta_data_get_serialized_size(const struct GNUNET_FS_MetaData *md)
Get the size of the full meta-data in serialized form.
Definition: meta_data.c:863
ssize_t GNUNET_FS_meta_data_serialize(const struct GNUNET_FS_MetaData *md, char **target, size_t max, enum GNUNET_FS_MetaDataSerializationOptions opt)
Serialize meta-data to target.
Definition: meta_data.c:642
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_create()
Create a fresh struct FS_MetaData token.
Definition: meta_data.c:132
void GNUNET_FS_meta_data_destroy(struct GNUNET_FS_MetaData *md)
Free meta data.
Definition: meta_data.c:171
@ GNUNET_FS_META_DATA_SERIALIZE_PART
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:69
Meta data to associate with a file, directory or namespace.
Definition: meta_data.c:97
struct ScanTreeNode * next
This is a doubly-linked list.

References buf, ScanTreeNode::children_head, ScanTreeNode::filename, GNUNET_break, GNUNET_FS_meta_data_create(), GNUNET_FS_meta_data_destroy(), GNUNET_FS_meta_data_get_serialized_size(), GNUNET_FS_meta_data_serialize(), GNUNET_FS_META_DATA_SERIALIZE_PART, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, meta, ScanTreeNode::next, plugins, size, and write_message().

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ ignore_sigpipe()

static void ignore_sigpipe ( )
static

Install a signal handler to ignore SIGPIPE.

Definition at line 446 of file gnunet-helper-fs-publish.c.

447 {
448  struct sigaction oldsig;
449  struct sigaction sig;
450 
451  memset (&sig, 0, sizeof(struct sigaction));
452  sig.sa_handler = SIG_IGN;
453  sigemptyset (&sig.sa_mask);
454 #ifdef SA_INTERRUPT
455  sig.sa_flags = SA_INTERRUPT; /* SunOS */
456 #else
457  sig.sa_flags = SA_RESTART;
458 #endif
459  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
460  fprintf (stderr,
461  "Failed to install SIGPIPE handler: %s\n",
462  strerror (errno));
463 }

Referenced by main().

Here is the caller graph for this function:

◆ make_dev_zero()

static void make_dev_zero ( int  fd,
int  flags 
)
static

Turn the given file descriptor in to '/dev/null'.

Parameters
fdfd to bind to /dev/null
flagsflags to use (O_RDONLY or O_WRONLY)

Definition at line 473 of file gnunet-helper-fs-publish.c.

474 {
475  int z;
476 
477  GNUNET_assert (0 == close (fd));
478  z = open ("/dev/null", flags);
479  GNUNET_assert (-1 != z);
480  if (z == fd)
481  return;
482  GNUNET_break (fd == dup2 (z, fd));
483  GNUNET_assert (0 == close (z));
484 }
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.

References GNUNET_assert, and GNUNET_break.

Referenced by main().

Here is the caller graph for this function:

◆ main()

int main ( int  argc,
char *const *  argv 
)

Main function of the helper process to extract meta data.

Parameters
argcshould be 3
argv[0] our binary name [1] name of the file or directory to process [2] "-" to disable extraction, NULL for defaults, otherwise custom plugins to load from LE
Returns
0 on success

Definition at line 498 of file gnunet-helper-fs-publish.c.

499 {
500  const char *filename_expanded;
501  const char *ex;
502  struct ScanTreeNode *root;
503 
504  ignore_sigpipe ();
505  /* move stdout to some other FD for IPC, bind
506  stdout/stderr to /dev/null */
507  output_stream = dup (1);
508  make_dev_zero (1, O_WRONLY);
509  make_dev_zero (2, O_WRONLY);
510 
511  /* parse command line */
512  if ((3 != argc) && (2 != argc))
513  {
514  fprintf (stderr,
515  "%s",
516  "gnunet-helper-fs-publish needs exactly one or two arguments\n");
517  return 1;
518  }
519  filename_expanded = argv[1];
520  ex = argv[2];
521  if ((NULL == ex) || (0 != strcmp (ex, "-")))
522  {
523 #if HAVE_LIBEXTRACTOR
524  plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
525  if (NULL != ex)
526  plugins = EXTRACTOR_plugin_add_config (plugins,
527  ex,
528  EXTRACTOR_OPTION_DEFAULT_POLICY);
529 #endif
530  }
531 
532  /* scan tree to find out how much work there is to be done */
533  if (GNUNET_OK != preprocess_file (filename_expanded, &root))
534  {
536 #if HAVE_LIBEXTRACTOR
537  EXTRACTOR_plugin_remove_all (plugins);
538 #endif
539  return 2;
540  }
541  /* signal that we're done counting files, so that a percentage of
542  progress can now be calculated */
543  if (GNUNET_OK !=
545  NULL,
546  0))
547  {
548 #if HAVE_LIBEXTRACTOR
549  EXTRACTOR_plugin_remove_all (plugins);
550 #endif
551  return 3;
552  }
553  if (NULL != root)
554  {
555  if (GNUNET_OK != extract_files (root))
556  {
558  NULL,
559  0);
560  free_tree (root);
561 #if HAVE_LIBEXTRACTOR
562  EXTRACTOR_plugin_remove_all (plugins);
563 #endif
564  return 4;
565  }
566  free_tree (root);
567  }
568  /* enable "clean" shutdown by telling parent that we are done */
570  NULL,
571  0);
572 #if HAVE_LIBEXTRACTOR
573  EXTRACTOR_plugin_remove_all (plugins);
574 #endif
575  return 0;
576 }
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to '/dev/null'.
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.

References extract_files(), free_tree(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_OK, ignore_sigpipe(), make_dev_zero(), output_stream, plugins, preprocess_file(), and write_message().

Here is the call graph for this function:

Variable Documentation

◆ output_stream

int output_stream
static

File descriptor we use for IPC with the parent.

Definition at line 94 of file gnunet-helper-fs-publish.c.

Referenced by main(), and write_all().