GNUnet 0.21.1
gnunet-helper-fs-publish.c File Reference
#include "platform.h"
#include "gnunet_fs_service.h"
Include dependency graph for gnunet-helper-fs-publish.c:

Go to the source code of this file.

Data Structures

struct  ScanTreeNode
 A node of a directory tree. More...
 
struct  RecursionContext
 Closure for the 'scan_callback'. More...
 

Functions

static void free_tree (struct ScanTreeNode *tree)
 Free memory of the tree structure. More...
 
static int write_all (const void *buf, size_t size)
 Write size bytes from buf into the output_stream. More...
 
static int write_message (uint16_t message_type, const char *data, size_t data_length)
 Write message to the master process. More...
 
static int preprocess_file (const char *filename, struct ScanTreeNode **dst)
 Function called to (recursively) add all of the files in the directory to the tree. More...
 
static int scan_callback (void *cls, const char *filename)
 Function called by the directory iterator to (recursively) add all of the files in the directory to the tree. More...
 
static int extract_files (struct ScanTreeNode *item)
 Extract metadata from files. More...
 
static void ignore_sigpipe ()
 Install a signal handler to ignore SIGPIPE. More...
 
static void make_dev_zero (int fd, int flags)
 Turn the given file descriptor in to '/dev/null'. More...
 
int main (int argc, char *const *argv)
 Main function of the helper process to extract meta data. More...
 

Variables

static int output_stream
 File descriptor we use for IPC with the parent. More...
 

Function Documentation

◆ free_tree()

static void free_tree ( struct ScanTreeNode tree)
static

Free memory of the tree structure.

Parameters
treetree to free

Definition at line 164 of file gnunet-helper-fs-publish.c.

165{
166 struct ScanTreeNode *pos;
167
168 while (NULL != (pos = tree->children_head))
169 free_tree (pos);
170 if (NULL != tree->parent)
172 tree->parent->children_tail,
173 tree);
174 GNUNET_free (tree->filename);
175 GNUNET_free (tree);
176}
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
#define GNUNET_free(ptr)
Wrapper around free.
A node of a directory tree.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, ScanTreeNode::filename, free_tree(), GNUNET_CONTAINER_DLL_remove, GNUNET_free, and ScanTreeNode::parent.

Referenced by free_tree(), main(), and preprocess_file().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ write_all()

static int write_all ( const void *  buf,
size_t  size 
)
static

Write size bytes from buf into the output_stream.

Parameters
bufbuffer with data to write
sizenumber of bytes to write
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 187 of file gnunet-helper-fs-publish.c.

188{
189 const char *cbuf = buf;
190 size_t total;
191 ssize_t wr;
192
193 total = 0;
194 do
195 {
196 wr = write (output_stream, &cbuf[total], size - total);
197 if (wr > 0)
198 total += wr;
199 }
200 while ((wr > 0) && (total < size));
201 if (wr <= 0)
203 "Failed to write to stdout: %s\n",
204 strerror (errno));
205 return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
206}
static int output_stream
File descriptor we use for IPC with the parent.
#define GNUNET_log(kind,...)
@ GNUNET_OK
@ GNUNET_SYSERR
@ GNUNET_ERROR_TYPE_DEBUG
static unsigned int size
Size of the "table".
Definition: peer.c:68

References GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_OK, GNUNET_SYSERR, output_stream, and size.

Referenced by write_message().

Here is the caller graph for this function:

◆ write_message()

static int write_message ( uint16_t  message_type,
const char *  data,
size_t  data_length 
)
static

Write message to the master process.

Parameters
message_typemessage type to use
datadata to append, NULL for none
data_lengthnumber of bytes in data
Returns
GNUNET_SYSERR to stop scanning (the pipe was broken somehow)

Definition at line 218 of file gnunet-helper-fs-publish.c.

219{
220 struct GNUNET_MessageHeader hdr;
221
222#if 0
223 fprintf (stderr,
224 "Helper sends %u-byte message of type %u\n",
225 (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
226 (unsigned int) message_type);
227#endif
228 hdr.type = htons (message_type);
229 hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
230 if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
231 (GNUNET_OK != write_all (data, data_length)))
232 return GNUNET_SYSERR;
233 return GNUNET_OK;
234}
static char * data
The data to insert into the dht.
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
Header for all communications.

References data, GNUNET_OK, GNUNET_SYSERR, GNUNET_MessageHeader::size, GNUNET_MessageHeader::type, and write_all().

Referenced by extract_files(), main(), and preprocess_file().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ preprocess_file()

static int preprocess_file ( const char *  filename,
struct ScanTreeNode **  dst 
)
static

Function called to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
filenamefile or directory to scan
dstwhere to store the resulting share tree item; NULL is stored in dst upon recoverable errors (GNUNET_OK is returned)
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 309 of file gnunet-helper-fs-publish.c.

310{
311 struct ScanTreeNode *item;
312 struct stat sbuf;
313 uint64_t fsize = 0;
314
315 if ((0 != stat (filename, &sbuf)) ||
316 ((! S_ISDIR (sbuf.st_mode)) &&
317 (GNUNET_OK !=
319 {
320 /* If the file doesn't exist (or is not stat-able for any other reason)
321 skip it (but report it), but do continue. */
322 if (GNUNET_OK !=
324 filename,
325 strlen (filename) + 1))
326 return GNUNET_SYSERR;
327 /* recoverable error, store 'NULL' in *dst */
328 *dst = NULL;
329 return GNUNET_OK;
330 }
331
332 /* Report the progress */
333 if (
334 GNUNET_OK !=
335 write_message (S_ISDIR (sbuf.st_mode)
338 filename,
339 strlen (filename) + 1))
340 return GNUNET_SYSERR;
341 item = GNUNET_new (struct ScanTreeNode);
343 item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
344 item->file_size = fsize;
345 if (GNUNET_YES == item->is_directory)
346 {
347 struct RecursionContext rc;
348
349 rc.parent = item;
350 rc.stop = GNUNET_NO;
352 if (
353 (GNUNET_YES == rc.stop) ||
354 (GNUNET_OK !=
356 "..",
357 3)))
358 {
359 free_tree (item);
360 return GNUNET_SYSERR;
361 }
362 }
363 *dst = item;
364 return GNUNET_OK;
365}
static char * filename
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:221
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:814
@ GNUNET_YES
@ GNUNET_NO
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
Closure for the 'scan_callback'.
struct ScanTreeNode * parent
Parent to add the files to.
uint64_t file_size
Size of the file (if it is a file), in bytes.
int is_directory
GNUNET_YES if this is a directory

References ScanTreeNode::file_size, filename, ScanTreeNode::filename, free_tree(), GNUNET_DISK_directory_scan(), GNUNET_DISK_file_size(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_new, GNUNET_NO, GNUNET_OK, GNUNET_strdup, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, RecursionContext::parent, scan_callback(), RecursionContext::stop, and write_message().

Referenced by main(), and scan_callback().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ scan_callback()

static int scan_callback ( void *  cls,
const char *  filename 
)
static

Function called by the directory iterator to (recursively) add all of the files in the directory to the tree.

Called by the directory scanner to initiate the scan. Does NOT yet add any metadata.

Parameters
clsthe struct RecursionContext
filenamefile or directory to scan
Returns
GNUNET_OK on success, GNUNET_SYSERR on error

Definition at line 278 of file gnunet-helper-fs-publish.c.

279{
280 struct RecursionContext *rc = cls;
281 struct ScanTreeNode *chld;
282
283 if (GNUNET_OK != preprocess_file (filename, &chld))
284 {
285 rc->stop = GNUNET_YES;
286 return GNUNET_SYSERR;
287 }
288 if (NULL == chld)
289 return GNUNET_OK;
290 chld->parent = rc->parent;
293 chld);
294 return GNUNET_OK;
295}
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
int stop
Flag to set to GNUNET_YES on serious errors.

References ScanTreeNode::children_head, ScanTreeNode::children_tail, filename, GNUNET_CONTAINER_DLL_insert, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::parent, RecursionContext::parent, preprocess_file(), and RecursionContext::stop.

Referenced by preprocess_file().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ extract_files()

static int extract_files ( struct ScanTreeNode item)
static

Extract metadata from files.

Parameters
itementry we are processing
Returns
GNUNET_OK on success, GNUNET_SYSERR on fatal errors

Definition at line 375 of file gnunet-helper-fs-publish.c.

376{
377 struct GNUNET_FS_MetaData *meta;
378 ssize_t size;
379 size_t slen;
380
381 if (GNUNET_YES == item->is_directory)
382 {
383 /* for directories, we simply only descent, no extraction, no
384 progress reporting */
385 struct ScanTreeNode *pos;
386
387 for (pos = item->children_head; NULL != pos; pos = pos->next)
388 if (GNUNET_OK != extract_files (pos))
389 return GNUNET_SYSERR;
390 return GNUNET_OK;
391 }
392
393 /* this is the expensive operation, *afterwards* we'll check for aborts */
395#if HAVE_LIBEXTRACTOR
396 EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
397#endif
398 slen = strlen (item->filename) + 1;
400 if (-1 == size)
401 {
402 /* no meta data */
404 if (GNUNET_OK !=
406 item->filename,
407 slen))
408 return GNUNET_SYSERR;
409 return GNUNET_OK;
410 }
411 else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
412 {
413 /* We can't transfer more than 64k bytes in one message. */
414 size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
415 }
416 {
417 char buf[size + slen];
418 char *dst = &buf[slen];
419
420 GNUNET_memcpy (buf, item->filename, slen);
422 meta,
423 &dst,
424 size,
426 if (size < 0)
427 {
428 GNUNET_break (0);
429 size = 0;
430 }
432 if (GNUNET_OK !=
434 buf,
435 slen + size))
436 return GNUNET_SYSERR;
437 }
438 return GNUNET_OK;
439}
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
static struct GNUNET_FS_MetaData * meta
Meta-data provided via command-line option.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
ssize_t GNUNET_FS_meta_data_get_serialized_size(const struct GNUNET_FS_MetaData *md)
Get the size of the full meta-data in serialized form.
Definition: meta_data.c:858
ssize_t GNUNET_FS_meta_data_serialize(const struct GNUNET_FS_MetaData *md, char **target, size_t max, enum GNUNET_FS_MetaDataSerializationOptions opt)
Serialize meta-data to target.
Definition: meta_data.c:637
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_create(void)
Create a fresh meta data container.
Definition: meta_data.c:127
void GNUNET_FS_meta_data_destroy(struct GNUNET_FS_MetaData *md)
Free meta data.
Definition: meta_data.c:166
@ GNUNET_FS_META_DATA_SERIALIZE_PART
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:69
Meta data to associate with a file, directory or namespace.
Definition: meta_data.c:92
struct ScanTreeNode * next
This is a doubly-linked list.

References ScanTreeNode::children_head, extract_files(), ScanTreeNode::filename, GNUNET_break, GNUNET_FS_meta_data_create(), GNUNET_FS_meta_data_destroy(), GNUNET_FS_meta_data_get_serialized_size(), GNUNET_FS_meta_data_serialize(), GNUNET_FS_META_DATA_SERIALIZE_PART, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_OK, GNUNET_SYSERR, GNUNET_YES, ScanTreeNode::is_directory, meta, ScanTreeNode::next, plugins, size, and write_message().

Referenced by extract_files(), and main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ ignore_sigpipe()

static void ignore_sigpipe ( )
static

Install a signal handler to ignore SIGPIPE.

Definition at line 446 of file gnunet-helper-fs-publish.c.

447{
448 struct sigaction oldsig;
449 struct sigaction sig;
450
451 memset (&sig, 0, sizeof(struct sigaction));
452 sig.sa_handler = SIG_IGN;
453 sigemptyset (&sig.sa_mask);
454#ifdef SA_INTERRUPT
455 sig.sa_flags = SA_INTERRUPT; /* SunOS */
456#else
457 sig.sa_flags = SA_RESTART;
458#endif
459 if (0 != sigaction (SIGPIPE, &sig, &oldsig))
460 fprintf (stderr,
461 "Failed to install SIGPIPE handler: %s\n",
462 strerror (errno));
463}

Referenced by main().

Here is the caller graph for this function:

◆ make_dev_zero()

static void make_dev_zero ( int  fd,
int  flags 
)
static

Turn the given file descriptor in to '/dev/null'.

Parameters
fdfd to bind to /dev/null
flagsflags to use (O_RDONLY or O_WRONLY)

Definition at line 473 of file gnunet-helper-fs-publish.c.

474{
475 int z;
476
477 GNUNET_assert (0 == close (fd));
478 z = open ("/dev/null", flags);
479 GNUNET_assert (-1 != z);
480 if (z == fd)
481 return;
482 GNUNET_break (fd == dup2 (z, fd));
483 GNUNET_assert (0 == close (z));
484}
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.

References GNUNET_assert, and GNUNET_break.

Referenced by main().

Here is the caller graph for this function:

◆ main()

int main ( int  argc,
char *const *  argv 
)

Main function of the helper process to extract meta data.

Parameters
argcshould be 3
argv[0] our binary name [1] name of the file or directory to process [2] "-" to disable extraction, NULL for defaults, otherwise custom plugins to load from LE
Returns
0 on success

Definition at line 498 of file gnunet-helper-fs-publish.c.

499{
500 const char *filename_expanded;
501 const char *ex;
502 struct ScanTreeNode *root;
503
505 /* move stdout to some other FD for IPC, bind
506 stdout/stderr to /dev/null */
507 output_stream = dup (1);
508 make_dev_zero (1, O_WRONLY);
509 make_dev_zero (2, O_WRONLY);
510
511 /* parse command line */
512 if ((3 != argc) && (2 != argc))
513 {
514 fprintf (stderr,
515 "%s",
516 "gnunet-helper-fs-publish needs exactly one or two arguments\n");
517 return 1;
518 }
519 filename_expanded = argv[1];
520 ex = argv[2];
521 if ((NULL == ex) || (0 != strcmp (ex, "-")))
522 {
523#if HAVE_LIBEXTRACTOR
524 plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
525 if (NULL != ex)
526 plugins = EXTRACTOR_plugin_add_config (plugins,
527 ex,
528 EXTRACTOR_OPTION_DEFAULT_POLICY);
529#endif
530 }
531
532 /* scan tree to find out how much work there is to be done */
533 if (GNUNET_OK != preprocess_file (filename_expanded, &root))
534 {
536#if HAVE_LIBEXTRACTOR
537 EXTRACTOR_plugin_remove_all (plugins);
538#endif
539 return 2;
540 }
541 /* signal that we're done counting files, so that a percentage of
542 progress can now be calculated */
543 if (GNUNET_OK !=
545 NULL,
546 0))
547 {
548#if HAVE_LIBEXTRACTOR
549 EXTRACTOR_plugin_remove_all (plugins);
550#endif
551 return 3;
552 }
553 if (NULL != root)
554 {
555 if (GNUNET_OK != extract_files (root))
556 {
558 NULL,
559 0);
560 free_tree (root);
561#if HAVE_LIBEXTRACTOR
562 EXTRACTOR_plugin_remove_all (plugins);
563#endif
564 return 4;
565 }
566 free_tree (root);
567 }
568 /* enable "clean" shutdown by telling parent that we are done */
570 NULL,
571 0);
572#if HAVE_LIBEXTRACTOR
573 EXTRACTOR_plugin_remove_all (plugins);
574#endif
575 return 0;
576}
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to '/dev/null'.
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.

References extract_files(), free_tree(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_OK, ignore_sigpipe(), make_dev_zero(), output_stream, plugins, preprocess_file(), and write_message().

Here is the call graph for this function:

Variable Documentation

◆ output_stream

int output_stream
static

File descriptor we use for IPC with the parent.

Definition at line 94 of file gnunet-helper-fs-publish.c.

Referenced by main(), and write_all().