GNUnet  0.10.x
gnunet-helper-fs-publish.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19 */
20 
29 #include "platform.h"
30 #include "gnunet_fs_service.h"
31 
32 
37 {
38 
42  struct ScanTreeNode *next;
43 
47  struct ScanTreeNode *prev;
48 
53 
59 
65 
69  char *filename;
70 
75  uint64_t file_size;
76 
81 
82 };
83 
84 
85 #if HAVE_LIBEXTRACTOR
86 
89 static struct EXTRACTOR_PluginList *plugins;
90 #endif
91 
95 static int output_stream;
96 
97 
98 #if HAVE_LIBEXTRACTOR
99 
116 static int
117 add_to_md (void *cls,
118  const char *plugin_name,
120  enum EXTRACTOR_MetaFormat format,
121  const char *data_mime_type,
122  const char *data,
123  size_t data_len)
124 {
125  struct GNUNET_CONTAINER_MetaData *md = cls;
126 
127  if ( ((EXTRACTOR_METAFORMAT_UTF8 == format) ||
128  (EXTRACTOR_METAFORMAT_C_STRING == format)) &&
129  ('\0' != data[data_len - 1]) )
130  {
131  char zdata[data_len + 1];
132  GNUNET_memcpy (zdata, data, data_len);
133  zdata[data_len] = '\0';
134  (void) GNUNET_CONTAINER_meta_data_insert (md, plugin_name, type, format,
135  data_mime_type, zdata, data_len + 1);
136  }
137  else
138  {
139  (void) GNUNET_CONTAINER_meta_data_insert (md, plugin_name, type, format,
140  data_mime_type, data, data_len);
141  }
142  return 0;
143 }
144 #endif
145 
146 
152 static void
153 free_tree (struct ScanTreeNode *tree)
154 {
155  struct ScanTreeNode *pos;
156 
157  while (NULL != (pos = tree->children_head))
158  free_tree (pos);
159  if (NULL != tree->parent)
161  tree->parent->children_tail,
162  tree);
163  GNUNET_free (tree->filename);
164  GNUNET_free (tree);
165 }
166 
167 
175 static int
176 write_all (const void *buf,
177  size_t size)
178 {
179  const char *cbuf = buf;
180  size_t total;
181  ssize_t wr;
182 
183  total = 0;
184  do
185  {
186  wr = write (output_stream,
187  &cbuf[total],
188  size - total);
189  if (wr > 0)
190  total += wr;
191  } while ( (wr > 0) && (total < size) );
192  if (wr <= 0)
194  "Failed to write to stdout: %s\n",
195  strerror (errno));
196  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
197 }
198 
199 
208 static int
209 write_message (uint16_t message_type,
210  const char *data,
211  size_t data_length)
212 {
213  struct GNUNET_MessageHeader hdr;
214 
215 #if 0
216  fprintf (stderr,
217  "Helper sends %u-byte message of type %u\n",
218  (unsigned int) (sizeof (struct GNUNET_MessageHeader) + data_length),
219  (unsigned int) message_type);
220 #endif
221  hdr.type = htons (message_type);
222  hdr.size = htons (sizeof (struct GNUNET_MessageHeader) + data_length);
223  if ( (GNUNET_OK !=
224  write_all (&hdr,
225  sizeof (hdr))) ||
226  (GNUNET_OK !=
227  write_all (data,
228  data_length)) )
229  return GNUNET_SYSERR;
230  return GNUNET_OK;
231 }
232 
233 
244 static int
245 preprocess_file (const char *filename,
246  struct ScanTreeNode **dst);
247 
248 
253 {
258 
262  int stop;
263 };
264 
265 
275 static int
276 scan_callback (void *cls,
277  const char *filename)
278 {
279  struct RecursionContext *rc = cls;
280  struct ScanTreeNode *chld;
281 
282  if (GNUNET_OK !=
283  preprocess_file (filename,
284  &chld))
285  {
286  rc->stop = GNUNET_YES;
287  return GNUNET_SYSERR;
288  }
289  if (NULL == chld)
290  return GNUNET_OK;
291  chld->parent = rc->parent;
293  rc->parent->children_tail,
294  chld);
295  return GNUNET_OK;
296 }
297 
298 
309 static int
311  struct ScanTreeNode **dst)
312 {
313  struct ScanTreeNode *item;
314  struct stat sbuf;
315  uint64_t fsize = 0;
316 
317  if ((0 != STAT (filename, &sbuf)) ||
318  ((!S_ISDIR (sbuf.st_mode)) && (GNUNET_OK != GNUNET_DISK_file_size (
319  filename, &fsize, GNUNET_NO, GNUNET_YES))))
320  {
321  /* If the file doesn't exist (or is not stat-able for any other reason)
322  skip it (but report it), but do continue. */
323  if (GNUNET_OK !=
325  filename, strlen (filename) + 1))
326  return GNUNET_SYSERR;
327  /* recoverable error, store 'NULL' in *dst */
328  *dst = NULL;
329  return GNUNET_OK;
330  }
331 
332  /* Report the progress */
333  if (GNUNET_OK !=
334  write_message (S_ISDIR (sbuf.st_mode)
337  filename, strlen (filename) + 1))
338  return GNUNET_SYSERR;
339  item = GNUNET_new (struct ScanTreeNode);
340  item->filename = GNUNET_strdup (filename);
341  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
342  item->file_size = fsize;
343  if (GNUNET_YES == item->is_directory)
344  {
345  struct RecursionContext rc;
346 
347  rc.parent = item;
348  rc.stop = GNUNET_NO;
349  GNUNET_DISK_directory_scan (filename,
350  &scan_callback,
351  &rc);
352  if ( (GNUNET_YES == rc.stop) ||
353  (GNUNET_OK !=
355  "..", 3)) )
356  {
357  free_tree (item);
358  return GNUNET_SYSERR;
359  }
360  }
361  *dst = item;
362  return GNUNET_OK;
363 }
364 
365 
372 static int
374 {
376  ssize_t size;
377  size_t slen;
378 
379  if (GNUNET_YES == item->is_directory)
380  {
381  /* for directories, we simply only descent, no extraction, no
382  progress reporting */
383  struct ScanTreeNode *pos;
384 
385  for (pos = item->children_head; NULL != pos; pos = pos->next)
386  if (GNUNET_OK !=
387  extract_files (pos))
388  return GNUNET_SYSERR;
389  return GNUNET_OK;
390  }
391 
392  /* this is the expensive operation, *afterwards* we'll check for aborts */
394 #if HAVE_LIBEXTRACTOR
395  EXTRACTOR_extract (plugins,
396  item->filename,
397  NULL, 0,
398  &add_to_md,
399  meta);
400 #endif
401  slen = strlen (item->filename) + 1;
403  if (-1 == size)
404  {
405  /* no meta data */
407  if (GNUNET_OK !=
409  item->filename, slen))
410  return GNUNET_SYSERR;
411  return GNUNET_OK;
412  }
413  else if (size > (UINT16_MAX - sizeof (struct GNUNET_MessageHeader) - slen))
414  {
415  /* We can't transfer more than 64k bytes in one message. */
416  size = UINT16_MAX - sizeof (struct GNUNET_MessageHeader) - slen;
417  }
418  {
419  char buf[size + slen];
420  char *dst = &buf[slen];
421 
422  GNUNET_memcpy (buf, item->filename, slen);
424  &dst, size,
426  if (size < 0)
427  {
428  GNUNET_break (0);
429  size = 0;
430  }
432  if (GNUNET_OK !=
434  buf,
435  slen + size))
436  return GNUNET_SYSERR;
437  }
438  return GNUNET_OK;
439 }
440 
441 
442 #ifndef WINDOWS
443 
446 static void
448 {
449  struct sigaction oldsig;
450  struct sigaction sig;
451 
452  memset (&sig, 0, sizeof (struct sigaction));
453  sig.sa_handler = SIG_IGN;
454  sigemptyset (&sig.sa_mask);
455 #ifdef SA_INTERRUPT
456  sig.sa_flags = SA_INTERRUPT; /* SunOS */
457 #else
458  sig.sa_flags = SA_RESTART;
459 #endif
460  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
461  fprintf (stderr,
462  "Failed to install SIGPIPE handler: %s\n", strerror (errno));
463 }
464 
465 
472 static void
474  int flags)
475 {
476  int z;
477 
478  GNUNET_assert (0 == close (fd));
479  z = open ("/dev/null", flags);
480  GNUNET_assert (-1 != z);
481  if (z == fd)
482  return;
483  GNUNET_break (fd == dup2 (z, fd));
484  GNUNET_assert (0 == close (z));
485 }
486 
487 #endif
488 
489 
500 int
501 main (int argc,
502  char *const *argv)
503 {
504  const char *filename_expanded;
505  const char *ex;
506  struct ScanTreeNode *root;
507 
508 #if WINDOWS
509  /* We're using stdout to communicate binary data back to the parent; use
510  * binary mode.
511  */
512  _setmode (1, _O_BINARY);
513  /* Get utf-8-encoded arguments */
514  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
515  return 5;
516  output_stream = 1; /* stdout */
517 #else
518  ignore_sigpipe ();
519  /* move stdout to some other FD for IPC, bind
520  stdout/stderr to /dev/null */
521  output_stream = dup (1);
522  make_dev_zero (1, O_WRONLY);
523  make_dev_zero (2, O_WRONLY);
524 #endif
525 
526  /* parse command line */
527  if ( (3 != argc) && (2 != argc) )
528  {
529  FPRINTF (stderr,
530  "%s",
531  "gnunet-helper-fs-publish needs exactly one or two arguments\n");
532 #if WINDOWS
533  GNUNET_free ((void*) argv);
534 #endif
535  return 1;
536  }
537  filename_expanded = argv[1];
538  ex = argv[2];
539  if ( (NULL == ex) ||
540  (0 != strcmp (ex, "-")) )
541  {
542 #if HAVE_LIBEXTRACTOR
543  plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
544  if (NULL != ex)
545  plugins = EXTRACTOR_plugin_add_config (plugins, ex,
546  EXTRACTOR_OPTION_DEFAULT_POLICY);
547 #endif
548  }
549 
550  /* scan tree to find out how much work there is to be done */
551  if (GNUNET_OK != preprocess_file (filename_expanded,
552  &root))
553  {
555 #if HAVE_LIBEXTRACTOR
556  EXTRACTOR_plugin_remove_all (plugins);
557 #endif
558 #if WINDOWS
559  GNUNET_free ((void*) argv);
560 #endif
561  return 2;
562  }
563  /* signal that we're done counting files, so that a percentage of
564  progress can now be calculated */
565  if (GNUNET_OK !=
567  {
568 #if HAVE_LIBEXTRACTOR
569  EXTRACTOR_plugin_remove_all (plugins);
570 #endif
571 #if WINDOWS
572  GNUNET_free ((void*) argv);
573 #endif
574  return 3;
575  }
576  if (NULL != root)
577  {
578  if (GNUNET_OK !=
579  extract_files (root))
580  {
582  free_tree (root);
583 #if HAVE_LIBEXTRACTOR
584  EXTRACTOR_plugin_remove_all (plugins);
585 #endif
586 #if WINDOWS
587  GNUNET_free ((void*) argv);
588 #endif
589  return 4;
590  }
591  free_tree (root);
592  }
593  /* enable "clean" shutdown by telling parent that we are done */
595 #if HAVE_LIBEXTRACTOR
596  EXTRACTOR_plugin_remove_all (plugins);
597 #endif
598 #if WINDOWS
599  GNUNET_free ((void*) argv);
600 #endif
601  return 0;
602 }
603 
604 /* end of gnunet-helper-fs-publish.c */
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to &#39;/dev/null&#39;.
EXTRACTOR_MetaFormat
Format in which the extracted meta data is presented.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
struct ScanTreeNode * parent
Parent to add the files to.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define FPRINTF
Definition: plibc.h:683
int GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1521
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int main(int argc, char *const *argv)
Main function of the helper process to extract meta data.
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:81
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:78
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define STAT(p, b)
Definition: plibc.h:663
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:1233
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format...
EXTRACTOR_MetaType
Enumeration defining various sources of keywords.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
Meta data to associate with a file, directory or namespace.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
struct ScanTreeNode * next
This is a doubly-linked list.
#define GNUNET_memcpy(dst, src, n)
struct ScanTreeNode * prev
This is a doubly-linked list.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
uint64_t file_size
Size of the file (if it is a file), in bytes.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char buf[2048]
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
static char * plugin_name
Solver plugin name as string.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
static unsigned int size
Size of the "table".
Definition: peer.c:67
static int output_stream
File descriptor we use for IPC with the parent.
int stop
Flag to set to GNUNET_YES on serious errors.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
int GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:289
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:68
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
A node of a directory tree.
#define GNUNET_log(kind,...)
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Header for all communications.
#define GNUNET_YES
Definition: gnunet_common.h:80
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
uint32_t data
The data value.
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
#define GNUNET_free(ptr)
Wrapper around free.
int is_directory
GNUNET_YES if this is a directory
Closure for the &#39;scan_callback&#39;.