GNUnet  0.11.x
gnunet-helper-fs-publish.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
29 #include "platform.h"
30 #include "gnunet_fs_service.h"
31 
32 
37 {
41  struct ScanTreeNode *next;
42 
46  struct ScanTreeNode *prev;
47 
52 
58 
64 
68  char *filename;
69 
74  uint64_t file_size;
75 
80 };
81 
82 
83 #if HAVE_LIBEXTRACTOR
84 
87 static struct EXTRACTOR_PluginList *plugins;
88 #endif
89 
93 static int output_stream;
94 
95 
96 #if HAVE_LIBEXTRACTOR
97 
114 static int
115 add_to_md (void *cls,
116  const char *plugin_name,
118  enum EXTRACTOR_MetaFormat format,
119  const char *data_mime_type,
120  const char *data,
121  size_t data_len)
122 {
123  struct GNUNET_CONTAINER_MetaData *md = cls;
124 
125  if (((EXTRACTOR_METAFORMAT_UTF8 == format) ||
126  (EXTRACTOR_METAFORMAT_C_STRING == format)) &&
127  ('\0' != data[data_len - 1]))
128  {
129  char zdata[data_len + 1];
130  GNUNET_memcpy (zdata, data, data_len);
131  zdata[data_len] = '\0';
133  plugin_name,
134  type,
135  format,
136  data_mime_type,
137  zdata,
138  data_len + 1);
139  }
140  else
141  {
143  plugin_name,
144  type,
145  format,
146  data_mime_type,
147  data,
148  data_len);
149  }
150  return 0;
151 }
152 
153 
154 #endif
155 
156 
162 static void
163 free_tree (struct ScanTreeNode *tree)
164 {
165  struct ScanTreeNode *pos;
166 
167  while (NULL != (pos = tree->children_head))
168  free_tree (pos);
169  if (NULL != tree->parent)
171  tree->parent->children_tail,
172  tree);
173  GNUNET_free (tree->filename);
174  GNUNET_free (tree);
175 }
176 
177 
185 static int
186 write_all (const void *buf, size_t size)
187 {
188  const char *cbuf = buf;
189  size_t total;
190  ssize_t wr;
191 
192  total = 0;
193  do
194  {
195  wr = write (output_stream, &cbuf[total], size - total);
196  if (wr > 0)
197  total += wr;
198  }
199  while ((wr > 0) && (total < size));
200  if (wr <= 0)
202  "Failed to write to stdout: %s\n",
203  strerror (errno));
204  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
205 }
206 
207 
216 static int
217 write_message (uint16_t message_type, const char *data, size_t data_length)
218 {
219  struct GNUNET_MessageHeader hdr;
220 
221 #if 0
222  fprintf (stderr,
223  "Helper sends %u-byte message of type %u\n",
224  (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
225  (unsigned int) message_type);
226 #endif
227  hdr.type = htons (message_type);
228  hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
229  if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
230  (GNUNET_OK != write_all (data, data_length)))
231  return GNUNET_SYSERR;
232  return GNUNET_OK;
233 }
234 
235 
246 static int
247 preprocess_file (const char *filename, struct ScanTreeNode **dst);
248 
249 
254 {
259 
263  int stop;
264 };
265 
266 
276 static int
277 scan_callback (void *cls, const char *filename)
278 {
279  struct RecursionContext *rc = cls;
280  struct ScanTreeNode *chld;
281 
282  if (GNUNET_OK != preprocess_file (filename, &chld))
283  {
284  rc->stop = GNUNET_YES;
285  return GNUNET_SYSERR;
286  }
287  if (NULL == chld)
288  return GNUNET_OK;
289  chld->parent = rc->parent;
291  rc->parent->children_tail,
292  chld);
293  return GNUNET_OK;
294 }
295 
296 
307 static int
308 preprocess_file (const char *filename, struct ScanTreeNode **dst)
309 {
310  struct ScanTreeNode *item;
311  struct stat sbuf;
312  uint64_t fsize = 0;
313 
314  if ((0 != stat (filename, &sbuf)) ||
315  ((! S_ISDIR (sbuf.st_mode)) &&
316  (GNUNET_OK !=
317  GNUNET_DISK_file_size (filename, &fsize, GNUNET_NO, GNUNET_YES))))
318  {
319  /* If the file doesn't exist (or is not stat-able for any other reason)
320  skip it (but report it), but do continue. */
321  if (GNUNET_OK !=
323  filename,
324  strlen (filename) + 1))
325  return GNUNET_SYSERR;
326  /* recoverable error, store 'NULL' in *dst */
327  *dst = NULL;
328  return GNUNET_OK;
329  }
330 
331  /* Report the progress */
332  if (
333  GNUNET_OK !=
334  write_message (S_ISDIR (sbuf.st_mode)
337  filename,
338  strlen (filename) + 1))
339  return GNUNET_SYSERR;
340  item = GNUNET_new (struct ScanTreeNode);
341  item->filename = GNUNET_strdup (filename);
342  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
343  item->file_size = fsize;
344  if (GNUNET_YES == item->is_directory)
345  {
346  struct RecursionContext rc;
347 
348  rc.parent = item;
349  rc.stop = GNUNET_NO;
350  GNUNET_DISK_directory_scan (filename, &scan_callback, &rc);
351  if (
352  (GNUNET_YES == rc.stop) ||
353  (GNUNET_OK !=
355  "..",
356  3)))
357  {
358  free_tree (item);
359  return GNUNET_SYSERR;
360  }
361  }
362  *dst = item;
363  return GNUNET_OK;
364 }
365 
366 
373 static int
375 {
377  ssize_t size;
378  size_t slen;
379 
380  if (GNUNET_YES == item->is_directory)
381  {
382  /* for directories, we simply only descent, no extraction, no
383  progress reporting */
384  struct ScanTreeNode *pos;
385 
386  for (pos = item->children_head; NULL != pos; pos = pos->next)
387  if (GNUNET_OK != extract_files (pos))
388  return GNUNET_SYSERR;
389  return GNUNET_OK;
390  }
391 
392  /* this is the expensive operation, *afterwards* we'll check for aborts */
394 #if HAVE_LIBEXTRACTOR
395  EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
396 #endif
397  slen = strlen (item->filename) + 1;
399  if (-1 == size)
400  {
401  /* no meta data */
403  if (GNUNET_OK !=
405  item->filename,
406  slen))
407  return GNUNET_SYSERR;
408  return GNUNET_OK;
409  }
410  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
411  {
412  /* We can't transfer more than 64k bytes in one message. */
413  size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
414  }
415  {
416  char buf[size + slen];
417  char *dst = &buf[slen];
418 
419  GNUNET_memcpy (buf, item->filename, slen);
421  meta,
422  &dst,
423  size,
425  if (size < 0)
426  {
427  GNUNET_break (0);
428  size = 0;
429  }
431  if (GNUNET_OK !=
433  buf,
434  slen + size))
435  return GNUNET_SYSERR;
436  }
437  return GNUNET_OK;
438 }
439 
440 
444 static void
446 {
447  struct sigaction oldsig;
448  struct sigaction sig;
449 
450  memset (&sig, 0, sizeof(struct sigaction));
451  sig.sa_handler = SIG_IGN;
452  sigemptyset (&sig.sa_mask);
453 #ifdef SA_INTERRUPT
454  sig.sa_flags = SA_INTERRUPT; /* SunOS */
455 #else
456  sig.sa_flags = SA_RESTART;
457 #endif
458  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
459  fprintf (stderr,
460  "Failed to install SIGPIPE handler: %s\n",
461  strerror (errno));
462 }
463 
464 
471 static void
472 make_dev_zero (int fd, int flags)
473 {
474  int z;
475 
476  GNUNET_assert (0 == close (fd));
477  z = open ("/dev/null", flags);
478  GNUNET_assert (-1 != z);
479  if (z == fd)
480  return;
481  GNUNET_break (fd == dup2 (z, fd));
482  GNUNET_assert (0 == close (z));
483 }
484 
485 
496 int
497 main (int argc, char *const *argv)
498 {
499  const char *filename_expanded;
500  const char *ex;
501  struct ScanTreeNode *root;
502 
503  ignore_sigpipe ();
504  /* move stdout to some other FD for IPC, bind
505  stdout/stderr to /dev/null */
506  output_stream = dup (1);
507  make_dev_zero (1, O_WRONLY);
508  make_dev_zero (2, O_WRONLY);
509 
510  /* parse command line */
511  if ((3 != argc) && (2 != argc))
512  {
513  fprintf (stderr,
514  "%s",
515  "gnunet-helper-fs-publish needs exactly one or two arguments\n");
516  return 1;
517  }
518  filename_expanded = argv[1];
519  ex = argv[2];
520  if ((NULL == ex) || (0 != strcmp (ex, "-")))
521  {
522 #if HAVE_LIBEXTRACTOR
523  plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
524  if (NULL != ex)
525  plugins = EXTRACTOR_plugin_add_config (plugins,
526  ex,
527  EXTRACTOR_OPTION_DEFAULT_POLICY);
528 #endif
529  }
530 
531  /* scan tree to find out how much work there is to be done */
532  if (GNUNET_OK != preprocess_file (filename_expanded, &root))
533  {
535 #if HAVE_LIBEXTRACTOR
536  EXTRACTOR_plugin_remove_all (plugins);
537 #endif
538  return 2;
539  }
540  /* signal that we're done counting files, so that a percentage of
541  progress can now be calculated */
542  if (GNUNET_OK !=
544  NULL,
545  0))
546  {
547 #if HAVE_LIBEXTRACTOR
548  EXTRACTOR_plugin_remove_all (plugins);
549 #endif
550  return 3;
551  }
552  if (NULL != root)
553  {
554  if (GNUNET_OK != extract_files (root))
555  {
557  NULL,
558  0);
559  free_tree (root);
560 #if HAVE_LIBEXTRACTOR
561  EXTRACTOR_plugin_remove_all (plugins);
562 #endif
563  return 4;
564  }
565  free_tree (root);
566  }
567  /* enable "clean" shutdown by telling parent that we are done */
569  NULL,
570  0);
571 #if HAVE_LIBEXTRACTOR
572  EXTRACTOR_plugin_remove_all (plugins);
573 #endif
574  return 0;
575 }
576 
577 
578 /* end of gnunet-helper-fs-publish.c */
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to &#39;/dev/null&#39;.
EXTRACTOR_MetaFormat
Format in which the extracted meta data is presented.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
struct ScanTreeNode * parent
Parent to add the files to.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int main(int argc, char *const *argv)
Main function of the helper process to extract meta data.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_new(type)
Allocate a struct or union of the given type.
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:912
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format...
EXTRACTOR_MetaType
Enumeration defining various sources of keywords.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
Meta data to associate with a file, directory or namespace.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
struct ScanTreeNode * next
This is a doubly-linked list.
struct ScanTreeNode * prev
This is a doubly-linked list.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
uint64_t file_size
Size of the file (if it is a file), in bytes.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char buf[2048]
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:67
static int output_stream
File descriptor we use for IPC with the parent.
int stop
Flag to set to GNUNET_YES on serious errors.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
int GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:257
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:68
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
A node of a directory tree.
#define GNUNET_log(kind,...)
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Header for all communications.
#define GNUNET_YES
Definition: gnunet_common.h:77
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
uint32_t data
The data value.
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static char * plugin_name
Name of our plugin.
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
#define GNUNET_free(ptr)
Wrapper around free.
int is_directory
GNUNET_YES if this is a directory
Closure for the &#39;scan_callback&#39;.