GNUnet  0.20.0
gnunet-helper-fs-publish.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
29 #include "platform.h"
30 
31 #include "gnunet_fs_service.h"
32 
33 
38 {
42  struct ScanTreeNode *next;
43 
47  struct ScanTreeNode *prev;
48 
53 
59 
65 
69  char *filename;
70 
75  uint64_t file_size;
76 
81 };
82 
83 
84 #if HAVE_LIBEXTRACTOR
88 static struct EXTRACTOR_PluginList *plugins;
89 #endif
90 
94 static int output_stream;
95 
96 
97 #if HAVE_LIBEXTRACTOR
115 static int
116 add_to_md (void *cls,
117  const char *plugin_name,
118  enum EXTRACTOR_MetaType type,
119  enum EXTRACTOR_MetaFormat format,
120  const char *data_mime_type,
121  const char *data,
122  size_t data_len)
123 {
124  struct GNUNET_FS_MetaData *md = cls;
125 
126  if (((EXTRACTOR_METAFORMAT_UTF8 == format) ||
127  (EXTRACTOR_METAFORMAT_C_STRING == format)) &&
128  ('\0' != data[data_len - 1]))
129  {
130  char zdata[data_len + 1];
131  GNUNET_memcpy (zdata, data, data_len);
132  zdata[data_len] = '\0';
133  (void) GNUNET_FS_meta_data_insert (md,
134  plugin_name,
135  type,
136  format,
137  data_mime_type,
138  zdata,
139  data_len + 1);
140  }
141  else
142  {
143  (void) GNUNET_FS_meta_data_insert (md,
144  plugin_name,
145  type,
146  format,
147  data_mime_type,
148  data,
149  data_len);
150  }
151  return 0;
152 }
153 
154 
155 #endif
156 
157 
163 static void
164 free_tree (struct ScanTreeNode *tree)
165 {
166  struct ScanTreeNode *pos;
167 
168  while (NULL != (pos = tree->children_head))
169  free_tree (pos);
170  if (NULL != tree->parent)
172  tree->parent->children_tail,
173  tree);
174  GNUNET_free (tree->filename);
175  GNUNET_free (tree);
176 }
177 
178 
186 static int
187 write_all (const void *buf, size_t size)
188 {
189  const char *cbuf = buf;
190  size_t total;
191  ssize_t wr;
192 
193  total = 0;
194  do
195  {
196  wr = write (output_stream, &cbuf[total], size - total);
197  if (wr > 0)
198  total += wr;
199  }
200  while ((wr > 0) && (total < size));
201  if (wr <= 0)
203  "Failed to write to stdout: %s\n",
204  strerror (errno));
205  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
206 }
207 
208 
217 static int
218 write_message (uint16_t message_type, const char *data, size_t data_length)
219 {
220  struct GNUNET_MessageHeader hdr;
221 
222 #if 0
223  fprintf (stderr,
224  "Helper sends %u-byte message of type %u\n",
225  (unsigned int) (sizeof(struct GNUNET_MessageHeader) + data_length),
226  (unsigned int) message_type);
227 #endif
228  hdr.type = htons (message_type);
229  hdr.size = htons (sizeof(struct GNUNET_MessageHeader) + data_length);
230  if ((GNUNET_OK != write_all (&hdr, sizeof(hdr))) ||
231  (GNUNET_OK != write_all (data, data_length)))
232  return GNUNET_SYSERR;
233  return GNUNET_OK;
234 }
235 
236 
247 static int
248 preprocess_file (const char *filename, struct ScanTreeNode **dst);
249 
250 
255 {
260 
264  int stop;
265 };
266 
267 
277 static int
278 scan_callback (void *cls, const char *filename)
279 {
280  struct RecursionContext *rc = cls;
281  struct ScanTreeNode *chld;
282 
283  if (GNUNET_OK != preprocess_file (filename, &chld))
284  {
285  rc->stop = GNUNET_YES;
286  return GNUNET_SYSERR;
287  }
288  if (NULL == chld)
289  return GNUNET_OK;
290  chld->parent = rc->parent;
292  rc->parent->children_tail,
293  chld);
294  return GNUNET_OK;
295 }
296 
297 
308 static int
309 preprocess_file (const char *filename, struct ScanTreeNode **dst)
310 {
311  struct ScanTreeNode *item;
312  struct stat sbuf;
313  uint64_t fsize = 0;
314 
315  if ((0 != stat (filename, &sbuf)) ||
316  ((! S_ISDIR (sbuf.st_mode)) &&
317  (GNUNET_OK !=
319  {
320  /* If the file doesn't exist (or is not stat-able for any other reason)
321  skip it (but report it), but do continue. */
322  if (GNUNET_OK !=
324  filename,
325  strlen (filename) + 1))
326  return GNUNET_SYSERR;
327  /* recoverable error, store 'NULL' in *dst */
328  *dst = NULL;
329  return GNUNET_OK;
330  }
331 
332  /* Report the progress */
333  if (
334  GNUNET_OK !=
335  write_message (S_ISDIR (sbuf.st_mode)
338  filename,
339  strlen (filename) + 1))
340  return GNUNET_SYSERR;
341  item = GNUNET_new (struct ScanTreeNode);
342  item->filename = GNUNET_strdup (filename);
343  item->is_directory = (S_ISDIR (sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
344  item->file_size = fsize;
345  if (GNUNET_YES == item->is_directory)
346  {
347  struct RecursionContext rc;
348 
349  rc.parent = item;
350  rc.stop = GNUNET_NO;
352  if (
353  (GNUNET_YES == rc.stop) ||
354  (GNUNET_OK !=
356  "..",
357  3)))
358  {
359  free_tree (item);
360  return GNUNET_SYSERR;
361  }
362  }
363  *dst = item;
364  return GNUNET_OK;
365 }
366 
367 
374 static int
376 {
377  struct GNUNET_FS_MetaData *meta;
378  ssize_t size;
379  size_t slen;
380 
381  if (GNUNET_YES == item->is_directory)
382  {
383  /* for directories, we simply only descent, no extraction, no
384  progress reporting */
385  struct ScanTreeNode *pos;
386 
387  for (pos = item->children_head; NULL != pos; pos = pos->next)
388  if (GNUNET_OK != extract_files (pos))
389  return GNUNET_SYSERR;
390  return GNUNET_OK;
391  }
392 
393  /* this is the expensive operation, *afterwards* we'll check for aborts */
395 #if HAVE_LIBEXTRACTOR
396  EXTRACTOR_extract (plugins, item->filename, NULL, 0, &add_to_md, meta);
397 #endif
398  slen = strlen (item->filename) + 1;
400  if (-1 == size)
401  {
402  /* no meta data */
404  if (GNUNET_OK !=
406  item->filename,
407  slen))
408  return GNUNET_SYSERR;
409  return GNUNET_OK;
410  }
411  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
412  {
413  /* We can't transfer more than 64k bytes in one message. */
414  size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
415  }
416  {
417  char buf[size + slen];
418  char *dst = &buf[slen];
419 
420  GNUNET_memcpy (buf, item->filename, slen);
422  meta,
423  &dst,
424  size,
426  if (size < 0)
427  {
428  GNUNET_break (0);
429  size = 0;
430  }
432  if (GNUNET_OK !=
434  buf,
435  slen + size))
436  return GNUNET_SYSERR;
437  }
438  return GNUNET_OK;
439 }
440 
441 
445 static void
447 {
448  struct sigaction oldsig;
449  struct sigaction sig;
450 
451  memset (&sig, 0, sizeof(struct sigaction));
452  sig.sa_handler = SIG_IGN;
453  sigemptyset (&sig.sa_mask);
454 #ifdef SA_INTERRUPT
455  sig.sa_flags = SA_INTERRUPT; /* SunOS */
456 #else
457  sig.sa_flags = SA_RESTART;
458 #endif
459  if (0 != sigaction (SIGPIPE, &sig, &oldsig))
460  fprintf (stderr,
461  "Failed to install SIGPIPE handler: %s\n",
462  strerror (errno));
463 }
464 
465 
472 static void
473 make_dev_zero (int fd, int flags)
474 {
475  int z;
476 
477  GNUNET_assert (0 == close (fd));
478  z = open ("/dev/null", flags);
479  GNUNET_assert (-1 != z);
480  if (z == fd)
481  return;
482  GNUNET_break (fd == dup2 (z, fd));
483  GNUNET_assert (0 == close (z));
484 }
485 
486 
497 int
498 main (int argc, char *const *argv)
499 {
500  const char *filename_expanded;
501  const char *ex;
502  struct ScanTreeNode *root;
503 
504  ignore_sigpipe ();
505  /* move stdout to some other FD for IPC, bind
506  stdout/stderr to /dev/null */
507  output_stream = dup (1);
508  make_dev_zero (1, O_WRONLY);
509  make_dev_zero (2, O_WRONLY);
510 
511  /* parse command line */
512  if ((3 != argc) && (2 != argc))
513  {
514  fprintf (stderr,
515  "%s",
516  "gnunet-helper-fs-publish needs exactly one or two arguments\n");
517  return 1;
518  }
519  filename_expanded = argv[1];
520  ex = argv[2];
521  if ((NULL == ex) || (0 != strcmp (ex, "-")))
522  {
523 #if HAVE_LIBEXTRACTOR
524  plugins = EXTRACTOR_plugin_add_defaults (EXTRACTOR_OPTION_DEFAULT_POLICY);
525  if (NULL != ex)
526  plugins = EXTRACTOR_plugin_add_config (plugins,
527  ex,
528  EXTRACTOR_OPTION_DEFAULT_POLICY);
529 #endif
530  }
531 
532  /* scan tree to find out how much work there is to be done */
533  if (GNUNET_OK != preprocess_file (filename_expanded, &root))
534  {
536 #if HAVE_LIBEXTRACTOR
537  EXTRACTOR_plugin_remove_all (plugins);
538 #endif
539  return 2;
540  }
541  /* signal that we're done counting files, so that a percentage of
542  progress can now be calculated */
543  if (GNUNET_OK !=
545  NULL,
546  0))
547  {
548 #if HAVE_LIBEXTRACTOR
549  EXTRACTOR_plugin_remove_all (plugins);
550 #endif
551  return 3;
552  }
553  if (NULL != root)
554  {
555  if (GNUNET_OK != extract_files (root))
556  {
558  NULL,
559  0);
560  free_tree (root);
561 #if HAVE_LIBEXTRACTOR
562  EXTRACTOR_plugin_remove_all (plugins);
563 #endif
564  return 4;
565  }
566  free_tree (root);
567  }
568  /* enable "clean" shutdown by telling parent that we are done */
570  NULL,
571  0);
572 #if HAVE_LIBEXTRACTOR
573  EXTRACTOR_plugin_remove_all (plugins);
574 #endif
575  return 0;
576 }
577 
578 
579 /* end of gnunet-helper-fs-publish.c */
static char * filename
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
static int output_stream
File descriptor we use for IPC with the parent.
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
int main(int argc, char *const *argv)
Main function of the helper process to extract meta data.
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to '/dev/null'.
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
uint32_t data
The data value.
static struct GNUNET_FS_MetaData * meta
Meta-data provided via command-line option.
static char * plugin_name
Name of our plugin.
static char buf[2048]
API for file sharing via GNUnet.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:221
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:814
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
#define GNUNET_log(kind,...)
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
@ GNUNET_ERROR_TYPE_DEBUG
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
ssize_t GNUNET_FS_meta_data_get_serialized_size(const struct GNUNET_FS_MetaData *md)
Get the size of the full meta-data in serialized form.
Definition: meta_data.c:863
ssize_t GNUNET_FS_meta_data_serialize(const struct GNUNET_FS_MetaData *md, char **target, size_t max, enum GNUNET_FS_MetaDataSerializationOptions opt)
Serialize meta-data to target.
Definition: meta_data.c:642
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_create()
Create a fresh struct FS_MetaData token.
Definition: meta_data.c:132
int GNUNET_FS_meta_data_insert(struct GNUNET_FS_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
Definition: meta_data.c:259
void GNUNET_FS_meta_data_destroy(struct GNUNET_FS_MetaData *md)
Free meta data.
Definition: meta_data.c:171
@ GNUNET_FS_META_DATA_SERIALIZE_PART
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
static unsigned int size
Size of the "table".
Definition: peer.c:68
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:69
Meta data to associate with a file, directory or namespace.
Definition: meta_data.c:97
Header for all communications.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format.
Closure for the 'scan_callback'.
struct ScanTreeNode * parent
Parent to add the files to.
int stop
Flag to set to GNUNET_YES on serious errors.
A node of a directory tree.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
struct ScanTreeNode * next
This is a doubly-linked list.
struct ScanTreeNode * prev
This is a doubly-linked list.
uint64_t file_size
Size of the file (if it is a file), in bytes.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
int is_directory
GNUNET_YES if this is a directory
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model