GNUnet  0.10.x
gnunet-helper-fs-publish.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
29 #include "platform.h"
30 #include "gnunet_fs_service.h"
31 
32 
36 struct ScanTreeNode {
40  struct ScanTreeNode *next;
41 
45  struct ScanTreeNode *prev;
46 
51 
57 
63 
67  char *filename;
68 
73  uint64_t file_size;
74 
79 };
80 
81 
82 #if HAVE_LIBEXTRACTOR
83 
86 static struct EXTRACTOR_PluginList *plugins;
87 #endif
88 
92 static int output_stream;
93 
94 
95 #if HAVE_LIBEXTRACTOR
96 
113 static int
114 add_to_md(void *cls,
115  const char *plugin_name,
117  enum EXTRACTOR_MetaFormat format,
118  const char *data_mime_type,
119  const char *data,
120  size_t data_len)
121 {
122  struct GNUNET_CONTAINER_MetaData *md = cls;
123 
124  if (((EXTRACTOR_METAFORMAT_UTF8 == format) ||
125  (EXTRACTOR_METAFORMAT_C_STRING == format)) &&
126  ('\0' != data[data_len - 1]))
127  {
128  char zdata[data_len + 1];
129  GNUNET_memcpy(zdata, data, data_len);
130  zdata[data_len] = '\0';
132  plugin_name,
133  type,
134  format,
135  data_mime_type,
136  zdata,
137  data_len + 1);
138  }
139  else
140  {
142  plugin_name,
143  type,
144  format,
145  data_mime_type,
146  data,
147  data_len);
148  }
149  return 0;
150 }
151 #endif
152 
153 
159 static void
160 free_tree(struct ScanTreeNode *tree)
161 {
162  struct ScanTreeNode *pos;
163 
164  while (NULL != (pos = tree->children_head))
165  free_tree(pos);
166  if (NULL != tree->parent)
168  tree->parent->children_tail,
169  tree);
170  GNUNET_free(tree->filename);
171  GNUNET_free(tree);
172 }
173 
174 
182 static int
183 write_all(const void *buf, size_t size)
184 {
185  const char *cbuf = buf;
186  size_t total;
187  ssize_t wr;
188 
189  total = 0;
190  do
191  {
192  wr = write(output_stream, &cbuf[total], size - total);
193  if (wr > 0)
194  total += wr;
195  }
196  while ((wr > 0) && (total < size));
197  if (wr <= 0)
199  "Failed to write to stdout: %s\n",
200  strerror(errno));
201  return (total == size) ? GNUNET_OK : GNUNET_SYSERR;
202 }
203 
204 
213 static int
214 write_message(uint16_t message_type, const char *data, size_t data_length)
215 {
216  struct GNUNET_MessageHeader hdr;
217 
218 #if 0
219  fprintf(stderr,
220  "Helper sends %u-byte message of type %u\n",
221  (unsigned int)(sizeof(struct GNUNET_MessageHeader) + data_length),
222  (unsigned int)message_type);
223 #endif
224  hdr.type = htons(message_type);
225  hdr.size = htons(sizeof(struct GNUNET_MessageHeader) + data_length);
226  if ((GNUNET_OK != write_all(&hdr, sizeof(hdr))) ||
227  (GNUNET_OK != write_all(data, data_length)))
228  return GNUNET_SYSERR;
229  return GNUNET_OK;
230 }
231 
232 
243 static int
244 preprocess_file(const char *filename, struct ScanTreeNode **dst);
245 
246 
255 
259  int stop;
260 };
261 
262 
272 static int
273 scan_callback(void *cls, const char *filename)
274 {
275  struct RecursionContext *rc = cls;
276  struct ScanTreeNode *chld;
277 
278  if (GNUNET_OK != preprocess_file(filename, &chld))
279  {
280  rc->stop = GNUNET_YES;
281  return GNUNET_SYSERR;
282  }
283  if (NULL == chld)
284  return GNUNET_OK;
285  chld->parent = rc->parent;
287  rc->parent->children_tail,
288  chld);
289  return GNUNET_OK;
290 }
291 
292 
303 static int
304 preprocess_file(const char *filename, struct ScanTreeNode **dst)
305 {
306  struct ScanTreeNode *item;
307  struct stat sbuf;
308  uint64_t fsize = 0;
309 
310  if ((0 != stat(filename, &sbuf)) ||
311  ((!S_ISDIR(sbuf.st_mode)) &&
312  (GNUNET_OK !=
313  GNUNET_DISK_file_size(filename, &fsize, GNUNET_NO, GNUNET_YES))))
314  {
315  /* If the file doesn't exist (or is not stat-able for any other reason)
316  skip it (but report it), but do continue. */
317  if (GNUNET_OK !=
319  filename,
320  strlen(filename) + 1))
321  return GNUNET_SYSERR;
322  /* recoverable error, store 'NULL' in *dst */
323  *dst = NULL;
324  return GNUNET_OK;
325  }
326 
327  /* Report the progress */
328  if (
329  GNUNET_OK !=
330  write_message(S_ISDIR(sbuf.st_mode)
333  filename,
334  strlen(filename) + 1))
335  return GNUNET_SYSERR;
336  item = GNUNET_new(struct ScanTreeNode);
337  item->filename = GNUNET_strdup(filename);
338  item->is_directory = (S_ISDIR(sbuf.st_mode)) ? GNUNET_YES : GNUNET_NO;
339  item->file_size = fsize;
340  if (GNUNET_YES == item->is_directory)
341  {
342  struct RecursionContext rc;
343 
344  rc.parent = item;
345  rc.stop = GNUNET_NO;
346  GNUNET_DISK_directory_scan(filename, &scan_callback, &rc);
347  if (
348  (GNUNET_YES == rc.stop) ||
349  (GNUNET_OK !=
351  "..",
352  3)))
353  {
354  free_tree(item);
355  return GNUNET_SYSERR;
356  }
357  }
358  *dst = item;
359  return GNUNET_OK;
360 }
361 
362 
369 static int
371 {
373  ssize_t size;
374  size_t slen;
375 
376  if (GNUNET_YES == item->is_directory)
377  {
378  /* for directories, we simply only descent, no extraction, no
379  progress reporting */
380  struct ScanTreeNode *pos;
381 
382  for (pos = item->children_head; NULL != pos; pos = pos->next)
383  if (GNUNET_OK != extract_files(pos))
384  return GNUNET_SYSERR;
385  return GNUNET_OK;
386  }
387 
388  /* this is the expensive operation, *afterwards* we'll check for aborts */
390 #if HAVE_LIBEXTRACTOR
391  EXTRACTOR_extract(plugins, item->filename, NULL, 0, &add_to_md, meta);
392 #endif
393  slen = strlen(item->filename) + 1;
395  if (-1 == size)
396  {
397  /* no meta data */
399  if (GNUNET_OK !=
401  item->filename,
402  slen))
403  return GNUNET_SYSERR;
404  return GNUNET_OK;
405  }
406  else if (size > (UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen))
407  {
408  /* We can't transfer more than 64k bytes in one message. */
409  size = UINT16_MAX - sizeof(struct GNUNET_MessageHeader) - slen;
410  }
411  {
412  char buf[size + slen];
413  char *dst = &buf[slen];
414 
415  GNUNET_memcpy(buf, item->filename, slen);
417  meta,
418  &dst,
419  size,
421  if (size < 0)
422  {
423  GNUNET_break(0);
424  size = 0;
425  }
427  if (GNUNET_OK !=
429  buf,
430  slen + size))
431  return GNUNET_SYSERR;
432  }
433  return GNUNET_OK;
434 }
435 
436 
440 static void
442 {
443  struct sigaction oldsig;
444  struct sigaction sig;
445 
446  memset(&sig, 0, sizeof(struct sigaction));
447  sig.sa_handler = SIG_IGN;
448  sigemptyset(&sig.sa_mask);
449 #ifdef SA_INTERRUPT
450  sig.sa_flags = SA_INTERRUPT; /* SunOS */
451 #else
452  sig.sa_flags = SA_RESTART;
453 #endif
454  if (0 != sigaction(SIGPIPE, &sig, &oldsig))
455  fprintf(stderr,
456  "Failed to install SIGPIPE handler: %s\n",
457  strerror(errno));
458 }
459 
460 
467 static void
468 make_dev_zero(int fd, int flags)
469 {
470  int z;
471 
472  GNUNET_assert(0 == close(fd));
473  z = open("/dev/null", flags);
474  GNUNET_assert(-1 != z);
475  if (z == fd)
476  return;
477  GNUNET_break(fd == dup2(z, fd));
478  GNUNET_assert(0 == close(z));
479 }
480 
481 
492 int
493 main(int argc, char *const *argv)
494 {
495  const char *filename_expanded;
496  const char *ex;
497  struct ScanTreeNode *root;
498 
499  ignore_sigpipe();
500  /* move stdout to some other FD for IPC, bind
501  stdout/stderr to /dev/null */
502  output_stream = dup(1);
503  make_dev_zero(1, O_WRONLY);
504  make_dev_zero(2, O_WRONLY);
505 
506  /* parse command line */
507  if ((3 != argc) && (2 != argc))
508  {
509  fprintf(stderr,
510  "%s",
511  "gnunet-helper-fs-publish needs exactly one or two arguments\n");
512  return 1;
513  }
514  filename_expanded = argv[1];
515  ex = argv[2];
516  if ((NULL == ex) || (0 != strcmp(ex, "-")))
517  {
518 #if HAVE_LIBEXTRACTOR
519  plugins = EXTRACTOR_plugin_add_defaults(EXTRACTOR_OPTION_DEFAULT_POLICY);
520  if (NULL != ex)
521  plugins = EXTRACTOR_plugin_add_config(plugins,
522  ex,
523  EXTRACTOR_OPTION_DEFAULT_POLICY);
524 #endif
525  }
526 
527  /* scan tree to find out how much work there is to be done */
528  if (GNUNET_OK != preprocess_file(filename_expanded, &root))
529  {
531 #if HAVE_LIBEXTRACTOR
532  EXTRACTOR_plugin_remove_all(plugins);
533 #endif
534  return 2;
535  }
536  /* signal that we're done counting files, so that a percentage of
537  progress can now be calculated */
538  if (GNUNET_OK !=
540  NULL,
541  0))
542  {
543 #if HAVE_LIBEXTRACTOR
544  EXTRACTOR_plugin_remove_all(plugins);
545 #endif
546  return 3;
547  }
548  if (NULL != root)
549  {
550  if (GNUNET_OK != extract_files(root))
551  {
553  NULL,
554  0);
555  free_tree(root);
556 #if HAVE_LIBEXTRACTOR
557  EXTRACTOR_plugin_remove_all(plugins);
558 #endif
559  return 4;
560  }
561  free_tree(root);
562  }
563  /* enable "clean" shutdown by telling parent that we are done */
565  NULL,
566  0);
567 #if HAVE_LIBEXTRACTOR
568  EXTRACTOR_plugin_remove_all(plugins);
569 #endif
570  return 0;
571 }
572 
573 /* end of gnunet-helper-fs-publish.c */
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
static void make_dev_zero(int fd, int flags)
Turn the given file descriptor in to &#39;/dev/null&#39;.
EXTRACTOR_MetaFormat
Format in which the extracted meta data is presented.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
struct ScanTreeNode * parent
Parent to add the files to.
struct ScanTreeNode * parent
Parent of this node, NULL for top-level entries.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
If not enough space is available, it is acceptable to only serialize some of the metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int main(int argc, char *const *argv)
Main function of the helper process to extract meta data.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_new(type)
Allocate a struct or union of the given type.
int GNUNET_DISK_directory_scan(const char *dir_name, GNUNET_FileNameCallback callback, void *callback_cls)
Scan a directory for files.
Definition: disk.c:909
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format...
EXTRACTOR_MetaType
Enumeration defining various sources of keywords.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
static int write_all(const void *buf, size_t size)
Write size bytes from buf into the output_stream.
Meta data to associate with a file, directory or namespace.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
struct ScanTreeNode * next
This is a doubly-linked list.
struct ScanTreeNode * prev
This is a doubly-linked list.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
uint64_t file_size
Size of the file (if it is a file), in bytes.
struct ScanTreeNode * children_head
This is a doubly-linked tree NULL for files and empty directories.
static char buf[2048]
static void ignore_sigpipe()
Install a signal handler to ignore SIGPIPE.
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
static char * plugin_name
Solver plugin name as string.
static int extract_files(struct ScanTreeNode *item)
Extract metadata from files.
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
static int write_message(uint16_t message_type, const char *data, size_t data_length)
Write message to the master process.
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:66
static int output_stream
File descriptor we use for IPC with the parent.
int stop
Flag to set to GNUNET_YES on serious errors.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
int GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:254
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
struct ScanTreeNode * children_tail
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
static struct PluginList * plugins
List of plugins we have loaded.
Definition: plugin.c:67
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
A node of a directory tree.
#define GNUNET_log(kind,...)
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Header for all communications.
#define GNUNET_YES
Definition: gnunet_common.h:77
static int preprocess_file(const char *filename, struct ScanTreeNode **dst)
Function called to (recursively) add all of the files in the directory to the tree.
uint32_t data
The data value.
static int scan_callback(void *cls, const char *filename)
Function called by the directory iterator to (recursively) add all of the files in the directory to t...
static void free_tree(struct ScanTreeNode *tree)
Free memory of the tree structure.
#define GNUNET_free(ptr)
Wrapper around free.
int is_directory
GNUNET_YES if this is a directory
Closure for the &#39;scan_callback&#39;.