GNUnet  0.20.0
fs_dirmetascan.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2005-2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
28 #include "platform.h"
29 
30 #include "gnunet_fs_service.h"
31 #include "gnunet_scheduler_lib.h"
32 #include <pthread.h>
33 
34 
40 {
45 
51 
55  char *ex_arg;
56 
62 
67 
74 
79 
84 
88  char *args[4];
89 };
90 
91 
98 void
100 {
101  /* terminate helper */
102  if (NULL != ds->helper)
104 
105  /* free resources */
106  if (NULL != ds->toplevel)
108  if (NULL != ds->stop_task)
110  GNUNET_free (ds->ex_arg);
112  GNUNET_free (ds);
113 }
114 
115 
118 {
120 
121  /* check that we're actually done */
122  GNUNET_assert (NULL == ds->helper);
123  /* preserve result */
124  result = ds->toplevel;
125  ds->toplevel = NULL;
127  return result;
128 }
129 
130 
138 static struct GNUNET_FS_ShareTreeItem *
140 {
141  int moved;
142 
143  GNUNET_assert (NULL != pos);
144  moved = 0; /* must not terminate, even on file, otherwise "normal" */
145  while ((pos->is_directory == GNUNET_YES) || (0 == moved))
146  {
147  if ((moved != -1) && (NULL != pos->children_head))
148  {
149  pos = pos->children_head;
150  moved = 1; /* can terminate if file */
151  continue;
152  }
153  if (NULL != pos->next)
154  {
155  pos = pos->next;
156  moved = 1; /* can terminate if file */
157  continue;
158  }
159  if (NULL != pos->parent)
160  {
161  pos = pos->parent;
162  moved = -1; /* force move to 'next' or 'parent' */
163  continue;
164  }
165  /* no more options, end of traversal */
166  return NULL;
167  }
168  return pos;
169 }
170 
171 
180 static struct GNUNET_FS_ShareTreeItem *
182  const char *filename,
183  int is_directory)
184 {
185  struct GNUNET_FS_ShareTreeItem *chld;
186  size_t slen;
187 
188  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
189  chld->parent = parent;
190  chld->filename = GNUNET_strdup (filename);
192  "%s%s",
194  is_directory == GNUNET_YES ? "/" : "");
195  /* make sure we do not end with '//' */
196  slen = strlen (chld->short_filename);
197  if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
198  (chld->short_filename[slen - 2] == '/'))
199  chld->short_filename[slen - 1] = '\0';
200  chld->is_directory = is_directory;
201  if (NULL != parent)
204  chld);
205  return chld;
206 }
207 
208 
214 static void
215 finish_scan (void *cls)
216 {
217  struct GNUNET_FS_DirScanner *ds = cls;
218 
219  ds->stop_task = NULL;
220  if (NULL != ds->helper)
221  {
223  ds->helper = NULL;
224  }
226  NULL,
229 }
230 
231 
242 static int
243 process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
244 {
245  struct GNUNET_FS_DirScanner *ds = cls;
246  const char *filename;
247  size_t left;
248 
249 #if 0
250  fprintf (stderr,
251  "DMS parses %u-byte message of type %u\n",
252  (unsigned int) ntohs (msg->size),
253  (unsigned int) ntohs (msg->type));
254 #endif
255  left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
256  filename = (const char *) &msg[1];
257  switch (ntohs (msg->type))
258  {
260  if (filename[left - 1] != '\0')
261  {
262  GNUNET_break (0);
263  break;
264  }
266  filename,
267  GNUNET_NO,
269  if (NULL == ds->toplevel)
270  {
272  }
273  else
274  {
275  GNUNET_assert (NULL != ds->pos);
276  (void) expand_tree (ds->pos, filename, GNUNET_NO);
277  }
278  return GNUNET_OK;
279 
281  if (filename[left - 1] != '\0')
282  {
283  GNUNET_break (0);
284  break;
285  }
286  if (0 == strcmp ("..", filename))
287  {
288  if (NULL == ds->pos)
289  {
290  GNUNET_break (0);
291  break;
292  }
293  ds->pos = ds->pos->parent;
294  return GNUNET_OK;
295  }
297  filename,
298  GNUNET_YES,
301  if (NULL == ds->toplevel)
302  ds->toplevel = ds->pos;
303  return GNUNET_OK;
304 
306  break;
307 
309  if ('\0' != filename[left - 1])
310  break;
312  filename,
315  return GNUNET_OK;
316 
318  if (0 != left)
319  {
320  GNUNET_break (0);
321  break;
322  }
323  if (NULL == ds->toplevel)
324  break;
326  NULL,
329  ds->pos = ds->toplevel;
330  if (GNUNET_YES == ds->pos->is_directory)
331  ds->pos = advance (ds->pos);
332  return GNUNET_OK;
333 
335  size_t nlen;
336  const char *end;
337 
338  if (NULL == ds->pos)
339  {
340  GNUNET_break (0);
341  break;
342  }
343  end = memchr (filename, 0, left);
344  if (NULL == end)
345  {
346  GNUNET_break (0);
347  break;
348  }
349  end++;
350  nlen = end - filename;
351  left -= nlen;
352  if (0 != strcmp (filename, ds->pos->filename))
353  {
354  GNUNET_break (0);
355  break;
356  }
358  filename,
359  GNUNET_YES,
361  if (0 < left)
362  {
364  if (NULL == ds->pos->meta)
365  {
366  GNUNET_break (0);
367  break;
368  }
369  /* having full filenames is too dangerous; always make sure we clean them up */
371  EXTRACTOR_METATYPE_FILENAME,
372  NULL,
373  0);
374  /* instead, put in our 'safer' original filename */
376  "<libgnunetfs>",
378  EXTRACTOR_METAFORMAT_UTF8,
379  "text/plain",
381  strlen (ds->pos->short_filename)
382  + 1);
383  }
385  ds->pos->meta);
386  ds->pos = advance (ds->pos);
387  return GNUNET_OK;
388  }
389 
391  if (NULL != ds->pos)
392  {
393  GNUNET_break (0);
394  break;
395  }
396  if (0 != left)
397  {
398  GNUNET_break (0);
399  break;
400  }
401  if (NULL == ds->toplevel)
402  break;
404  return GNUNET_OK;
405 
406  default:
407  GNUNET_break (0);
408  break;
409  }
411  NULL,
414  return GNUNET_OK;
415 }
416 
417 
423 static void
424 helper_died_cb (void *cls)
425 {
426  struct GNUNET_FS_DirScanner *ds = cls;
427 
428  ds->helper = NULL;
429  if (NULL != ds->stop_task)
430  return; /* normal death, was finished */
432  NULL,
435 }
436 
437 
449 struct GNUNET_FS_DirScanner *
451  int disable_extractor,
452  const char *ex,
454  void *cb_cls)
455 {
456  struct stat sbuf;
457  char *filename_expanded;
458  struct GNUNET_FS_DirScanner *ds;
459 
460  if (0 != stat (filename, &sbuf))
461  return NULL;
463  if (NULL == filename_expanded)
464  return NULL;
466  "Starting to scan directory `%s'\n",
469  ds->progress_callback = cb;
470  ds->progress_callback_cls = cb_cls;
472  if (disable_extractor)
473  ds->ex_arg = GNUNET_strdup ("-");
474  else
475  ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
476  ds->args[0] = "gnunet-helper-fs-publish";
477  ds->args[1] = ds->filename_expanded;
478  ds->args[2] = ds->ex_arg;
479  ds->args[3] = NULL;
481  "gnunet-helper-fs-publish",
482  ds->args,
485  ds);
486  if (NULL == ds->helper)
487  {
489  GNUNET_free (ds);
490  return NULL;
491  }
492  return ds;
493 }
494 
495 
496 /* end of fs_dirmetascan.c */
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
static struct GNUNET_FS_ShareTreeItem * advance(struct GNUNET_FS_ShareTreeItem *pos)
Move in the directory from the given position to the next file in DFS traversal.
static struct GNUNET_FS_ShareTreeItem * expand_tree(struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
Add another child node to the tree.
static void helper_died_cb(void *cls)
Function called if our helper process died.
static int process_helper_msgs(void *cls, const struct GNUNET_MessageHeader *msg)
Called every time there is data to read from the scanner.
static void finish_scan(void *cls)
Task run last to shut everything down.
static int end
Set if we are to shutdown all services (including ARM).
Definition: gnunet-arm.c:34
static int disable_extractor
Disable extractor option to use for publishing.
static char * filename
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
static int result
Global testing status.
API for file sharing via GNUnet.
API to schedule computations using continuation passing style.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
#define EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
void GNUNET_FS_share_tree_free(struct GNUNET_FS_ShareTreeItem *toplevel)
Release memory of a share item tree.
Definition: fs_sharetree.c:437
struct GNUNET_FS_Uri * GNUNET_FS_uri_ksk_create_from_meta_data(const struct GNUNET_FS_MetaData *md)
Construct a keyword-URI from meta-data (take all entries in the meta-data and construct one large key...
Definition: fs_uri.c:1771
void GNUNET_FS_directory_scan_abort(struct GNUNET_FS_DirScanner *ds)
Abort the scan.
struct GNUNET_FS_ShareTreeItem * GNUNET_FS_directory_scan_get_result(struct GNUNET_FS_DirScanner *ds)
Obtain the result of the scan after the scan has signalled completion.
void(* GNUNET_FS_DirScannerProgressCallback)(void *cls, const char *filename, int is_directory, enum GNUNET_FS_DirScannerProgressUpdateReason reason)
Function called over time as the directory scanner makes progress on the job at hand.
struct GNUNET_FS_DirScanner * GNUNET_FS_directory_scan_start(const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
Start a directory scanner thread.
@ GNUNET_FS_DIRSCANNER_INTERNAL_ERROR
There was an internal error.
@ GNUNET_FS_DIRSCANNER_FILE_IGNORED
We're having trouble accessing a file (soft-error); it will be ignored.
@ GNUNET_FS_DIRSCANNER_FINISHED
Last call to the progress function: we have finished scanning the directory.
@ GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED
We've finished extracting meta data from a file.
@ GNUNET_FS_DIRSCANNER_ALL_COUNTED
We've found all files (in the pre-pass).
@ GNUNET_FS_DIRSCANNER_FILE_START
We've started processing a file or directory.
struct GNUNET_HELPER_Handle * GNUNET_HELPER_start(int with_control_pipe, const char *binary_name, char *const binary_argv[], GNUNET_MessageTokenizerCallback cb, GNUNET_HELPER_ExceptionCallback exp_cb, void *cb_cls)
Starts a helper and begins reading from it.
Definition: helper.c:462
void GNUNET_HELPER_stop(struct GNUNET_HELPER_Handle *h, int soft_kill)
Kills the helper, closes the pipe, frees the handle and calls wait() on the helper process.
Definition: helper.c:538
#define GNUNET_log(kind,...)
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
@ GNUNET_ERROR_TYPE_DEBUG
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
int GNUNET_FS_meta_data_insert(struct GNUNET_FS_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
Definition: meta_data.c:259
int GNUNET_FS_meta_data_delete(struct GNUNET_FS_MetaData *md, enum EXTRACTOR_MetaType type, const char *data, size_t data_size)
Remove an item.
Definition: meta_data.c:369
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
Definition: meta_data.c:890
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1299
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:975
char * GNUNET_STRINGS_filename_expand(const char *fil)
Complete filename (a la shell) from abbrevition.
Definition: strings.c:494
const char * GNUNET_STRINGS_get_short_name(const char *filename)
"man basename" Returns a pointer to a part of filename (allocates nothing)!
Definition: strings.c:639
An opaque structure a pointer to which is returned to the caller to be used to control the scanner.
struct GNUNET_FS_ShareTreeItem * toplevel
After the scan is finished, it will contain a pointer to the top-level directory entry in the directo...
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there's a progress message.
struct GNUNET_FS_ShareTreeItem * pos
Current position during processing.
char * args[4]
Arguments for helper.
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
char * ex_arg
Second argument to helper process.
void * progress_callback_cls
A closure for progress_callback.
struct GNUNET_HELPER_Handle * helper
Helper process.
char * filename_expanded
Expanded filename (as given by the scan initiator).
A node of a directory tree (produced by dirscanner)
int is_directory
GNUNET_YES if this is a directory
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_Uri * ksk_uri
Keywords for this file or directory (derived from metadata).
char * short_filename
Base name of the file/directory.
struct GNUNET_FS_ShareTreeItem * next
This is a doubly-linked list.
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.
struct GNUNET_FS_MetaData * meta
Metadata for this file or directory.
char * filename
Name of the file/directory.
struct GNUNET_FS_ShareTreeItem * children_tail
This is a doubly-linked tree NULL for files and empty directories.
The handle to a helper process.
Definition: helper.c:79
Header for all communications.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format.
Entry in list of pending tasks.
Definition: scheduler.c:136