GNUnet  0.11.x
fs_dirmetascan.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2005-2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
28 #include "platform.h"
29 #include "gnunet_fs_service.h"
30 #include "gnunet_scheduler_lib.h"
31 #include <pthread.h>
32 
33 
39 {
44 
50 
54  char *ex_arg;
55 
61 
66 
73 
78 
83 
87  char *args[4];
88 };
89 
90 
97 void
99 {
100  /* terminate helper */
101  if (NULL != ds->helper)
103 
104  /* free resources */
105  if (NULL != ds->toplevel)
107  if (NULL != ds->stop_task)
109  GNUNET_free (ds->ex_arg);
111  GNUNET_free (ds);
112 }
113 
114 
125 {
127 
128  /* check that we're actually done */
129  GNUNET_assert (NULL == ds->helper);
130  /* preserve result */
131  result = ds->toplevel;
132  ds->toplevel = NULL;
134  return result;
135 }
136 
137 
145 static struct GNUNET_FS_ShareTreeItem *
147 {
148  int moved;
149 
150  GNUNET_assert (NULL != pos);
151  moved = 0; /* must not terminate, even on file, otherwise "normal" */
152  while ((pos->is_directory == GNUNET_YES) || (0 == moved))
153  {
154  if ((moved != -1) && (NULL != pos->children_head))
155  {
156  pos = pos->children_head;
157  moved = 1; /* can terminate if file */
158  continue;
159  }
160  if (NULL != pos->next)
161  {
162  pos = pos->next;
163  moved = 1; /* can terminate if file */
164  continue;
165  }
166  if (NULL != pos->parent)
167  {
168  pos = pos->parent;
169  moved = -1; /* force move to 'next' or 'parent' */
170  continue;
171  }
172  /* no more options, end of traversal */
173  return NULL;
174  }
175  return pos;
176 }
177 
178 
187 static struct GNUNET_FS_ShareTreeItem *
189  const char *filename,
190  int is_directory)
191 {
192  struct GNUNET_FS_ShareTreeItem *chld;
193  size_t slen;
194 
195  chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
196  chld->parent = parent;
197  chld->filename = GNUNET_strdup (filename);
199  "%s%s",
201  is_directory == GNUNET_YES ? "/" : "");
202  /* make sure we do not end with '//' */
203  slen = strlen (chld->short_filename);
204  if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
205  (chld->short_filename[slen - 2] == '/'))
206  chld->short_filename[slen - 1] = '\0';
207  chld->is_directory = is_directory;
208  if (NULL != parent)
211  chld);
212  return chld;
213 }
214 
215 
221 static void
222 finish_scan (void *cls)
223 {
224  struct GNUNET_FS_DirScanner *ds = cls;
225 
226  ds->stop_task = NULL;
227  if (NULL != ds->helper)
228  {
230  ds->helper = NULL;
231  }
233  NULL,
236 }
237 
238 
249 static int
250 process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
251 {
252  struct GNUNET_FS_DirScanner *ds = cls;
253  const char *filename;
254  size_t left;
255 
256 #if 0
257  fprintf (stderr,
258  "DMS parses %u-byte message of type %u\n",
259  (unsigned int) ntohs (msg->size),
260  (unsigned int) ntohs (msg->type));
261 #endif
262  left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
263  filename = (const char *) &msg[1];
264  switch (ntohs (msg->type))
265  {
267  if (filename[left - 1] != '\0')
268  {
269  GNUNET_break (0);
270  break;
271  }
273  filename,
274  GNUNET_NO,
276  if (NULL == ds->toplevel)
277  {
279  }
280  else
281  {
282  GNUNET_assert (NULL != ds->pos);
283  (void) expand_tree (ds->pos, filename, GNUNET_NO);
284  }
285  return GNUNET_OK;
286 
288  if (filename[left - 1] != '\0')
289  {
290  GNUNET_break (0);
291  break;
292  }
293  if (0 == strcmp ("..", filename))
294  {
295  if (NULL == ds->pos)
296  {
297  GNUNET_break (0);
298  break;
299  }
300  ds->pos = ds->pos->parent;
301  return GNUNET_OK;
302  }
304  filename,
305  GNUNET_YES,
308  if (NULL == ds->toplevel)
309  ds->toplevel = ds->pos;
310  return GNUNET_OK;
311 
313  break;
314 
316  if ('\0' != filename[left - 1])
317  break;
319  filename,
322  return GNUNET_OK;
323 
325  if (0 != left)
326  {
327  GNUNET_break (0);
328  break;
329  }
330  if (NULL == ds->toplevel)
331  break;
333  NULL,
336  ds->pos = ds->toplevel;
337  if (GNUNET_YES == ds->pos->is_directory)
338  ds->pos = advance (ds->pos);
339  return GNUNET_OK;
340 
342  size_t nlen;
343  const char *end;
344 
345  if (NULL == ds->pos)
346  {
347  GNUNET_break (0);
348  break;
349  }
350  end = memchr (filename, 0, left);
351  if (NULL == end)
352  {
353  GNUNET_break (0);
354  break;
355  }
356  end++;
357  nlen = end - filename;
358  left -= nlen;
359  if (0 != strcmp (filename, ds->pos->filename))
360  {
361  GNUNET_break (0);
362  break;
363  }
365  filename,
366  GNUNET_YES,
368  if (0 < left)
369  {
371  if (NULL == ds->pos->meta)
372  {
373  GNUNET_break (0);
374  break;
375  }
376  /* having full filenames is too dangerous; always make sure we clean them up */
379  NULL,
380  0);
381  /* instead, put in our 'safer' original filename */
383  "<libgnunetfs>",
386  "text/plain",
388  strlen (ds->pos->short_filename)
389  + 1);
390  }
392  ds->pos->meta);
393  ds->pos = advance (ds->pos);
394  return GNUNET_OK;
395  }
396 
398  if (NULL != ds->pos)
399  {
400  GNUNET_break (0);
401  break;
402  }
403  if (0 != left)
404  {
405  GNUNET_break (0);
406  break;
407  }
408  if (NULL == ds->toplevel)
409  break;
411  return GNUNET_OK;
412 
413  default:
414  GNUNET_break (0);
415  break;
416  }
418  NULL,
421  return GNUNET_OK;
422 }
423 
424 
430 static void
431 helper_died_cb (void *cls)
432 {
433  struct GNUNET_FS_DirScanner *ds = cls;
434 
435  ds->helper = NULL;
436  if (NULL != ds->stop_task)
437  return; /* normal death, was finished */
439  NULL,
442 }
443 
444 
456 struct GNUNET_FS_DirScanner *
458  int disable_extractor,
459  const char *ex,
461  void *cb_cls)
462 {
463  struct stat sbuf;
464  char *filename_expanded;
465  struct GNUNET_FS_DirScanner *ds;
466 
467  if (0 != stat (filename, &sbuf))
468  return NULL;
470  if (NULL == filename_expanded)
471  return NULL;
473  "Starting to scan directory `%s'\n",
476  ds->progress_callback = cb;
477  ds->progress_callback_cls = cb_cls;
479  if (disable_extractor)
480  ds->ex_arg = GNUNET_strdup ("-");
481  else
482  ds->ex_arg = (NULL != ex) ? GNUNET_strdup (ex) : NULL;
483  ds->args[0] = "gnunet-helper-fs-publish";
484  ds->args[1] = ds->filename_expanded;
485  ds->args[2] = ds->ex_arg;
486  ds->args[3] = NULL;
488  "gnunet-helper-fs-publish",
489  ds->args,
492  ds);
493  if (NULL == ds->helper)
494  {
496  GNUNET_free (ds);
497  return NULL;
498  }
499  return ds;
500 }
501 
502 
503 /* end of fs_dirmetascan.c */
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
static struct GNUNET_FS_ShareTreeItem * advance(struct GNUNET_FS_ShareTreeItem *pos)
Move in the directory from the given position to the next file in DFS traversal.
static struct GNUNET_FS_ShareTreeItem * expand_tree(struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
Add another child node to the tree.
static void helper_died_cb(void *cls)
Function called if our helper process died.
static int process_helper_msgs(void *cls, const struct GNUNET_MessageHeader *msg)
Called every time there is data to read from the scanner.
static void finish_scan(void *cls)
Task run last to shut everything down.
static int end
Set if we are to shutdown all services (including ARM).
Definition: gnunet-arm.c:34
static int disable_extractor
Disable extractor option to use for publishing.
static char * filename
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
static int result
Global testing status.
#define GNUNET_log(kind,...)
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
@ EXTRACTOR_METAFORMAT_UTF8
0-terminated, UTF-8 encoded string.
@ EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
@ EXTRACTOR_METATYPE_FILENAME
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
void GNUNET_FS_share_tree_free(struct GNUNET_FS_ShareTreeItem *toplevel)
Release memory of a share item tree.
Definition: fs_sharetree.c:436
struct GNUNET_FS_Uri * GNUNET_FS_uri_ksk_create_from_meta_data(const struct GNUNET_CONTAINER_MetaData *md)
Construct a keyword-URI from meta-data (take all entries in the meta-data and construct one large key...
Definition: fs_uri.c:1781
void GNUNET_FS_directory_scan_abort(struct GNUNET_FS_DirScanner *ds)
Abort the scan.
struct GNUNET_FS_ShareTreeItem * GNUNET_FS_directory_scan_get_result(struct GNUNET_FS_DirScanner *ds)
Obtain the result of the scan after the scan has signalled completion.
void(* GNUNET_FS_DirScannerProgressCallback)(void *cls, const char *filename, int is_directory, enum GNUNET_FS_DirScannerProgressUpdateReason reason)
Function called over time as the directory scanner makes progress on the job at hand.
struct GNUNET_FS_DirScanner * GNUNET_FS_directory_scan_start(const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
Start a directory scanner thread.
@ GNUNET_FS_DIRSCANNER_INTERNAL_ERROR
There was an internal error.
@ GNUNET_FS_DIRSCANNER_FILE_IGNORED
We're having trouble accessing a file (soft-error); it will be ignored.
@ GNUNET_FS_DIRSCANNER_FINISHED
Last call to the progress function: we have finished scanning the directory.
@ GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED
We've finished extracting meta data from a file.
@ GNUNET_FS_DIRSCANNER_ALL_COUNTED
We've found all files (in the pre-pass).
@ GNUNET_FS_DIRSCANNER_FILE_START
We've started processing a file or directory.
struct GNUNET_HELPER_Handle * GNUNET_HELPER_start(int with_control_pipe, const char *binary_name, char *const binary_argv[], GNUNET_MessageTokenizerCallback cb, GNUNET_HELPER_ExceptionCallback exp_cb, void *cb_cls)
Starts a helper and begins reading from it.
Definition: helper.c:491
void GNUNET_HELPER_stop(struct GNUNET_HELPER_Handle *h, int soft_kill)
Kills the helper, closes the pipe, frees the handle and calls wait() on the helper process.
Definition: helper.c:567
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
@ GNUNET_ERROR_TYPE_DEBUG
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
int GNUNET_CONTAINER_meta_data_delete(struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type, const char *data, size_t data_size)
Remove an item.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1296
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:972
char * GNUNET_STRINGS_filename_expand(const char *fil)
Complete filename (a la shell) from abbrevition.
Definition: strings.c:482
const char * GNUNET_STRINGS_get_short_name(const char *filename)
"man basename" Returns a pointer to a part of filename (allocates nothing)!
Definition: strings.c:626
An opaque structure a pointer to which is returned to the caller to be used to control the scanner.
struct GNUNET_FS_ShareTreeItem * toplevel
After the scan is finished, it will contain a pointer to the top-level directory entry in the directo...
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there's a progress message.
struct GNUNET_FS_ShareTreeItem * pos
Current position during processing.
char * args[4]
Arguments for helper.
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
char * ex_arg
Second argument to helper process.
void * progress_callback_cls
A closure for progress_callback.
struct GNUNET_HELPER_Handle * helper
Helper process.
char * filename_expanded
Expanded filename (as given by the scan initiator).
A node of a directory tree (produced by dirscanner)
int is_directory
GNUNET_YES if this is a directory
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_Uri * ksk_uri
Keywords for this file or directory (derived from metadata).
char * short_filename
Base name of the file/directory.
struct GNUNET_FS_ShareTreeItem * next
This is a doubly-linked list.
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.
char * filename
Name of the file/directory.
struct GNUNET_FS_ShareTreeItem * children_tail
This is a doubly-linked tree NULL for files and empty directories.
struct GNUNET_CONTAINER_MetaData * meta
Metadata for this file or directory.
The handle to a helper process.
Definition: helper.c:79
Header for all communications.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format.
Entry in list of pending tasks.
Definition: scheduler.c:135