GNUnet  0.10.x
fs_dirmetascan.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2005-2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
28 #include "platform.h"
29 #include "gnunet_fs_service.h"
30 #include "gnunet_scheduler_lib.h"
31 #include <pthread.h>
32 
33 
43 
49 
53  char *ex_arg;
54 
60 
65 
72 
77 
82 
86  char *args[4];
87 };
88 
89 
96 void
98 {
99  /* terminate helper */
100  if (NULL != ds->helper)
102 
103  /* free resources */
104  if (NULL != ds->toplevel)
106  if (NULL != ds->stop_task)
110  GNUNET_free(ds);
111 }
112 
113 
124 {
126 
127  /* check that we're actually done */
128  GNUNET_assert(NULL == ds->helper);
129  /* preserve result */
130  result = ds->toplevel;
131  ds->toplevel = NULL;
133  return result;
134 }
135 
136 
144 static struct GNUNET_FS_ShareTreeItem *
146 {
147  int moved;
148 
149  GNUNET_assert(NULL != pos);
150  moved = 0; /* must not terminate, even on file, otherwise "normal" */
151  while ((pos->is_directory == GNUNET_YES) || (0 == moved))
152  {
153  if ((moved != -1) && (NULL != pos->children_head))
154  {
155  pos = pos->children_head;
156  moved = 1; /* can terminate if file */
157  continue;
158  }
159  if (NULL != pos->next)
160  {
161  pos = pos->next;
162  moved = 1; /* can terminate if file */
163  continue;
164  }
165  if (NULL != pos->parent)
166  {
167  pos = pos->parent;
168  moved = -1; /* force move to 'next' or 'parent' */
169  continue;
170  }
171  /* no more options, end of traversal */
172  return NULL;
173  }
174  return pos;
175 }
176 
177 
186 static struct GNUNET_FS_ShareTreeItem *
188  const char *filename,
189  int is_directory)
190 {
191  struct GNUNET_FS_ShareTreeItem *chld;
192  size_t slen;
193 
194  chld = GNUNET_new(struct GNUNET_FS_ShareTreeItem);
195  chld->parent = parent;
196  chld->filename = GNUNET_strdup(filename);
198  "%s%s",
200  is_directory == GNUNET_YES ? "/" : "");
201  /* make sure we do not end with '//' */
202  slen = strlen(chld->short_filename);
203  if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
204  (chld->short_filename[slen - 2] == '/'))
205  chld->short_filename[slen - 1] = '\0';
206  chld->is_directory = is_directory;
207  if (NULL != parent)
209  parent->children_tail,
210  chld);
211  return chld;
212 }
213 
214 
220 static void
221 finish_scan(void *cls)
222 {
223  struct GNUNET_FS_DirScanner *ds = cls;
224 
225  ds->stop_task = NULL;
226  if (NULL != ds->helper)
227  {
229  ds->helper = NULL;
230  }
232  NULL,
235 }
236 
237 
248 static int
250 {
251  struct GNUNET_FS_DirScanner *ds = cls;
252  const char *filename;
253  size_t left;
254 
255 #if 0
256  fprintf(stderr,
257  "DMS parses %u-byte message of type %u\n",
258  (unsigned int)ntohs(msg->size),
259  (unsigned int)ntohs(msg->type));
260 #endif
261  left = ntohs(msg->size) - sizeof(struct GNUNET_MessageHeader);
262  filename = (const char *)&msg[1];
263  switch (ntohs(msg->type))
264  {
266  if (filename[left - 1] != '\0')
267  {
268  GNUNET_break(0);
269  break;
270  }
272  filename,
273  GNUNET_NO,
275  if (NULL == ds->toplevel)
276  {
277  ds->toplevel = expand_tree(ds->pos, filename, GNUNET_NO);
278  }
279  else
280  {
281  GNUNET_assert(NULL != ds->pos);
282  (void)expand_tree(ds->pos, filename, GNUNET_NO);
283  }
284  return GNUNET_OK;
285 
287  if (filename[left - 1] != '\0')
288  {
289  GNUNET_break(0);
290  break;
291  }
292  if (0 == strcmp("..", filename))
293  {
294  if (NULL == ds->pos)
295  {
296  GNUNET_break(0);
297  break;
298  }
299  ds->pos = ds->pos->parent;
300  return GNUNET_OK;
301  }
303  filename,
304  GNUNET_YES,
306  ds->pos = expand_tree(ds->pos, filename, GNUNET_YES);
307  if (NULL == ds->toplevel)
308  ds->toplevel = ds->pos;
309  return GNUNET_OK;
310 
312  break;
313 
315  if ('\0' != filename[left - 1])
316  break;
318  filename,
321  return GNUNET_OK;
322 
324  if (0 != left)
325  {
326  GNUNET_break(0);
327  break;
328  }
329  if (NULL == ds->toplevel)
330  break;
332  NULL,
335  ds->pos = ds->toplevel;
336  if (GNUNET_YES == ds->pos->is_directory)
337  ds->pos = advance(ds->pos);
338  return GNUNET_OK;
339 
341  size_t nlen;
342  const char *end;
343 
344  if (NULL == ds->pos)
345  {
346  GNUNET_break(0);
347  break;
348  }
349  end = memchr(filename, 0, left);
350  if (NULL == end)
351  {
352  GNUNET_break(0);
353  break;
354  }
355  end++;
356  nlen = end - filename;
357  left -= nlen;
358  if (0 != strcmp(filename, ds->pos->filename))
359  {
360  GNUNET_break(0);
361  break;
362  }
364  filename,
365  GNUNET_YES,
367  if (0 < left)
368  {
370  if (NULL == ds->pos->meta)
371  {
372  GNUNET_break(0);
373  break;
374  }
375  /* having full filenames is too dangerous; always make sure we clean them up */
378  NULL,
379  0);
380  /* instead, put in our 'safer' original filename */
382  "<libgnunetfs>",
385  "text/plain",
386  ds->pos->short_filename,
387  strlen(ds->pos->short_filename) + 1);
388  }
390  ds->pos = advance(ds->pos);
391  return GNUNET_OK;
392  }
393 
395  if (NULL != ds->pos)
396  {
397  GNUNET_break(0);
398  break;
399  }
400  if (0 != left)
401  {
402  GNUNET_break(0);
403  break;
404  }
405  if (NULL == ds->toplevel)
406  break;
408  return GNUNET_OK;
409 
410  default:
411  GNUNET_break(0);
412  break;
413  }
415  NULL,
418  return GNUNET_OK;
419 }
420 
421 
427 static void
428 helper_died_cb(void *cls)
429 {
430  struct GNUNET_FS_DirScanner *ds = cls;
431 
432  ds->helper = NULL;
433  if (NULL != ds->stop_task)
434  return; /* normal death, was finished */
436  NULL,
439 }
440 
441 
453 struct GNUNET_FS_DirScanner *
455  int disable_extractor,
456  const char *ex,
458  void *cb_cls)
459 {
460  struct stat sbuf;
461  char *filename_expanded;
462  struct GNUNET_FS_DirScanner *ds;
463 
464  if (0 != stat(filename, &sbuf))
465  return NULL;
466  filename_expanded = GNUNET_STRINGS_filename_expand(filename);
467  if (NULL == filename_expanded)
468  return NULL;
470  "Starting to scan directory `%s'\n",
471  filename_expanded);
472  ds = GNUNET_new(struct GNUNET_FS_DirScanner);
473  ds->progress_callback = cb;
474  ds->progress_callback_cls = cb_cls;
476  if (disable_extractor)
477  ds->ex_arg = GNUNET_strdup("-");
478  else
479  ds->ex_arg = (NULL != ex) ? GNUNET_strdup(ex) : NULL;
480  ds->args[0] = "gnunet-helper-fs-publish";
481  ds->args[1] = ds->filename_expanded;
482  ds->args[2] = ds->ex_arg;
483  ds->args[3] = NULL;
485  "gnunet-helper-fs-publish",
486  ds->args,
489  ds);
490  if (NULL == ds->helper)
491  {
492  GNUNET_free(filename_expanded);
493  GNUNET_free(ds);
494  return NULL;
495  }
496  return ds;
497 }
498 
499 
500 /* end of fs_dirmetascan.c */
static void finish_scan(void *cls)
Task run last to shut everything down.
static void helper_died_cb(void *cls)
Function called if our helper process died.
struct GNUNET_HELPER_Handle * helper
Helper process.
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
void(* GNUNET_FS_DirScannerProgressCallback)(void *cls, const char *filename, int is_directory, enum GNUNET_FS_DirScannerProgressUpdateReason reason)
Function called over time as the directory scanner makes progress on the job at hand.
static int end
Set if we are to shutdown all services (including ARM).
Definition: gnunet-arm.c:34
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
struct GNUNET_FS_DirScanner * GNUNET_FS_directory_scan_start(const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
Start a directory scanner thread.
static struct GNUNET_FS_ShareTreeItem * expand_tree(struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
Add another child node to the tree.
The handle to a helper process.
Definition: helper.c:77
struct GNUNET_HELPER_Handle * GNUNET_HELPER_start(int with_control_pipe, const char *binary_name, char *const binary_argv[], GNUNET_MessageTokenizerCallback cb, GNUNET_HELPER_ExceptionCallback exp_cb, void *cb_cls)
Starts a helper and begins reading from it.
Definition: helper.c:487
void * progress_callback_cls
A closure for progress_callback.
char * args[4]
Arguments for helper.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_ShareTreeItem * children_tail
This is a doubly-linked tree NULL for files and empty directories.
void GNUNET_HELPER_stop(struct GNUNET_HELPER_Handle *h, int soft_kill)
Kills the helper, closes the pipe, frees the handle and calls wait() on the helper process...
Definition: helper.c:563
struct GNUNET_FS_Uri * ksk_uri
Keywords for this file or directory (derived from metadata).
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
An opaque structure a pointer to which is returned to the caller to be used to control the scanner...
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
char * filename_expanded
Expanded filename (as given by the scan initiator).
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format...
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
We&#39;ve found all files (in the pre-pass).
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
struct GNUNET_FS_ShareTreeItem * next
This is a doubly-linked list.
int GNUNET_asprintf(char **buf, const char *format,...)
Like asprintf, just portable.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
A node of a directory tree (produced by dirscanner)
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there&#39;s a progress message.
struct GNUNET_FS_ShareTreeItem * pos
Current position during processing.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1264
static int disable_extractor
Disable extractor option to use for publishing.
static char * filename
static int result
Global testing status.
void GNUNET_FS_directory_scan_abort(struct GNUNET_FS_DirScanner *ds)
Abort the scan.
There was an internal error.
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
int GNUNET_CONTAINER_meta_data_delete(struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type, const char *data, size_t data_size)
Remove an item.
char * short_filename
Base name of the file/directory.
const char * GNUNET_STRINGS_get_short_name(const char *filename)
"man basename" Returns a pointer to a part of filename (allocates nothing)!
Definition: strings.c:776
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
char * GNUNET_STRINGS_filename_expand(const char *fil)
Complete filename (a la shell) from abbrevition.
Definition: strings.c:602
Last call to the progress function: we have finished scanning the directory.
static int process_helper_msgs(void *cls, const struct GNUNET_MessageHeader *msg)
Called every time there is data to read from the scanner.
int is_directory
GNUNET_YES if this is a directory
char * filename
Name of the file/directory.
static struct GNUNET_FS_ShareTreeItem * advance(struct GNUNET_FS_ShareTreeItem *pos)
Move in the directory from the given position to the next file in DFS traversal.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
struct GNUNET_CONTAINER_MetaData * meta
Metadata for this file or directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
We&#39;ve started processing a file or directory.
struct GNUNET_FS_Uri * GNUNET_FS_uri_ksk_create_from_meta_data(const struct GNUNET_CONTAINER_MetaData *md)
Construct a keyword-URI from meta-data (take all entries in the meta-data and construct one large key...
Definition: fs_uri.c:1778
void GNUNET_FS_share_tree_free(struct GNUNET_FS_ShareTreeItem *toplevel)
Release memory of a share item tree.
Definition: fs_sharetree.c:423
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
#define GNUNET_log(kind,...)
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
Entry in list of pending tasks.
Definition: scheduler.c:131
We&#39;ve finished extracting meta data from a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
char * ex_arg
Second argument to helper process.
struct GNUNET_FS_ShareTreeItem * toplevel
After the scan is finished, it will contain a pointer to the top-level directory entry in the directo...
Header for all communications.
struct GNUNET_FS_ShareTreeItem * GNUNET_FS_directory_scan_get_result(struct GNUNET_FS_DirScanner *ds)
Obtain the result of the scan after the scan has signalled completion.
#define GNUNET_YES
Definition: gnunet_common.h:77
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
#define GNUNET_free(ptr)
Wrapper around free.
We&#39;re having trouble accessing a file (soft-error); it will be ignored.
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:956