GNUnet 0.21.1
fs_dirmetascan.c File Reference

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'. More...

#include "platform.h"
#include "gnunet_fs_service.h"
#include "gnunet_scheduler_lib.h"
#include <pthread.h>
Include dependency graph for fs_dirmetascan.c:

Go to the source code of this file.

Data Structures

struct  GNUNET_FS_DirScanner
 An opaque structure a pointer to which is returned to the caller to be used to control the scanner. More...
 

Functions

void GNUNET_FS_directory_scan_abort (struct GNUNET_FS_DirScanner *ds)
 Abort the scan. More...
 
struct GNUNET_FS_ShareTreeItemGNUNET_FS_directory_scan_get_result (struct GNUNET_FS_DirScanner *ds)
 Obtain the result of the scan after the scan has signalled completion. More...
 
static struct GNUNET_FS_ShareTreeItemadvance (struct GNUNET_FS_ShareTreeItem *pos)
 Move in the directory from the given position to the next file in DFS traversal. More...
 
static struct GNUNET_FS_ShareTreeItemexpand_tree (struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
 Add another child node to the tree. More...
 
static void finish_scan (void *cls)
 Task run last to shut everything down. More...
 
static int process_helper_msgs (void *cls, const struct GNUNET_MessageHeader *msg)
 Called every time there is data to read from the scanner. More...
 
static void helper_died_cb (void *cls)
 Function called if our helper process died. More...
 
struct GNUNET_FS_DirScannerGNUNET_FS_directory_scan_start (const char *filename, int disable_extractor, const char *ex, GNUNET_FS_DirScannerProgressCallback cb, void *cb_cls)
 Start a directory scanner thread. More...
 

Detailed Description

code to asynchronously build a 'struct GNUNET_FS_ShareTreeItem' from an on-disk directory for publishing; use the 'gnunet-helper-fs-publish'.

Author
LRN
Christian Grothoff

Definition in file fs_dirmetascan.c.

Function Documentation

◆ advance()

static struct GNUNET_FS_ShareTreeItem * advance ( struct GNUNET_FS_ShareTreeItem pos)
static

Move in the directory from the given position to the next file in DFS traversal.

Parameters
poscurrent position
Returns
next file, NULL for none

Definition at line 139 of file fs_dirmetascan.c.

140{
141 int moved;
142
143 GNUNET_assert (NULL != pos);
144 moved = 0; /* must not terminate, even on file, otherwise "normal" */
145 while ((pos->is_directory == GNUNET_YES) || (0 == moved))
146 {
147 if ((moved != -1) && (NULL != pos->children_head))
148 {
149 pos = pos->children_head;
150 moved = 1; /* can terminate if file */
151 continue;
152 }
153 if (NULL != pos->next)
154 {
155 pos = pos->next;
156 moved = 1; /* can terminate if file */
157 continue;
158 }
159 if (NULL != pos->parent)
160 {
161 pos = pos->parent;
162 moved = -1; /* force move to 'next' or 'parent' */
163 continue;
164 }
165 /* no more options, end of traversal */
166 return NULL;
167 }
168 return pos;
169}
@ GNUNET_YES
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
int is_directory
GNUNET_YES if this is a directory
struct GNUNET_FS_ShareTreeItem * parent
This is a doubly-linked tree NULL for top-level entries.
struct GNUNET_FS_ShareTreeItem * next
This is a doubly-linked list.
struct GNUNET_FS_ShareTreeItem * children_head
This is a doubly-linked tree NULL for files and empty directories.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_assert, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::next, and GNUNET_FS_ShareTreeItem::parent.

Referenced by process_helper_msgs().

Here is the caller graph for this function:

◆ expand_tree()

static struct GNUNET_FS_ShareTreeItem * expand_tree ( struct GNUNET_FS_ShareTreeItem parent,
const char *  filename,
int  is_directory 
)
static

Add another child node to the tree.

Parameters
parentparent of the child, NULL for top level
filenamename of the file or directory
is_directoryGNUNET_YES for directories
Returns
new entry that was just created

Definition at line 181 of file fs_dirmetascan.c.

184{
185 struct GNUNET_FS_ShareTreeItem *chld;
186 size_t slen;
187
188 chld = GNUNET_new (struct GNUNET_FS_ShareTreeItem);
189 chld->parent = parent;
192 "%s%s",
194 is_directory == GNUNET_YES ? "/" : "");
195 /* make sure we do not end with '//' */
196 slen = strlen (chld->short_filename);
197 if ((slen >= 2) && (chld->short_filename[slen - 1] == '/') &&
198 (chld->short_filename[slen - 2] == '/'))
199 chld->short_filename[slen - 1] = '\0';
201 if (NULL != parent)
204 chld);
205 return chld;
206}
static char * filename
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
const char * GNUNET_STRINGS_get_short_name(const char *filename)
"man basename" Returns a pointer to a part of filename (allocates nothing)!
Definition: strings.c:640
A node of a directory tree (produced by dirscanner)
char * short_filename
Base name of the file/directory.
char * filename
Name of the file/directory.
struct GNUNET_FS_ShareTreeItem * children_tail
This is a doubly-linked tree NULL for files and empty directories.

References GNUNET_FS_ShareTreeItem::children_head, GNUNET_FS_ShareTreeItem::children_tail, filename, GNUNET_FS_ShareTreeItem::filename, GNUNET_asprintf(), GNUNET_CONTAINER_DLL_insert, GNUNET_new, GNUNET_strdup, GNUNET_STRINGS_get_short_name(), GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::parent, and GNUNET_FS_ShareTreeItem::short_filename.

Referenced by process_helper_msgs().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ finish_scan()

static void finish_scan ( void *  cls)
static

Task run last to shut everything down.

Parameters
clsthe 'struct GNUNET_FS_DirScanner'

Definition at line 215 of file fs_dirmetascan.c.

216{
217 struct GNUNET_FS_DirScanner *ds = cls;
218
219 ds->stop_task = NULL;
220 if (NULL != ds->helper)
221 {
223 ds->helper = NULL;
224 }
226 NULL,
229}
static struct GNUNET_FS_DirScanner * ds
Handle to the directory scanner (for recursive insertions).
@ GNUNET_FS_DIRSCANNER_FINISHED
Last call to the progress function: we have finished scanning the directory.
void GNUNET_HELPER_stop(struct GNUNET_HELPER_Handle *h, int soft_kill)
Kills the helper, closes the pipe, frees the handle and calls wait() on the helper process.
Definition: helper.c:536
@ GNUNET_NO
@ GNUNET_SYSERR
An opaque structure a pointer to which is returned to the caller to be used to control the scanner.
GNUNET_FS_DirScannerProgressCallback progress_callback
The function that will be called every time there's a progress message.
struct GNUNET_SCHEDULER_Task * stop_task
Task scheduled when we are done.
void * progress_callback_cls
A closure for progress_callback.
struct GNUNET_HELPER_Handle * helper
Helper process.

References ds, GNUNET_FS_DIRSCANNER_FINISHED, GNUNET_HELPER_stop(), GNUNET_NO, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by process_helper_msgs().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ process_helper_msgs()

static int process_helper_msgs ( void *  cls,
const struct GNUNET_MessageHeader msg 
)
static

Called every time there is data to read from the scanner.

Calls the scanner progress handler.

Parameters
clsthe closure (directory scanner object)
msgmessage from the helper process
Returns
GNUNET_OK on success, GNUNET_NO to stop further processing (no error) GNUNET_SYSERR to stop further processing with error

Definition at line 243 of file fs_dirmetascan.c.

244{
245 struct GNUNET_FS_DirScanner *ds = cls;
246 const char *filename;
247 size_t left;
248
249#if 0
250 fprintf (stderr,
251 "DMS parses %u-byte message of type %u\n",
252 (unsigned int) ntohs (msg->size),
253 (unsigned int) ntohs (msg->type));
254#endif
255 left = ntohs (msg->size) - sizeof(struct GNUNET_MessageHeader);
256 filename = (const char *) &msg[1];
257 switch (ntohs (msg->type))
258 {
260 if (filename[left - 1] != '\0')
261 {
262 GNUNET_break (0);
263 break;
264 }
266 filename,
267 GNUNET_NO,
269 if (NULL == ds->toplevel)
270 {
272 }
273 else
274 {
275 GNUNET_assert (NULL != ds->pos);
276 (void) expand_tree (ds->pos, filename, GNUNET_NO);
277 }
278 return GNUNET_OK;
279
281 if (filename[left - 1] != '\0')
282 {
283 GNUNET_break (0);
284 break;
285 }
286 if (0 == strcmp ("..", filename))
287 {
288 if (NULL == ds->pos)
289 {
290 GNUNET_break (0);
291 break;
292 }
293 ds->pos = ds->pos->parent;
294 return GNUNET_OK;
295 }
297 filename,
301 if (NULL == ds->toplevel)
302 ds->toplevel = ds->pos;
303 return GNUNET_OK;
304
306 break;
307
309 if ('\0' != filename[left - 1])
310 break;
312 filename,
315 return GNUNET_OK;
316
318 if (0 != left)
319 {
320 GNUNET_break (0);
321 break;
322 }
323 if (NULL == ds->toplevel)
324 break;
326 NULL,
329 ds->pos = ds->toplevel;
330 if (GNUNET_YES == ds->pos->is_directory)
331 ds->pos = advance (ds->pos);
332 return GNUNET_OK;
333
335 size_t nlen;
336 const char *end;
337
338 if (NULL == ds->pos)
339 {
340 GNUNET_break (0);
341 break;
342 }
343 end = memchr (filename, 0, left);
344 if (NULL == end)
345 {
346 GNUNET_break (0);
347 break;
348 }
349 end++;
350 nlen = end - filename;
351 left -= nlen;
352 if (0 != strcmp (filename, ds->pos->filename))
353 {
354 GNUNET_break (0);
355 break;
356 }
358 filename,
361 if (0 < left)
362 {
364 if (NULL == ds->pos->meta)
365 {
366 GNUNET_break (0);
367 break;
368 }
369 /* having full filenames is too dangerous; always make sure we clean them up */
371 EXTRACTOR_METATYPE_FILENAME,
372 NULL,
373 0);
374 /* instead, put in our 'safer' original filename */
376 "<libgnunetfs>",
378 EXTRACTOR_METAFORMAT_UTF8,
379 "text/plain",
381 strlen (ds->pos->short_filename)
382 + 1);
383 }
385 ds->pos->meta);
386 ds->pos = advance (ds->pos);
387 return GNUNET_OK;
388 }
389
391 if (NULL != ds->pos)
392 {
393 GNUNET_break (0);
394 break;
395 }
396 if (0 != left)
397 {
398 GNUNET_break (0);
399 break;
400 }
401 if (NULL == ds->toplevel)
402 break;
404 return GNUNET_OK;
405
406 default:
407 GNUNET_break (0);
408 break;
409 }
411 NULL,
414 return GNUNET_OK;
415}
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
static struct GNUNET_FS_ShareTreeItem * expand_tree(struct GNUNET_FS_ShareTreeItem *parent, const char *filename, int is_directory)
Add another child node to the tree.
static void finish_scan(void *cls)
Task run last to shut everything down.
static struct GNUNET_FS_ShareTreeItem * advance(struct GNUNET_FS_ShareTreeItem *pos)
Move in the directory from the given position to the next file in DFS traversal.
static int end
Set if we are to shutdown all services (including ARM).
Definition: gnunet-arm.c:34
#define EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
struct GNUNET_FS_Uri * GNUNET_FS_uri_ksk_create_from_meta_data(const struct GNUNET_FS_MetaData *md)
Construct a keyword-URI from meta-data (take all entries in the meta-data and construct one large key...
Definition: fs_uri.c:1788
@ GNUNET_FS_DIRSCANNER_INTERNAL_ERROR
There was an internal error.
@ GNUNET_FS_DIRSCANNER_FILE_IGNORED
We're having trouble accessing a file (soft-error); it will be ignored.
@ GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED
We've finished extracting meta data from a file.
@ GNUNET_FS_DIRSCANNER_ALL_COUNTED
We've found all files (in the pre-pass).
@ GNUNET_FS_DIRSCANNER_FILE_START
We've started processing a file or directory.
uint16_t type
The type of the message (GNUNET_MESSAGE_TYPE_XXXX), in big-endian format.
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format.
@ GNUNET_OK
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
int GNUNET_FS_meta_data_insert(struct GNUNET_FS_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
Definition: meta_data.c:254
int GNUNET_FS_meta_data_delete(struct GNUNET_FS_MetaData *md, enum EXTRACTOR_MetaType type, const char *data, size_t data_size)
Remove an item.
Definition: meta_data.c:364
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
Definition: meta_data.c:885
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY
Progress information from the helper: found a directory.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED
Signal that helper is done.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA
Extracted meta data from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR
Error signal from the helper.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE
Signal that helper skipped a file.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE
Signal that helper is done scanning the directory tree.
#define GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE
Progress information from the helper: found a file.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1305
struct GNUNET_FS_ShareTreeItem * toplevel
After the scan is finished, it will contain a pointer to the top-level directory entry in the directo...
struct GNUNET_FS_ShareTreeItem * pos
Current position during processing.
struct GNUNET_FS_Uri * ksk_uri
Keywords for this file or directory (derived from metadata).
struct GNUNET_FS_MetaData * meta
Metadata for this file or directory.
Header for all communications.

References advance(), ds, end, expand_tree(), EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME, filename, GNUNET_FS_ShareTreeItem::filename, finish_scan(), GNUNET_assert, GNUNET_break, GNUNET_FS_DIRSCANNER_ALL_COUNTED, GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED, GNUNET_FS_DIRSCANNER_FILE_IGNORED, GNUNET_FS_DIRSCANNER_FILE_START, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_FS_meta_data_delete(), GNUNET_FS_meta_data_deserialize(), GNUNET_FS_meta_data_insert(), GNUNET_FS_uri_ksk_create_from_meta_data(), GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_COUNTING_DONE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_ERROR, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_FINISHED, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_META_DATA, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_DIRECTORY, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_PROGRESS_FILE, GNUNET_MESSAGE_TYPE_FS_PUBLISH_HELPER_SKIP_FILE, GNUNET_NO, GNUNET_OK, GNUNET_SCHEDULER_add_now(), GNUNET_SYSERR, GNUNET_YES, GNUNET_FS_ShareTreeItem::is_directory, GNUNET_FS_ShareTreeItem::ksk_uri, GNUNET_FS_ShareTreeItem::meta, msg, GNUNET_FS_ShareTreeItem::parent, GNUNET_FS_DirScanner::pos, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, GNUNET_FS_ShareTreeItem::short_filename, GNUNET_MessageHeader::size, GNUNET_FS_DirScanner::stop_task, GNUNET_FS_DirScanner::toplevel, and GNUNET_MessageHeader::type.

Referenced by GNUNET_FS_directory_scan_start().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ helper_died_cb()

static void helper_died_cb ( void *  cls)
static

Function called if our helper process died.

Parameters
clsthe 'struct GNUNET_FS_DirScanner' callback.

Definition at line 424 of file fs_dirmetascan.c.

425{
426 struct GNUNET_FS_DirScanner *ds = cls;
427
428 ds->helper = NULL;
429 if (NULL != ds->stop_task)
430 return; /* normal death, was finished */
432 NULL,
435}

References ds, GNUNET_FS_DIRSCANNER_INTERNAL_ERROR, GNUNET_SYSERR, GNUNET_FS_DirScanner::helper, GNUNET_FS_DirScanner::progress_callback, GNUNET_FS_DirScanner::progress_callback_cls, and GNUNET_FS_DirScanner::stop_task.

Referenced by GNUNET_FS_directory_scan_start().

Here is the caller graph for this function: