GNUnet 0.21.1
fs_directory.c
Go to the documentation of this file.
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
35#include "platform.h"
36
37#include "gnunet_fs_service.h"
38#include "fs_api.h"
39
44#define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
45
46
54int
57{
58 char *mime;
59 int ret;
60
61 if (NULL == md)
62 return GNUNET_SYSERR;
64 EXTRACTOR_METATYPE_MIMETYPE);
65 if (NULL == mime)
66 return GNUNET_SYSERR;
67 ret = (0 == strcasecmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES :
69 GNUNET_free (mime);
70 return ret;
71}
72
73
80void
82{
83 char *mime;
84
85 mime =
86 GNUNET_FS_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
87 if (mime != NULL)
88 {
89 GNUNET_break (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME));
90 GNUNET_free (mime);
91 return;
92 }
93 GNUNET_FS_meta_data_insert (md, "<gnunet>",
94 EXTRACTOR_METATYPE_MIMETYPE,
95 EXTRACTOR_METAFORMAT_UTF8, "text/plain",
97 strlen (GNUNET_FS_DIRECTORY_MIME) + 1);
98}
99
100
105{
109 void *data;
110
114 size_t size;
115};
116
117
135static int
136find_full_data (void *cls, const char *plugin_name,
137 enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
138 const char *data_mime_type, const char *data, size_t data_len)
139{
140 struct GetFullDataClosure *gfdc = cls;
141
142 if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
143 {
144 gfdc->size = data_len;
145 if (data_len > 0)
146 {
147 gfdc->data = GNUNET_malloc (data_len);
148 GNUNET_memcpy (gfdc->data, data, data_len);
149 }
150 return 1;
151 }
152 return 0;
153}
154
155
181int
183 const void *data,
184 uint64_t offset,
186 void *dep_cls)
187{
188 struct GetFullDataClosure full_data;
189 const char *cdata = data;
190 char *emsg;
191 uint64_t pos;
192 uint64_t align;
193 uint32_t mdSize;
194 uint64_t epos;
195 struct GNUNET_FS_Uri *uri;
196 struct GNUNET_FS_MetaData *md;
197 char *filename;
198
199 if ((offset == 0) &&
200 ((size < 8 + sizeof(uint32_t)) ||
201 (0 != memcmp (cdata,
203 8))))
204 return GNUNET_SYSERR;
205 pos = offset;
206 if (offset == 0)
207 {
208 GNUNET_memcpy (&mdSize,
209 &cdata[8],
210 sizeof(uint32_t));
211 mdSize = ntohl (mdSize);
212 if (mdSize > size - 8 - sizeof(uint32_t))
213 {
214 /* invalid size */
216 _ ("MAGIC mismatch. This is not a GNUnet directory.\n"));
217 return GNUNET_SYSERR;
218 }
219 md = GNUNET_FS_meta_data_deserialize (&cdata[8 + sizeof(uint32_t)],
220 mdSize);
221 if (md == NULL)
222 {
223 GNUNET_break (0);
224 return GNUNET_SYSERR; /* malformed ! */
225 }
226 dep (dep_cls,
227 NULL,
228 NULL,
229 md,
230 0,
231 NULL);
233 pos = 8 + sizeof(uint32_t) + mdSize;
234 }
235 while (pos < size)
236 {
237 /* find end of URI */
238 if (cdata[pos] == '\0')
239 {
240 /* URI is never empty, must be end of block,
241 * skip to next alignment */
242 align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE;
243 if (align == pos)
244 {
245 /* if we were already aligned, still skip a block! */
246 align += DBLOCK_SIZE;
247 }
248 pos = align;
249 if (pos >= size)
250 {
251 /* malformed - or partial download... */
252 break;
253 }
254 }
255 epos = pos;
256 while ((epos < size) && (cdata[epos] != '\0'))
257 epos++;
258 if (epos >= size)
259 return GNUNET_NO; /* malformed - or partial download */
260
261 uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg);
262 pos = epos + 1;
263 if (NULL == uri)
264 {
265 GNUNET_free (emsg);
266 pos--; /* go back to '\0' to force going to next alignment */
267 continue;
268 }
270 {
272 GNUNET_break (0);
273 return GNUNET_NO; /* illegal in directory! */
274 }
275
276 GNUNET_memcpy (&mdSize,
277 &cdata[pos],
278 sizeof(uint32_t));
279 mdSize = ntohl (mdSize);
280 pos += sizeof(uint32_t);
281 if (pos + mdSize > size)
282 {
284 return GNUNET_NO; /* malformed - or partial download */
285 }
286
287 md = GNUNET_FS_meta_data_deserialize (&cdata[pos],
288 mdSize);
289 if (NULL == md)
290 {
292 GNUNET_break (0);
293 return GNUNET_NO; /* malformed ! */
294 }
295 pos += mdSize;
296 filename =
299 full_data.size = 0;
300 full_data.data = NULL;
303 &full_data);
304 if (NULL != dep)
305 {
306 dep (dep_cls,
307 filename,
308 uri,
309 md,
310 full_data.size,
311 full_data.data);
312 }
313 GNUNET_free (full_data.data);
317 }
318 return GNUNET_OK;
319}
320
321
326{
331
335 size_t len;
336};
337
342{
347
352
356 unsigned int count;
357};
358
359
367 *mdir)
368{
370
372 if (mdir != NULL)
373 ret->meta = GNUNET_FS_meta_data_duplicate (mdir);
374 else
377 return ret;
378}
379
380
391void
393 const struct GNUNET_FS_Uri *uri,
394 const struct GNUNET_FS_MetaData *md,
395 const void *data)
396{
397 struct GNUNET_FS_Uri *curi;
398 struct BuilderEntry *e;
399 uint64_t fsize;
400 uint32_t big;
401 ssize_t ret;
402 size_t mds;
403 size_t mdxs;
404 char *uris;
405 char *serialized;
406 char *sptr;
407 size_t slen;
408 struct GNUNET_FS_MetaData *meta;
409 const struct GNUNET_FS_MetaData *meta_use;
410
412 if (NULL != data)
413 {
416 {
418 }
419 else
420 {
422 GNUNET_assert (NULL != curi);
423 fsize = GNUNET_FS_uri_chk_get_file_size (curi);
425 }
426 }
427 else
428 {
429 fsize = 0; /* not given */
430 }
431 if (fsize > MAX_INLINE_SIZE)
432 fsize = 0; /* too large */
434 slen = strlen (uris) + 1;
436 meta_use = md;
437 meta = NULL;
438 if (fsize > 0)
439 {
441 GNUNET_FS_meta_data_insert (meta, "<gnunet>",
442 EXTRACTOR_METATYPE_GNUNET_FULL_DATA,
443 EXTRACTOR_METAFORMAT_BINARY, NULL, data,
444 fsize);
446 if ((slen + sizeof(uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
447 (slen + sizeof(uint32_t) + mds - 1) / DBLOCK_SIZE)
448 {
449 /* adding full data would not cause us to cross
450 * additional blocks, so add it! */
451 meta_use = meta;
452 mds = mdxs;
453 }
454 }
455
456 if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
458 e = GNUNET_malloc (sizeof(struct BuilderEntry) + slen + mds
459 + sizeof(uint32_t));
460 serialized = (char *) &e[1];
461 GNUNET_memcpy (serialized, uris, slen);
462 GNUNET_free (uris);
463 sptr = &serialized[slen + sizeof(uint32_t)];
464 ret =
465 GNUNET_FS_meta_data_serialize (meta_use, &sptr, mds,
467 if (NULL != meta)
469 if (ret == -1)
470 mds = 0;
471 else
472 mds = ret;
473 big = htonl (mds);
474 GNUNET_memcpy (&serialized[slen], &big, sizeof(uint32_t));
475 e->len = slen + sizeof(uint32_t) + mds;
476 e->next = bld->head;
477 bld->head = e;
478 bld->count++;
479}
480
481
487static size_t
488do_align (size_t start_position, size_t end_position)
489{
490 size_t align;
491
492 align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE;
493 if ((start_position < align) && (end_position > align))
494 return align + end_position - start_position;
495 return end_position;
496}
497
498
508static void
509block_align (size_t start, unsigned int count, const size_t *sizes,
510 unsigned int *perm)
511{
512 unsigned int i;
513 unsigned int j;
514 unsigned int tmp;
515 unsigned int best;
516 ssize_t badness;
517 size_t cpos;
518 size_t cend;
519 ssize_t cbad;
520 unsigned int cval;
521
522 cpos = start;
523 for (i = 0; i < count; i++)
524 {
525 start = cpos;
526 badness = 0x7FFFFFFF;
527 best = -1;
528 for (j = i; j < count; j++)
529 {
530 cval = perm[j];
531 cend = cpos + sizes[cval];
532 if (cpos % DBLOCK_SIZE == 0)
533 {
534 /* prefer placing the largest blocks first */
535 cbad = -(cend % DBLOCK_SIZE);
536 }
537 else
538 {
539 if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE)
540 {
541 /* Data fits into the same block! Prefer small left-overs! */
542 cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE;
543 }
544 else
545 {
546 /* Would have to waste space to re-align, add big factor, this
547 * case is a real loss (proportional to space wasted)! */
548 cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE);
549 }
550 }
551 if (cbad < badness)
552 {
553 best = j;
554 badness = cbad;
555 }
556 }
557 GNUNET_assert (best != -1);
558 tmp = perm[i];
559 perm[i] = perm[best];
560 perm[best] = tmp;
561 cpos += sizes[perm[i]];
562 cpos = do_align (start, cpos);
563 }
564}
565
566
577int
579 size_t *rsize,
580 void **rdata)
581{
582 char *data;
583 char *sptr;
584 size_t *sizes;
585 unsigned int *perm;
586 unsigned int i;
587 unsigned int j;
588 struct BuilderEntry *pos;
589 struct BuilderEntry **bes;
590 size_t size;
591 size_t psize;
592 size_t off;
593 ssize_t ret;
594 uint32_t big;
595
596 size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof(uint32_t);
598 sizes = NULL;
599 perm = NULL;
600 bes = NULL;
601 if (0 < bld->count)
602 {
603 sizes = GNUNET_new_array (bld->count,
604 size_t);
605 perm = GNUNET_new_array (bld->count,
606 unsigned int);
607 bes = GNUNET_new_array (bld->count,
608 struct BuilderEntry *);
609 pos = bld->head;
610 for (i = 0; i < bld->count; i++)
611 {
612 perm[i] = i;
613 bes[i] = pos;
614 sizes[i] = pos->len;
615 pos = pos->next;
616 }
617 block_align (size, bld->count, sizes, perm);
618 /* compute final size with alignment */
619 for (i = 0; i < bld->count; i++)
620 {
621 psize = size;
622 size += sizes[perm[i]];
623 size = do_align (psize, size);
624 }
625 }
626 *rsize = size;
628 if (data == NULL)
629 {
631 "malloc");
632 *rsize = 0;
633 *rdata = NULL;
634 GNUNET_free (sizes);
635 GNUNET_free (perm);
636 GNUNET_free (bes);
637 return GNUNET_SYSERR;
638 }
639 *rdata = data;
642 strlen (GNUNET_DIRECTORY_MAGIC));
643 off = strlen (GNUNET_DIRECTORY_MAGIC);
644
645 sptr = &data[off + sizeof(uint32_t)];
646 ret =
648 &sptr,
649 size - off - sizeof(uint32_t),
651 GNUNET_assert (ret != -1);
652 big = htonl (ret);
653 GNUNET_memcpy (&data[off],
654 &big,
655 sizeof(uint32_t));
656 off += sizeof(uint32_t) + ret;
657 for (j = 0; j < bld->count; j++)
658 {
659 i = perm[j];
660 psize = off;
661 off += sizes[i];
662 off = do_align (psize, off);
663 GNUNET_memcpy (&data[off - sizes[i]], &(bes[i])[1], sizes[i]);
664 GNUNET_free (bes[i]);
665 }
666 GNUNET_free (sizes);
667 GNUNET_free (perm);
668 GNUNET_free (bes);
669 GNUNET_assert (off == size);
671 GNUNET_free (bld);
672 return GNUNET_OK;
673}
674
675
676/* end of fs_directory.c */
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:41
shared definitions for the FS library
#define MAX_INLINE_SIZE
Maximum size for a file to be considered for inlining in a directory.
Definition: fs_api.h:50
static int find_full_data(void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_len)
Type of a function that libextractor calls for each meta data item found.
Definition: fs_directory.c:136
static size_t do_align(size_t start_position, size_t end_position)
Given the start and end position of a block of data, return the end position of that data after align...
Definition: fs_directory.c:488
#define GNUNET_DIRECTORY_MAGIC
String that is used to indicate that a file is a GNUnet directory.
Definition: fs_directory.c:44
static void block_align(size_t start, unsigned int count, const size_t *sizes, unsigned int *perm)
Compute a permutation of the blocks to minimize the cost of alignment.
Definition: fs_directory.c:509
static int start
Set if we are to start default services (including ARM).
Definition: gnunet-arm.c:39
static int ret
Final status code.
Definition: gnunet-arm.c:94
static char * data
The data to insert into the dht.
static char * filename
static uint32_t type
Type string converted to DNS type value.
static struct GNUNET_FS_Uri * uri
Value of URI provided on command-line (when not publishing a file but just creating UBlocks to refer ...
static struct GNUNET_FS_MetaData * meta
Meta-data provided via command-line option.
static char * plugin_name
Name of our plugin.
API for file sharing via GNUnet.
#define EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
void GNUNET_FS_meta_data_make_directory(struct GNUNET_FS_MetaData *md)
Set the MIMETYPE information for the given metadata to "application/gnunet-directory".
Definition: fs_directory.c:81
int GNUNET_FS_directory_list_contents(size_t size, const void *data, uint64_t offset, GNUNET_FS_DirectoryEntryProcessor dep, void *dep_cls)
Iterate over all entries in a directory.
Definition: fs_directory.c:182
int GNUNET_FS_directory_builder_finish(struct GNUNET_FS_DirectoryBuilder *bld, size_t *rsize, void **rdata)
Finish building the directory.
Definition: fs_directory.c:578
uint64_t GNUNET_FS_uri_chk_get_file_size(const struct GNUNET_FS_Uri *uri)
What is the size of the file that this URI refers to?
Definition: fs_uri.c:1360
int GNUNET_FS_uri_test_ksk(const struct GNUNET_FS_Uri *uri)
Is this a keyword URI?
Definition: fs_uri.c:1324
#define GNUNET_FS_DIRECTORY_MAGIC
#define GNUNET_FS_DIRECTORY_MIME
void(* GNUNET_FS_DirectoryEntryProcessor)(void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, const struct GNUNET_FS_MetaData *meta, size_t length, const void *data)
Function used to process entries in a directory.
char * GNUNET_FS_uri_to_string(const struct GNUNET_FS_Uri *uri)
Convert a URI to a UTF-8 String.
Definition: fs_uri.c:2017
void GNUNET_FS_uri_destroy(struct GNUNET_FS_Uri *uri)
Free URI.
Definition: fs_uri.c:677
int GNUNET_FS_uri_test_sks(const struct GNUNET_FS_Uri *uri)
Is this a namespace URI?
Definition: fs_uri.c:1271
struct GNUNET_FS_DirectoryBuilder * GNUNET_FS_directory_builder_create(const struct GNUNET_FS_MetaData *mdir)
Create a directory builder.
Definition: fs_directory.c:366
int GNUNET_FS_meta_data_test_for_directory(const struct GNUNET_FS_MetaData *md)
Does the meta-data claim that this is a directory? Checks if the mime-type is that of a GNUnet direct...
Definition: fs_directory.c:55
int GNUNET_FS_uri_test_chk(const struct GNUNET_FS_Uri *uri)
Is this a file (or directory) URI?
Definition: fs_uri.c:1346
struct GNUNET_FS_Uri * GNUNET_FS_uri_parse(const char *uri, char **emsg)
Convert a UTF-8 String to a URI.
Definition: fs_uri.c:637
struct GNUNET_FS_Uri * GNUNET_FS_uri_loc_get_uri(const struct GNUNET_FS_Uri *uri)
Obtain the URI of the content itself.
Definition: fs_uri.c:843
void GNUNET_FS_directory_builder_add(struct GNUNET_FS_DirectoryBuilder *bld, const struct GNUNET_FS_Uri *uri, const struct GNUNET_FS_MetaData *md, const void *data)
Add an entry to a directory.
Definition: fs_directory.c:392
#define GNUNET_log(kind,...)
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
#define GNUNET_log_strerror(level, cmd)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
#define GNUNET_MAX_MALLOC_CHECKED
Maximum allocation with GNUNET_malloc macro.
#define GNUNET_malloc_large(size)
Wrapper around malloc.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_new_array(n, type)
Allocate a size n array with structs or unions of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
int GNUNET_FS_meta_data_iterate(const struct GNUNET_FS_MetaData *md, EXTRACTOR_MetaDataProcessor iter, void *iter_cls)
Iterate over MD entries.
Definition: meta_data.c:418
char * GNUNET_FS_meta_data_get_by_type(const struct GNUNET_FS_MetaData *md, enum EXTRACTOR_MetaType type)
Get the first MD entry of the given type.
Definition: meta_data.c:438
ssize_t GNUNET_FS_meta_data_get_serialized_size(const struct GNUNET_FS_MetaData *md)
Get the size of the full meta-data in serialized form.
Definition: meta_data.c:858
ssize_t GNUNET_FS_meta_data_serialize(const struct GNUNET_FS_MetaData *md, char **target, size_t max, enum GNUNET_FS_MetaDataSerializationOptions opt)
Serialize meta-data to target.
Definition: meta_data.c:637
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_create(void)
Create a fresh meta data container.
Definition: meta_data.c:127
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_duplicate(const struct GNUNET_FS_MetaData *md)
Duplicate a MetaData token.
Definition: meta_data.c:527
int GNUNET_FS_meta_data_insert(struct GNUNET_FS_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
Definition: meta_data.c:254
void GNUNET_FS_meta_data_destroy(struct GNUNET_FS_MetaData *md)
Free meta data.
Definition: meta_data.c:166
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
Definition: meta_data.c:885
@ GNUNET_FS_META_DATA_SERIALIZE_FULL
Serialize all of the data.
@ GNUNET_FS_META_DATA_SERIALIZE_PART
If not enough space is available, it is acceptable to only serialize some of the metadata.
static unsigned int size
Size of the "table".
Definition: peer.c:68
#define _(String)
GNU gettext support macro.
Definition: platform.h:178
Entries in the directory (builder).
Definition: fs_directory.c:326
size_t len
Length of this entry.
Definition: fs_directory.c:335
struct BuilderEntry * next
This is a linked list.
Definition: fs_directory.c:330
Internal state of a directory builder.
Definition: fs_directory.c:342
struct GNUNET_FS_MetaData * meta
Meta-data for the directory itself.
Definition: fs_directory.c:346
struct BuilderEntry * head
Head of linked list of entries.
Definition: fs_directory.c:351
unsigned int count
Number of entries in the directory.
Definition: fs_directory.c:356
Meta data to associate with a file, directory or namespace.
Definition: meta_data.c:92
A Universal Resource Identifier (URI), opaque.
Definition: fs_api.h:167
Closure for 'find_full_data'.
Definition: fs_directory.c:105
size_t size
Number of bytes stored in data.
Definition: fs_directory.c:114
void * data
Extracted binary meta data.
Definition: fs_directory.c:109