GNUnet  0.11.x
fs_directory.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
35 #include "platform.h"
36 #include "gnunet_fs_service.h"
37 #include "fs_api.h"
38 
43 #define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
44 
45 
53 int
56 {
57  char *mime;
58  int ret;
59 
60  if (NULL == md)
61  return GNUNET_SYSERR;
64  if (NULL == mime)
65  return GNUNET_SYSERR;
66  ret = (0 == strcasecmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES :
67  GNUNET_NO;
68  GNUNET_free (mime);
69  return ret;
70 }
71 
72 
79 void
81 {
82  char *mime;
83 
84  mime =
86  if (mime != NULL)
87  {
88  GNUNET_break (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME));
89  GNUNET_free (mime);
90  return;
91  }
92  GNUNET_CONTAINER_meta_data_insert (md, "<gnunet>",
94  EXTRACTOR_METAFORMAT_UTF8, "text/plain",
96  strlen (GNUNET_FS_DIRECTORY_MIME) + 1);
97 }
98 
99 
104 {
108  void *data;
109 
113  size_t size;
114 };
115 
116 
134 static int
135 find_full_data (void *cls, const char *plugin_name,
136  enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
137  const char *data_mime_type, const char *data, size_t data_len)
138 {
139  struct GetFullDataClosure *gfdc = cls;
140 
142  {
143  gfdc->size = data_len;
144  if (data_len > 0)
145  {
146  gfdc->data = GNUNET_malloc (data_len);
147  GNUNET_memcpy (gfdc->data, data, data_len);
148  }
149  return 1;
150  }
151  return 0;
152 }
153 
154 
180 int
182  const void *data,
183  uint64_t offset,
185  void *dep_cls)
186 {
187  struct GetFullDataClosure full_data;
188  const char *cdata = data;
189  char *emsg;
190  uint64_t pos;
191  uint64_t align;
192  uint32_t mdSize;
193  uint64_t epos;
194  struct GNUNET_FS_Uri *uri;
195  struct GNUNET_CONTAINER_MetaData *md;
196  char *filename;
197 
198  if ((offset == 0) &&
199  ((size < 8 + sizeof(uint32_t)) ||
200  (0 != memcmp (cdata,
202  8))))
203  return GNUNET_SYSERR;
204  pos = offset;
205  if (offset == 0)
206  {
207  GNUNET_memcpy (&mdSize,
208  &cdata[8],
209  sizeof(uint32_t));
210  mdSize = ntohl (mdSize);
211  if (mdSize > size - 8 - sizeof(uint32_t))
212  {
213  /* invalid size */
215  _ ("MAGIC mismatch. This is not a GNUnet directory.\n"));
216  return GNUNET_SYSERR;
217  }
218  md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[8 + sizeof(uint32_t)],
219  mdSize);
220  if (md == NULL)
221  {
222  GNUNET_break (0);
223  return GNUNET_SYSERR; /* malformed ! */
224  }
225  dep (dep_cls,
226  NULL,
227  NULL,
228  md,
229  0,
230  NULL);
232  pos = 8 + sizeof(uint32_t) + mdSize;
233  }
234  while (pos < size)
235  {
236  /* find end of URI */
237  if (cdata[pos] == '\0')
238  {
239  /* URI is never empty, must be end of block,
240  * skip to next alignment */
241  align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE;
242  if (align == pos)
243  {
244  /* if we were already aligned, still skip a block! */
245  align += DBLOCK_SIZE;
246  }
247  pos = align;
248  if (pos >= size)
249  {
250  /* malformed - or partial download... */
251  break;
252  }
253  }
254  epos = pos;
255  while ((epos < size) && (cdata[epos] != '\0'))
256  epos++;
257  if (epos >= size)
258  return GNUNET_NO; /* malformed - or partial download */
259 
260  uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg);
261  pos = epos + 1;
262  if (NULL == uri)
263  {
264  GNUNET_free (emsg);
265  pos--; /* go back to '\0' to force going to next alignment */
266  continue;
267  }
268  if (GNUNET_FS_uri_test_ksk (uri))
269  {
270  GNUNET_FS_uri_destroy (uri);
271  GNUNET_break (0);
272  return GNUNET_NO; /* illegal in directory! */
273  }
274 
275  GNUNET_memcpy (&mdSize,
276  &cdata[pos],
277  sizeof(uint32_t));
278  mdSize = ntohl (mdSize);
279  pos += sizeof(uint32_t);
280  if (pos + mdSize > size)
281  {
282  GNUNET_FS_uri_destroy (uri);
283  return GNUNET_NO; /* malformed - or partial download */
284  }
285 
286  md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[pos],
287  mdSize);
288  if (NULL == md)
289  {
290  GNUNET_FS_uri_destroy (uri);
291  GNUNET_break (0);
292  return GNUNET_NO; /* malformed ! */
293  }
294  pos += mdSize;
295  filename =
298  full_data.size = 0;
299  full_data.data = NULL;
302  &full_data);
303  if (NULL != dep)
304  {
305  dep (dep_cls,
306  filename,
307  uri,
308  md,
309  full_data.size,
310  full_data.data);
311  }
312  GNUNET_free_non_null (full_data.data);
313  GNUNET_free_non_null (filename);
315  GNUNET_FS_uri_destroy (uri);
316  }
317  return GNUNET_OK;
318 }
319 
320 
325 {
330 
334  size_t len;
335 };
336 
341 {
346 
351 
355  unsigned int count;
356 };
357 
358 
366  *mdir)
367 {
369 
370  ret = GNUNET_new (struct GNUNET_FS_DirectoryBuilder);
371  if (mdir != NULL)
373  else
376  return ret;
377 }
378 
379 
390 void
392  const struct GNUNET_FS_Uri *uri,
393  const struct GNUNET_CONTAINER_MetaData *md,
394  const void *data)
395 {
396  struct GNUNET_FS_Uri *curi;
397  struct BuilderEntry *e;
398  uint64_t fsize;
399  uint32_t big;
400  ssize_t ret;
401  size_t mds;
402  size_t mdxs;
403  char *uris;
404  char *ser;
405  char *sptr;
406  size_t slen;
408  const struct GNUNET_CONTAINER_MetaData *meta_use;
409 
411  if (NULL != data)
412  {
414  if (GNUNET_FS_uri_test_chk (uri))
415  {
416  fsize = GNUNET_FS_uri_chk_get_file_size (uri);
417  }
418  else
419  {
420  curi = GNUNET_FS_uri_loc_get_uri (uri);
421  GNUNET_assert (NULL != curi);
422  fsize = GNUNET_FS_uri_chk_get_file_size (curi);
423  GNUNET_FS_uri_destroy (curi);
424  }
425  }
426  else
427  {
428  fsize = 0; /* not given */
429  }
430  if (fsize > MAX_INLINE_SIZE)
431  fsize = 0; /* too large */
432  uris = GNUNET_FS_uri_to_string (uri);
433  slen = strlen (uris) + 1;
435  meta_use = md;
436  meta = NULL;
437  if (fsize > 0)
438  {
440  GNUNET_CONTAINER_meta_data_insert (meta, "<gnunet>",
442  EXTRACTOR_METAFORMAT_BINARY, NULL, data,
443  fsize);
445  if ((slen + sizeof(uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
446  (slen + sizeof(uint32_t) + mds - 1) / DBLOCK_SIZE)
447  {
448  /* adding full data would not cause us to cross
449  * additional blocks, so add it! */
450  meta_use = meta;
451  mds = mdxs;
452  }
453  }
454 
455  if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
456  mds = GNUNET_MAX_MALLOC_CHECKED / 2;
457  e = GNUNET_malloc (sizeof(struct BuilderEntry) + slen + mds
458  + sizeof(uint32_t));
459  ser = (char *) &e[1];
460  GNUNET_memcpy (ser, uris, slen);
461  GNUNET_free (uris);
462  sptr = &ser[slen + sizeof(uint32_t)];
463  ret =
464  GNUNET_CONTAINER_meta_data_serialize (meta_use, &sptr, mds,
466  if (NULL != meta)
468  if (ret == -1)
469  mds = 0;
470  else
471  mds = ret;
472  big = htonl (mds);
473  GNUNET_memcpy (&ser[slen], &big, sizeof(uint32_t));
474  e->len = slen + sizeof(uint32_t) + mds;
475  e->next = bld->head;
476  bld->head = e;
477  bld->count++;
478 }
479 
480 
486 static size_t
487 do_align (size_t start_position, size_t end_position)
488 {
489  size_t align;
490 
491  align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE;
492  if ((start_position < align) && (end_position > align))
493  return align + end_position - start_position;
494  return end_position;
495 }
496 
497 
507 static void
508 block_align (size_t start, unsigned int count, const size_t *sizes,
509  unsigned int *perm)
510 {
511  unsigned int i;
512  unsigned int j;
513  unsigned int tmp;
514  unsigned int best;
515  ssize_t badness;
516  size_t cpos;
517  size_t cend;
518  ssize_t cbad;
519  unsigned int cval;
520 
521  cpos = start;
522  for (i = 0; i < count; i++)
523  {
524  start = cpos;
525  badness = 0x7FFFFFFF;
526  best = -1;
527  for (j = i; j < count; j++)
528  {
529  cval = perm[j];
530  cend = cpos + sizes[cval];
531  if (cpos % DBLOCK_SIZE == 0)
532  {
533  /* prefer placing the largest blocks first */
534  cbad = -(cend % DBLOCK_SIZE);
535  }
536  else
537  {
538  if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE)
539  {
540  /* Data fits into the same block! Prefer small left-overs! */
541  cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE;
542  }
543  else
544  {
545  /* Would have to waste space to re-align, add big factor, this
546  * case is a real loss (proportional to space wasted)! */
547  cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE);
548  }
549  }
550  if (cbad < badness)
551  {
552  best = j;
553  badness = cbad;
554  }
555  }
556  GNUNET_assert (best != -1);
557  tmp = perm[i];
558  perm[i] = perm[best];
559  perm[best] = tmp;
560  cpos += sizes[perm[i]];
561  cpos = do_align (start, cpos);
562  }
563 }
564 
565 
576 int
578  size_t *rsize,
579  void **rdata)
580 {
581  char *data;
582  char *sptr;
583  size_t *sizes;
584  unsigned int *perm;
585  unsigned int i;
586  unsigned int j;
587  struct BuilderEntry *pos;
588  struct BuilderEntry **bes;
589  size_t size;
590  size_t psize;
591  size_t off;
592  ssize_t ret;
593  uint32_t big;
594 
595  size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof(uint32_t);
597  sizes = NULL;
598  perm = NULL;
599  bes = NULL;
600  if (0 < bld->count)
601  {
602  sizes = GNUNET_new_array (bld->count,
603  size_t);
604  perm = GNUNET_new_array (bld->count,
605  unsigned int);
606  bes = GNUNET_new_array (bld->count,
607  struct BuilderEntry *);
608  pos = bld->head;
609  for (i = 0; i < bld->count; i++)
610  {
611  perm[i] = i;
612  bes[i] = pos;
613  sizes[i] = pos->len;
614  pos = pos->next;
615  }
616  block_align (size, bld->count, sizes, perm);
617  /* compute final size with alignment */
618  for (i = 0; i < bld->count; i++)
619  {
620  psize = size;
621  size += sizes[perm[i]];
622  size = do_align (psize, size);
623  }
624  }
625  *rsize = size;
626  data = GNUNET_malloc_large (size);
627  if (data == NULL)
628  {
630  "malloc");
631  *rsize = 0;
632  *rdata = NULL;
633  GNUNET_free_non_null (sizes);
634  GNUNET_free_non_null (perm);
635  GNUNET_free_non_null (bes);
636  return GNUNET_SYSERR;
637  }
638  *rdata = data;
639  GNUNET_memcpy (data,
641  strlen (GNUNET_DIRECTORY_MAGIC));
642  off = strlen (GNUNET_DIRECTORY_MAGIC);
643 
644  sptr = &data[off + sizeof(uint32_t)];
645  ret =
647  &sptr,
648  size - off - sizeof(uint32_t),
650  GNUNET_assert (ret != -1);
651  big = htonl (ret);
652  GNUNET_memcpy (&data[off],
653  &big,
654  sizeof(uint32_t));
655  off += sizeof(uint32_t) + ret;
656  for (j = 0; j < bld->count; j++)
657  {
658  i = perm[j];
659  psize = off;
660  off += sizes[i];
661  off = do_align (psize, off);
662  GNUNET_memcpy (&data[off - sizes[i]], &(bes[i])[1], sizes[i]);
663  GNUNET_free (bes[i]);
664  }
665  GNUNET_free_non_null (sizes);
666  GNUNET_free_non_null (perm);
667  GNUNET_free_non_null (bes);
668  GNUNET_assert (off == size);
670  GNUNET_free (bld);
671  return GNUNET_OK;
672 }
673 
674 
675 /* end of fs_directory.c */
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
int GNUNET_FS_uri_test_ksk(const struct GNUNET_FS_Uri *uri)
Is this a keyword URI?
Definition: fs_uri.c:1335
int GNUNET_FS_directory_builder_finish(struct GNUNET_FS_DirectoryBuilder *bld, size_t *rsize, void **rdata)
Finish building the directory.
Definition: fs_directory.c:577
EXTRACTOR_MetaFormat
Format in which the extracted meta data is presented.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
size_t size
Number of bytes stored in data.
Definition: fs_directory.c:113
void GNUNET_FS_meta_data_make_directory(struct GNUNET_CONTAINER_MetaData *md)
Set the MIMETYPE information for the given metadata to "application/gnunet-directory".
Definition: fs_directory.c:80
If not enough space is available, it is acceptable to only serialize some of the metadata.
Closure for &#39;find_full_data&#39;.
Definition: fs_directory.c:103
static size_t do_align(size_t start_position, size_t end_position)
Given the start and end position of a block of data, return the end position of that data after align...
Definition: fs_directory.c:487
Some kind of binary format, see given Mime type.
struct GNUNET_FS_Uri * GNUNET_FS_uri_parse(const char *uri, char **emsg)
Convert a UTF-8 String to a URI.
Definition: fs_uri.c:644
static int start
Set if we are to start default services (including ARM).
Definition: gnunet-arm.c:39
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct BuilderEntry * head
Head of linked list of entries.
Definition: fs_directory.c:350
#define GNUNET_FS_DIRECTORY_MIME
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static struct Experiment * e
Internal state of a directory builder.
Definition: fs_directory.c:340
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_DIRECTORY_MAGIC
String that is used to indicate that a file is a GNUnet directory.
Definition: fs_directory.c:43
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
static void block_align(size_t start, unsigned int count, const size_t *sizes, unsigned int *perm)
Compute a permuation of the blocks to minimize the cost of alignment.
Definition: fs_directory.c:508
char * GNUNET_CONTAINER_meta_data_get_by_type(const struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type)
Get the first MD entry of the given type.
int GNUNET_FS_uri_test_sks(const struct GNUNET_FS_Uri *uri)
Is this a namespace URI?
Definition: fs_uri.c:1282
EXTRACTOR_MetaType
Enumeration defining various sources of keywords.
#define GNUNET_malloc_large(size)
Wrapper around malloc.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
char * GNUNET_FS_uri_to_string(const struct GNUNET_FS_Uri *uri)
Convert a URI to a UTF-8 String.
Definition: fs_uri.c:2028
size_t len
Length of this entry.
Definition: fs_directory.c:334
#define GNUNET_FS_DIRECTORY_MAGIC
#define MAX_INLINE_SIZE
Maximum size for a file to be considered for inlining in a directory.
Definition: fs_api.h:49
Entries in the directory (builder).
Definition: fs_directory.c:324
static struct GNUNET_FS_Uri * uri
Value of URI provided on command-line (when not publishing a file but just creating UBlocks to refer ...
struct GNUNET_CONTAINER_MetaData * meta
Meta-data for the directory itself.
Definition: fs_directory.c:345
#define GNUNET_log_strerror(level, cmd)
Log an error message at log-level &#39;level&#39; that indicates a failure of the command &#39;cmd&#39; with the mess...
Meta data to associate with a file, directory or namespace.
void GNUNET_FS_directory_builder_add(struct GNUNET_FS_DirectoryBuilder *bld, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *md, const void *data)
Add an entry to a directory.
Definition: fs_directory.c:391
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
#define GNUNET_new_array(n, type)
Allocate a size n array with structs or unions of the given type.
static char * filename
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
void(* GNUNET_FS_DirectoryEntryProcessor)(void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, size_t length, const void *data)
Function used to process entries in a directory.
void GNUNET_FS_uri_destroy(struct GNUNET_FS_Uri *uri)
Free URI.
Definition: fs_uri.c:678
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
int GNUNET_FS_uri_test_chk(const struct GNUNET_FS_Uri *uri)
Is this a file (or directory) URI?
Definition: fs_uri.c:1357
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:40
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
unsigned int count
Number of entires in the directory.
Definition: fs_directory.c:355
int GNUNET_FS_meta_data_test_for_directory(const struct GNUNET_CONTAINER_MetaData *md)
Does the meta-data claim that this is a directory? Checks if the mime-type is that of a GNUnet direct...
Definition: fs_directory.c:54
int GNUNET_FS_directory_list_contents(size_t size, const void *data, uint64_t offset, GNUNET_FS_DirectoryEntryProcessor dep, void *dep_cls)
Iterate over all entries in a directory.
Definition: fs_directory.c:181
#define GNUNET_MAX_MALLOC_CHECKED
Maximum allocation with GNUNET_malloc macro.
static int find_full_data(void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_len)
Type of a function that libextractor calls for each meta data item found.
Definition: fs_directory.c:135
uint64_t GNUNET_FS_uri_chk_get_file_size(const struct GNUNET_FS_Uri *uri)
What is the size of the file that this URI refers to?
Definition: fs_uri.c:1371
shared definitions for the FS library
A Universal Resource Identifier (URI), opaque.
Definition: fs_api.h:165
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
int GNUNET_CONTAINER_meta_data_iterate(const struct GNUNET_CONTAINER_MetaData *md, EXTRACTOR_MetaDataProcessor iter, void *iter_cls)
Iterate over MD entries.
#define GNUNET_log(kind,...)
struct GNUNET_FS_Uri * GNUNET_FS_uri_loc_get_uri(const struct GNUNET_FS_Uri *uri)
Obtain the URI of the content itself.
Definition: fs_uri.c:853
struct BuilderEntry * next
This is a linked list.
Definition: fs_directory.c:329
void * data
Extracted binary meta data.
Definition: fs_directory.c:108
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
#define GNUNET_YES
Definition: gnunet_common.h:77
struct GNUNET_FS_DirectoryBuilder * GNUNET_FS_directory_builder_create(const struct GNUNET_CONTAINER_MetaData *mdir)
Create a directory builder.
Definition: fs_directory.c:365
static char * plugin_name
Name of our plugin.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_duplicate(const struct GNUNET_CONTAINER_MetaData *md)
Duplicate a MetaData token.