GNUnet  0.10.x
fs_directory.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19 */
20 
35 #include "platform.h"
36 #include "gnunet_fs_service.h"
37 #include "fs_api.h"
38 
43 #define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
44 
45 
53 int
55 {
56  char *mime;
57  int ret;
58 
59  if (NULL == md)
60  return GNUNET_SYSERR;
62  if (NULL == mime)
63  return GNUNET_SYSERR;
64  ret = (0 == strcasecmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO;
65  GNUNET_free (mime);
66  return ret;
67 }
68 
69 
76 void
78 {
79  char *mime;
80 
81  mime =
83  if (mime != NULL)
84  {
85  GNUNET_break (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME));
86  GNUNET_free (mime);
87  return;
88  }
89  GNUNET_CONTAINER_meta_data_insert (md, "<gnunet>",
91  EXTRACTOR_METAFORMAT_UTF8, "text/plain",
93  strlen (GNUNET_FS_DIRECTORY_MIME) + 1);
94 }
95 
96 
101 {
102 
106  void *data;
107 
111  size_t size;
112 };
113 
114 
132 static int
133 find_full_data (void *cls, const char *plugin_name,
134  enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
135  const char *data_mime_type, const char *data, size_t data_len)
136 {
137  struct GetFullDataClosure *gfdc = cls;
138 
140  {
141  gfdc->size = data_len;
142  if (data_len > 0)
143  {
144  gfdc->data = GNUNET_malloc (data_len);
145  GNUNET_memcpy (gfdc->data, data, data_len);
146  }
147  return 1;
148  }
149  return 0;
150 }
151 
152 
178 int
180  const void *data,
181  uint64_t offset,
183  void *dep_cls)
184 {
185  struct GetFullDataClosure full_data;
186  const char *cdata = data;
187  char *emsg;
188  uint64_t pos;
189  uint64_t align;
190  uint32_t mdSize;
191  uint64_t epos;
192  struct GNUNET_FS_Uri *uri;
193  struct GNUNET_CONTAINER_MetaData *md;
194  char *filename;
195 
196  if ((offset == 0) &&
197  ((size < 8 + sizeof (uint32_t)) ||
198  (0 != memcmp (cdata,
200  8))))
201  return GNUNET_SYSERR;
202  pos = offset;
203  if (offset == 0)
204  {
205  GNUNET_memcpy (&mdSize,
206  &cdata[8],
207  sizeof (uint32_t));
208  mdSize = ntohl (mdSize);
209  if (mdSize > size - 8 - sizeof (uint32_t))
210  {
211  /* invalid size */
213  _("MAGIC mismatch. This is not a GNUnet directory.\n"));
214  return GNUNET_SYSERR;
215  }
216  md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[8 + sizeof (uint32_t)],
217  mdSize);
218  if (md == NULL)
219  {
220  GNUNET_break (0);
221  return GNUNET_SYSERR; /* malformed ! */
222  }
223  dep (dep_cls,
224  NULL,
225  NULL,
226  md,
227  0,
228  NULL);
230  pos = 8 + sizeof (uint32_t) + mdSize;
231  }
232  while (pos < size)
233  {
234  /* find end of URI */
235  if (cdata[pos] == '\0')
236  {
237  /* URI is never empty, must be end of block,
238  * skip to next alignment */
239  align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE;
240  if (align == pos)
241  {
242  /* if we were already aligned, still skip a block! */
243  align += DBLOCK_SIZE;
244  }
245  pos = align;
246  if (pos >= size)
247  {
248  /* malformed - or partial download... */
249  break;
250  }
251  }
252  epos = pos;
253  while ((epos < size) && (cdata[epos] != '\0'))
254  epos++;
255  if (epos >= size)
256  return GNUNET_NO; /* malformed - or partial download */
257 
258  uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg);
259  pos = epos + 1;
260  if (NULL == uri)
261  {
262  GNUNET_free (emsg);
263  pos--; /* go back to '\0' to force going to next alignment */
264  continue;
265  }
266  if (GNUNET_FS_uri_test_ksk (uri))
267  {
268  GNUNET_FS_uri_destroy (uri);
269  GNUNET_break (0);
270  return GNUNET_NO; /* illegal in directory! */
271  }
272 
273  GNUNET_memcpy (&mdSize,
274  &cdata[pos],
275  sizeof (uint32_t));
276  mdSize = ntohl (mdSize);
277  pos += sizeof (uint32_t);
278  if (pos + mdSize > size)
279  {
280  GNUNET_FS_uri_destroy (uri);
281  return GNUNET_NO; /* malformed - or partial download */
282  }
283 
284  md = GNUNET_CONTAINER_meta_data_deserialize (&cdata[pos],
285  mdSize);
286  if (NULL == md)
287  {
288  GNUNET_FS_uri_destroy (uri);
289  GNUNET_break (0);
290  return GNUNET_NO; /* malformed ! */
291  }
292  pos += mdSize;
293  filename =
296  full_data.size = 0;
297  full_data.data = NULL;
300  &full_data);
301  if (NULL != dep)
302  {
303  dep (dep_cls,
304  filename,
305  uri,
306  md,
307  full_data.size,
308  full_data.data);
309  }
310  GNUNET_free_non_null (full_data.data);
311  GNUNET_free_non_null (filename);
313  GNUNET_FS_uri_destroy (uri);
314  }
315  return GNUNET_OK;
316 }
317 
322 {
327 
331  size_t len;
332 };
333 
338 {
343 
348 
352  unsigned int count;
353 };
354 
355 
363  *mdir)
364 {
366 
367  ret = GNUNET_new (struct GNUNET_FS_DirectoryBuilder);
368  if (mdir != NULL)
370  else
373  return ret;
374 }
375 
376 
387 void
389  const struct GNUNET_FS_Uri *uri,
390  const struct GNUNET_CONTAINER_MetaData *md,
391  const void *data)
392 {
393  struct GNUNET_FS_Uri *curi;
394  struct BuilderEntry *e;
395  uint64_t fsize;
396  uint32_t big;
397  ssize_t ret;
398  size_t mds;
399  size_t mdxs;
400  char *uris;
401  char *ser;
402  char *sptr;
403  size_t slen;
405  const struct GNUNET_CONTAINER_MetaData *meta_use;
406 
408  if (NULL != data)
409  {
411  if (GNUNET_FS_uri_test_chk (uri))
412  {
413  fsize = GNUNET_FS_uri_chk_get_file_size (uri);
414  }
415  else
416  {
417  curi = GNUNET_FS_uri_loc_get_uri (uri);
418  GNUNET_assert (NULL != curi);
419  fsize = GNUNET_FS_uri_chk_get_file_size (curi);
420  GNUNET_FS_uri_destroy (curi);
421  }
422  }
423  else
424  {
425  fsize = 0; /* not given */
426  }
427  if (fsize > MAX_INLINE_SIZE)
428  fsize = 0; /* too large */
429  uris = GNUNET_FS_uri_to_string (uri);
430  slen = strlen (uris) + 1;
432  meta_use = md;
433  meta = NULL;
434  if (fsize > 0)
435  {
437  GNUNET_CONTAINER_meta_data_insert (meta, "<gnunet>",
439  EXTRACTOR_METAFORMAT_BINARY, NULL, data,
440  fsize);
442  if ((slen + sizeof (uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
443  (slen + sizeof (uint32_t) + mds - 1) / DBLOCK_SIZE)
444  {
445  /* adding full data would not cause us to cross
446  * additional blocks, so add it! */
447  meta_use = meta;
448  mds = mdxs;
449  }
450  }
451 
452  if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
453  mds = GNUNET_MAX_MALLOC_CHECKED / 2;
454  e = GNUNET_malloc (sizeof (struct BuilderEntry) + slen + mds +
455  sizeof (uint32_t));
456  ser = (char *) &e[1];
457  GNUNET_memcpy (ser, uris, slen);
458  GNUNET_free (uris);
459  sptr = &ser[slen + sizeof (uint32_t)];
460  ret =
461  GNUNET_CONTAINER_meta_data_serialize (meta_use, &sptr, mds,
463  if (NULL != meta)
465  if (ret == -1)
466  mds = 0;
467  else
468  mds = ret;
469  big = htonl (mds);
470  GNUNET_memcpy (&ser[slen], &big, sizeof (uint32_t));
471  e->len = slen + sizeof (uint32_t) + mds;
472  e->next = bld->head;
473  bld->head = e;
474  bld->count++;
475 }
476 
477 
483 static size_t
484 do_align (size_t start_position, size_t end_position)
485 {
486  size_t align;
487 
488  align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE;
489  if ((start_position < align) && (end_position > align))
490  return align + end_position - start_position;
491  return end_position;
492 }
493 
494 
504 static void
505 block_align (size_t start, unsigned int count, const size_t * sizes,
506  unsigned int *perm)
507 {
508  unsigned int i;
509  unsigned int j;
510  unsigned int tmp;
511  unsigned int best;
512  ssize_t badness;
513  size_t cpos;
514  size_t cend;
515  ssize_t cbad;
516  unsigned int cval;
517 
518  cpos = start;
519  for (i = 0; i < count; i++)
520  {
521  start = cpos;
522  badness = 0x7FFFFFFF;
523  best = -1;
524  for (j = i; j < count; j++)
525  {
526  cval = perm[j];
527  cend = cpos + sizes[cval];
528  if (cpos % DBLOCK_SIZE == 0)
529  {
530  /* prefer placing the largest blocks first */
531  cbad = -(cend % DBLOCK_SIZE);
532  }
533  else
534  {
535  if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE)
536  {
537  /* Data fits into the same block! Prefer small left-overs! */
538  cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE;
539  }
540  else
541  {
542  /* Would have to waste space to re-align, add big factor, this
543  * case is a real loss (proportional to space wasted)! */
544  cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE);
545  }
546  }
547  if (cbad < badness)
548  {
549  best = j;
550  badness = cbad;
551  }
552  }
553  GNUNET_assert (best != -1);
554  tmp = perm[i];
555  perm[i] = perm[best];
556  perm[best] = tmp;
557  cpos += sizes[perm[i]];
558  cpos = do_align (start, cpos);
559  }
560 }
561 
562 
573 int
575  size_t * rsize,
576  void **rdata)
577 {
578  char *data;
579  char *sptr;
580  size_t *sizes;
581  unsigned int *perm;
582  unsigned int i;
583  unsigned int j;
584  struct BuilderEntry *pos;
585  struct BuilderEntry **bes;
586  size_t size;
587  size_t psize;
588  size_t off;
589  ssize_t ret;
590  uint32_t big;
591 
592  size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof (uint32_t);
594  sizes = NULL;
595  perm = NULL;
596  bes = NULL;
597  if (0 < bld->count)
598  {
599  sizes = GNUNET_new_array (bld->count,
600  size_t);
601  perm = GNUNET_new_array (bld->count,
602  unsigned int);
603  bes = GNUNET_new_array (bld->count,
604  struct BuilderEntry *);
605  pos = bld->head;
606  for (i = 0; i < bld->count; i++)
607  {
608  perm[i] = i;
609  bes[i] = pos;
610  sizes[i] = pos->len;
611  pos = pos->next;
612  }
613  block_align (size, bld->count, sizes, perm);
614  /* compute final size with alignment */
615  for (i = 0; i < bld->count; i++)
616  {
617  psize = size;
618  size += sizes[perm[i]];
619  size = do_align (psize, size);
620  }
621  }
622  *rsize = size;
623  data = GNUNET_malloc_large (size);
624  if (data == NULL)
625  {
627  "malloc");
628  *rsize = 0;
629  *rdata = NULL;
630  GNUNET_free_non_null (sizes);
631  GNUNET_free_non_null (perm);
632  GNUNET_free_non_null (bes);
633  return GNUNET_SYSERR;
634  }
635  *rdata = data;
636  GNUNET_memcpy (data,
638  strlen (GNUNET_DIRECTORY_MAGIC));
639  off = strlen (GNUNET_DIRECTORY_MAGIC);
640 
641  sptr = &data[off + sizeof (uint32_t)];
642  ret =
644  &sptr,
645  size - off - sizeof (uint32_t),
647  GNUNET_assert (ret != -1);
648  big = htonl (ret);
649  GNUNET_memcpy (&data[off],
650  &big,
651  sizeof (uint32_t));
652  off += sizeof (uint32_t) + ret;
653  for (j = 0; j < bld->count; j++)
654  {
655  i = perm[j];
656  psize = off;
657  off += sizes[i];
658  off = do_align (psize, off);
659  GNUNET_memcpy (&data[off - sizes[i]], &(bes[i])[1], sizes[i]);
660  GNUNET_free (bes[i]);
661  }
662  GNUNET_free_non_null (sizes);
663  GNUNET_free_non_null (perm);
664  GNUNET_free_non_null (bes);
665  GNUNET_assert (off == size);
667  GNUNET_free (bld);
668  return GNUNET_OK;
669 }
670 
671 
672 /* end of fs_directory.c */
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
int GNUNET_FS_uri_test_ksk(const struct GNUNET_FS_Uri *uri)
Is this a keyword URI?
Definition: fs_uri.c:1323
int GNUNET_FS_directory_builder_finish(struct GNUNET_FS_DirectoryBuilder *bld, size_t *rsize, void **rdata)
Finish building the directory.
Definition: fs_directory.c:574
EXTRACTOR_MetaFormat
Format in which the extracted meta data is presented.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
size_t size
Number of bytes stored in data.
Definition: fs_directory.c:111
void GNUNET_FS_meta_data_make_directory(struct GNUNET_CONTAINER_MetaData *md)
Set the MIMETYPE information for the given metadata to "application/gnunet-directory".
Definition: fs_directory.c:77
If not enough space is available, it is acceptable to only serialize some of the metadata.
Closure for &#39;find_full_data&#39;.
Definition: fs_directory.c:100
static size_t do_align(size_t start_position, size_t end_position)
Given the start and end position of a block of data, return the end position of that data after align...
Definition: fs_directory.c:484
Some kind of binary format, see given Mime type.
struct GNUNET_FS_Uri * GNUNET_FS_uri_parse(const char *uri, char **emsg)
Convert a UTF-8 String to a URI.
Definition: fs_uri.c:635
static int start
Set if we are to start default services (including ARM).
Definition: gnunet-arm.c:39
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct BuilderEntry * head
Head of linked list of entries.
Definition: fs_directory.c:347
#define GNUNET_FS_DIRECTORY_MIME
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
static struct Experiment * e
Internal state of a directory builder.
Definition: fs_directory.c:337
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:81
#define GNUNET_DIRECTORY_MAGIC
String that is used to indicate that a file is a GNUnet directory.
Definition: fs_directory.c:43
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:78
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
static void block_align(size_t start, unsigned int count, const size_t *sizes, unsigned int *perm)
Compute a permuation of the blocks to minimize the cost of alignment.
Definition: fs_directory.c:505
char * GNUNET_CONTAINER_meta_data_get_by_type(const struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type)
Get the first MD entry of the given type.
int GNUNET_FS_uri_test_sks(const struct GNUNET_FS_Uri *uri)
Is this a namespace URI?
Definition: fs_uri.c:1270
EXTRACTOR_MetaType
Enumeration defining various sources of keywords.
static int ret
Final status code.
Definition: gnunet-arm.c:89
#define GNUNET_malloc_large(size)
Wrapper around malloc.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
#define _(String)
GNU gettext support macro.
Definition: platform.h:208
char * GNUNET_FS_uri_to_string(const struct GNUNET_FS_Uri *uri)
Convert a URI to a UTF-8 String.
Definition: fs_uri.c:1988
size_t len
Length of this entry.
Definition: fs_directory.c:331
#define GNUNET_FS_DIRECTORY_MAGIC
#define MAX_INLINE_SIZE
Maximum size for a file to be considered for inlining in a directory.
Definition: fs_api.h:49
Entries in the directory (builder).
Definition: fs_directory.c:321
static struct GNUNET_FS_Uri * uri
Value of URI provided on command-line (when not publishing a file but just creating UBlocks to refer ...
struct GNUNET_CONTAINER_MetaData * meta
Meta-data for the directory itself.
Definition: fs_directory.c:342
#define GNUNET_log_strerror(level, cmd)
Log an error message at log-level &#39;level&#39; that indicates a failure of the command &#39;cmd&#39; with the mess...
Meta data to associate with a file, directory or namespace.
void GNUNET_FS_directory_builder_add(struct GNUNET_FS_DirectoryBuilder *bld, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *md, const void *data)
Add an entry to a directory.
Definition: fs_directory.c:388
#define GNUNET_memcpy(dst, src, n)
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
#define GNUNET_new_array(n, type)
Allocate a size n array with structs or unions of the given type.
static char * filename
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
static char * plugin_name
Solver plugin name as string.
void(* GNUNET_FS_DirectoryEntryProcessor)(void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, size_t length, const void *data)
Function used to process entries in a directory.
void GNUNET_FS_uri_destroy(struct GNUNET_FS_Uri *uri)
Free URI.
Definition: fs_uri.c:670
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
int GNUNET_FS_uri_test_chk(const struct GNUNET_FS_Uri *uri)
Is this a file (or directory) URI?
Definition: fs_uri.c:1345
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:40
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
unsigned int count
Number of entires in the directory.
Definition: fs_directory.c:352
int GNUNET_FS_meta_data_test_for_directory(const struct GNUNET_CONTAINER_MetaData *md)
Does the meta-data claim that this is a directory? Checks if the mime-type is that of a GNUnet direct...
Definition: fs_directory.c:54
int GNUNET_FS_directory_list_contents(size_t size, const void *data, uint64_t offset, GNUNET_FS_DirectoryEntryProcessor dep, void *dep_cls)
Iterate over all entries in a directory.
Definition: fs_directory.c:179
#define GNUNET_MAX_MALLOC_CHECKED
Maximum allocation with GNUNET_malloc macro.
static int find_full_data(void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_len)
Type of a function that libextractor calls for each meta data item found.
Definition: fs_directory.c:133
uint64_t GNUNET_FS_uri_chk_get_file_size(const struct GNUNET_FS_Uri *uri)
What is the size of the file that this URI refers to?
Definition: fs_uri.c:1359
shared definitions for the FS library
A Universal Resource Identifier (URI), opaque.
Definition: fs_api.h:168
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
int GNUNET_CONTAINER_meta_data_iterate(const struct GNUNET_CONTAINER_MetaData *md, EXTRACTOR_MetaDataProcessor iter, void *iter_cls)
Iterate over MD entries.
#define GNUNET_log(kind,...)
struct GNUNET_FS_Uri * GNUNET_FS_uri_loc_get_uri(const struct GNUNET_FS_Uri *uri)
Obtain the URI of the content itself.
Definition: fs_uri.c:841
struct BuilderEntry * next
This is a linked list.
Definition: fs_directory.c:326
void * data
Extracted binary meta data.
Definition: fs_directory.c:106
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
#define GNUNET_YES
Definition: gnunet_common.h:80
struct GNUNET_FS_DirectoryBuilder * GNUNET_FS_directory_builder_create(const struct GNUNET_CONTAINER_MetaData *mdir)
Create a directory builder.
Definition: fs_directory.c:362
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_duplicate(const struct GNUNET_CONTAINER_MetaData *md)
Duplicate a MetaData token.