GNUnet  0.10.x
fs_directory.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
35 #include "platform.h"
36 #include "gnunet_fs_service.h"
37 #include "fs_api.h"
38 
43 #define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
44 
45 
53 int
55 {
56  char *mime;
57  int ret;
58 
59  if (NULL == md)
60  return GNUNET_SYSERR;
62  if (NULL == mime)
63  return GNUNET_SYSERR;
64  ret = (0 == strcasecmp(mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES : GNUNET_NO;
65  GNUNET_free(mime);
66  return ret;
67 }
68 
69 
76 void
78 {
79  char *mime;
80 
81  mime =
83  if (mime != NULL)
84  {
85  GNUNET_break(0 == strcmp(mime, GNUNET_FS_DIRECTORY_MIME));
86  GNUNET_free(mime);
87  return;
88  }
89  GNUNET_CONTAINER_meta_data_insert(md, "<gnunet>",
91  EXTRACTOR_METAFORMAT_UTF8, "text/plain",
93  strlen(GNUNET_FS_DIRECTORY_MIME) + 1);
94 }
95 
96 
104  void *data;
105 
109  size_t size;
110 };
111 
112 
130 static int
131 find_full_data(void *cls, const char *plugin_name,
132  enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
133  const char *data_mime_type, const char *data, size_t data_len)
134 {
135  struct GetFullDataClosure *gfdc = cls;
136 
138  {
139  gfdc->size = data_len;
140  if (data_len > 0)
141  {
142  gfdc->data = GNUNET_malloc(data_len);
143  GNUNET_memcpy(gfdc->data, data, data_len);
144  }
145  return 1;
146  }
147  return 0;
148 }
149 
150 
176 int
178  const void *data,
179  uint64_t offset,
181  void *dep_cls)
182 {
183  struct GetFullDataClosure full_data;
184  const char *cdata = data;
185  char *emsg;
186  uint64_t pos;
187  uint64_t align;
188  uint32_t mdSize;
189  uint64_t epos;
190  struct GNUNET_FS_Uri *uri;
191  struct GNUNET_CONTAINER_MetaData *md;
192  char *filename;
193 
194  if ((offset == 0) &&
195  ((size < 8 + sizeof(uint32_t)) ||
196  (0 != memcmp(cdata,
198  8))))
199  return GNUNET_SYSERR;
200  pos = offset;
201  if (offset == 0)
202  {
203  GNUNET_memcpy(&mdSize,
204  &cdata[8],
205  sizeof(uint32_t));
206  mdSize = ntohl(mdSize);
207  if (mdSize > size - 8 - sizeof(uint32_t))
208  {
209  /* invalid size */
211  _("MAGIC mismatch. This is not a GNUnet directory.\n"));
212  return GNUNET_SYSERR;
213  }
214  md = GNUNET_CONTAINER_meta_data_deserialize(&cdata[8 + sizeof(uint32_t)],
215  mdSize);
216  if (md == NULL)
217  {
218  GNUNET_break(0);
219  return GNUNET_SYSERR; /* malformed ! */
220  }
221  dep(dep_cls,
222  NULL,
223  NULL,
224  md,
225  0,
226  NULL);
228  pos = 8 + sizeof(uint32_t) + mdSize;
229  }
230  while (pos < size)
231  {
232  /* find end of URI */
233  if (cdata[pos] == '\0')
234  {
235  /* URI is never empty, must be end of block,
236  * skip to next alignment */
237  align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE;
238  if (align == pos)
239  {
240  /* if we were already aligned, still skip a block! */
241  align += DBLOCK_SIZE;
242  }
243  pos = align;
244  if (pos >= size)
245  {
246  /* malformed - or partial download... */
247  break;
248  }
249  }
250  epos = pos;
251  while ((epos < size) && (cdata[epos] != '\0'))
252  epos++;
253  if (epos >= size)
254  return GNUNET_NO; /* malformed - or partial download */
255 
256  uri = GNUNET_FS_uri_parse(&cdata[pos], &emsg);
257  pos = epos + 1;
258  if (NULL == uri)
259  {
260  GNUNET_free(emsg);
261  pos--; /* go back to '\0' to force going to next alignment */
262  continue;
263  }
264  if (GNUNET_FS_uri_test_ksk(uri))
265  {
267  GNUNET_break(0);
268  return GNUNET_NO; /* illegal in directory! */
269  }
270 
271  GNUNET_memcpy(&mdSize,
272  &cdata[pos],
273  sizeof(uint32_t));
274  mdSize = ntohl(mdSize);
275  pos += sizeof(uint32_t);
276  if (pos + mdSize > size)
277  {
279  return GNUNET_NO; /* malformed - or partial download */
280  }
281 
283  mdSize);
284  if (NULL == md)
285  {
287  GNUNET_break(0);
288  return GNUNET_NO; /* malformed ! */
289  }
290  pos += mdSize;
291  filename =
294  full_data.size = 0;
295  full_data.data = NULL;
298  &full_data);
299  if (NULL != dep)
300  {
301  dep(dep_cls,
302  filename,
303  uri,
304  md,
305  full_data.size,
306  full_data.data);
307  }
308  GNUNET_free_non_null(full_data.data);
309  GNUNET_free_non_null(filename);
312  }
313  return GNUNET_OK;
314 }
315 
319 struct BuilderEntry {
324 
328  size_t len;
329 };
330 
339 
344 
348  unsigned int count;
349 };
350 
351 
359  *mdir)
360 {
362 
364  if (mdir != NULL)
366  else
369  return ret;
370 }
371 
372 
383 void
385  const struct GNUNET_FS_Uri *uri,
386  const struct GNUNET_CONTAINER_MetaData *md,
387  const void *data)
388 {
389  struct GNUNET_FS_Uri *curi;
390  struct BuilderEntry *e;
391  uint64_t fsize;
392  uint32_t big;
393  ssize_t ret;
394  size_t mds;
395  size_t mdxs;
396  char *uris;
397  char *ser;
398  char *sptr;
399  size_t slen;
401  const struct GNUNET_CONTAINER_MetaData *meta_use;
402 
404  if (NULL != data)
405  {
407  if (GNUNET_FS_uri_test_chk(uri))
408  {
409  fsize = GNUNET_FS_uri_chk_get_file_size(uri);
410  }
411  else
412  {
413  curi = GNUNET_FS_uri_loc_get_uri(uri);
414  GNUNET_assert(NULL != curi);
415  fsize = GNUNET_FS_uri_chk_get_file_size(curi);
416  GNUNET_FS_uri_destroy(curi);
417  }
418  }
419  else
420  {
421  fsize = 0; /* not given */
422  }
423  if (fsize > MAX_INLINE_SIZE)
424  fsize = 0; /* too large */
425  uris = GNUNET_FS_uri_to_string(uri);
426  slen = strlen(uris) + 1;
428  meta_use = md;
429  meta = NULL;
430  if (fsize > 0)
431  {
433  GNUNET_CONTAINER_meta_data_insert(meta, "<gnunet>",
435  EXTRACTOR_METAFORMAT_BINARY, NULL, data,
436  fsize);
438  if ((slen + sizeof(uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
439  (slen + sizeof(uint32_t) + mds - 1) / DBLOCK_SIZE)
440  {
441  /* adding full data would not cause us to cross
442  * additional blocks, so add it! */
443  meta_use = meta;
444  mds = mdxs;
445  }
446  }
447 
448  if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
449  mds = GNUNET_MAX_MALLOC_CHECKED / 2;
450  e = GNUNET_malloc(sizeof(struct BuilderEntry) + slen + mds +
451  sizeof(uint32_t));
452  ser = (char *)&e[1];
453  GNUNET_memcpy(ser, uris, slen);
454  GNUNET_free(uris);
455  sptr = &ser[slen + sizeof(uint32_t)];
456  ret =
457  GNUNET_CONTAINER_meta_data_serialize(meta_use, &sptr, mds,
459  if (NULL != meta)
461  if (ret == -1)
462  mds = 0;
463  else
464  mds = ret;
465  big = htonl(mds);
466  GNUNET_memcpy(&ser[slen], &big, sizeof(uint32_t));
467  e->len = slen + sizeof(uint32_t) + mds;
468  e->next = bld->head;
469  bld->head = e;
470  bld->count++;
471 }
472 
473 
479 static size_t
480 do_align(size_t start_position, size_t end_position)
481 {
482  size_t align;
483 
484  align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE;
485  if ((start_position < align) && (end_position > align))
486  return align + end_position - start_position;
487  return end_position;
488 }
489 
490 
500 static void
501 block_align(size_t start, unsigned int count, const size_t * sizes,
502  unsigned int *perm)
503 {
504  unsigned int i;
505  unsigned int j;
506  unsigned int tmp;
507  unsigned int best;
508  ssize_t badness;
509  size_t cpos;
510  size_t cend;
511  ssize_t cbad;
512  unsigned int cval;
513 
514  cpos = start;
515  for (i = 0; i < count; i++)
516  {
517  start = cpos;
518  badness = 0x7FFFFFFF;
519  best = -1;
520  for (j = i; j < count; j++)
521  {
522  cval = perm[j];
523  cend = cpos + sizes[cval];
524  if (cpos % DBLOCK_SIZE == 0)
525  {
526  /* prefer placing the largest blocks first */
527  cbad = -(cend % DBLOCK_SIZE);
528  }
529  else
530  {
531  if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE)
532  {
533  /* Data fits into the same block! Prefer small left-overs! */
534  cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE;
535  }
536  else
537  {
538  /* Would have to waste space to re-align, add big factor, this
539  * case is a real loss (proportional to space wasted)! */
540  cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE);
541  }
542  }
543  if (cbad < badness)
544  {
545  best = j;
546  badness = cbad;
547  }
548  }
549  GNUNET_assert(best != -1);
550  tmp = perm[i];
551  perm[i] = perm[best];
552  perm[best] = tmp;
553  cpos += sizes[perm[i]];
554  cpos = do_align(start, cpos);
555  }
556 }
557 
558 
569 int
571  size_t * rsize,
572  void **rdata)
573 {
574  char *data;
575  char *sptr;
576  size_t *sizes;
577  unsigned int *perm;
578  unsigned int i;
579  unsigned int j;
580  struct BuilderEntry *pos;
581  struct BuilderEntry **bes;
582  size_t size;
583  size_t psize;
584  size_t off;
585  ssize_t ret;
586  uint32_t big;
587 
588  size = strlen(GNUNET_DIRECTORY_MAGIC) + sizeof(uint32_t);
590  sizes = NULL;
591  perm = NULL;
592  bes = NULL;
593  if (0 < bld->count)
594  {
595  sizes = GNUNET_new_array(bld->count,
596  size_t);
597  perm = GNUNET_new_array(bld->count,
598  unsigned int);
599  bes = GNUNET_new_array(bld->count,
600  struct BuilderEntry *);
601  pos = bld->head;
602  for (i = 0; i < bld->count; i++)
603  {
604  perm[i] = i;
605  bes[i] = pos;
606  sizes[i] = pos->len;
607  pos = pos->next;
608  }
609  block_align(size, bld->count, sizes, perm);
610  /* compute final size with alignment */
611  for (i = 0; i < bld->count; i++)
612  {
613  psize = size;
614  size += sizes[perm[i]];
615  size = do_align(psize, size);
616  }
617  }
618  *rsize = size;
619  data = GNUNET_malloc_large(size);
620  if (data == NULL)
621  {
623  "malloc");
624  *rsize = 0;
625  *rdata = NULL;
626  GNUNET_free_non_null(sizes);
627  GNUNET_free_non_null(perm);
629  return GNUNET_SYSERR;
630  }
631  *rdata = data;
632  GNUNET_memcpy(data,
634  strlen(GNUNET_DIRECTORY_MAGIC));
635  off = strlen(GNUNET_DIRECTORY_MAGIC);
636 
637  sptr = &data[off + sizeof(uint32_t)];
638  ret =
640  &sptr,
641  size - off - sizeof(uint32_t),
643  GNUNET_assert(ret != -1);
644  big = htonl(ret);
645  GNUNET_memcpy(&data[off],
646  &big,
647  sizeof(uint32_t));
648  off += sizeof(uint32_t) + ret;
649  for (j = 0; j < bld->count; j++)
650  {
651  i = perm[j];
652  psize = off;
653  off += sizes[i];
654  off = do_align(psize, off);
655  GNUNET_memcpy(&data[off - sizes[i]], &(bes[i])[1], sizes[i]);
656  GNUNET_free(bes[i]);
657  }
658  GNUNET_free_non_null(sizes);
659  GNUNET_free_non_null(perm);
661  GNUNET_assert(off == size);
663  GNUNET_free(bld);
664  return GNUNET_OK;
665 }
666 
667 
668 /* end of fs_directory.c */
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
int GNUNET_FS_uri_test_ksk(const struct GNUNET_FS_Uri *uri)
Is this a keyword URI?
Definition: fs_uri.c:1332
int GNUNET_FS_directory_builder_finish(struct GNUNET_FS_DirectoryBuilder *bld, size_t *rsize, void **rdata)
Finish building the directory.
Definition: fs_directory.c:570
EXTRACTOR_MetaFormat
Format in which the extracted meta data is presented.
ssize_t GNUNET_CONTAINER_meta_data_serialize(const struct GNUNET_CONTAINER_MetaData *md, char **target, size_t max, enum GNUNET_CONTAINER_MetaDataSerializationOptions opt)
Serialize meta-data to target.
size_t size
Number of bytes stored in data.
Definition: fs_directory.c:109
void GNUNET_FS_meta_data_make_directory(struct GNUNET_CONTAINER_MetaData *md)
Set the MIMETYPE information for the given metadata to "application/gnunet-directory".
Definition: fs_directory.c:77
If not enough space is available, it is acceptable to only serialize some of the metadata.
Closure for &#39;find_full_data&#39;.
Definition: fs_directory.c:100
static size_t do_align(size_t start_position, size_t end_position)
Given the start and end position of a block of data, return the end position of that data after align...
Definition: fs_directory.c:480
Some kind of binary format, see given Mime type.
struct GNUNET_FS_Uri * GNUNET_FS_uri_parse(const char *uri, char **emsg)
Convert a UTF-8 String to a URI.
Definition: fs_uri.c:641
static int start
Set if we are to start default services (including ARM).
Definition: gnunet-arm.c:39
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct BuilderEntry * head
Head of linked list of entries.
Definition: fs_directory.c:343
#define GNUNET_FS_DIRECTORY_MIME
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
static struct Experiment * e
Internal state of a directory builder.
Definition: fs_directory.c:334
0-terminated, UTF-8 encoded string.
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_DIRECTORY_MAGIC
String that is used to indicate that a file is a GNUnet directory.
Definition: fs_directory.c:43
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
static void block_align(size_t start, unsigned int count, const size_t *sizes, unsigned int *perm)
Compute a permuation of the blocks to minimize the cost of alignment.
Definition: fs_directory.c:501
char * GNUNET_CONTAINER_meta_data_get_by_type(const struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type)
Get the first MD entry of the given type.
int GNUNET_FS_uri_test_sks(const struct GNUNET_FS_Uri *uri)
Is this a namespace URI?
Definition: fs_uri.c:1279
EXTRACTOR_MetaType
Enumeration defining various sources of keywords.
static int ret
Final status code.
Definition: gnunet-arm.c:89
#define GNUNET_malloc_large(size)
Wrapper around malloc.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
char * GNUNET_FS_uri_to_string(const struct GNUNET_FS_Uri *uri)
Convert a URI to a UTF-8 String.
Definition: fs_uri.c:2024
size_t len
Length of this entry.
Definition: fs_directory.c:328
#define GNUNET_FS_DIRECTORY_MAGIC
#define MAX_INLINE_SIZE
Maximum size for a file to be considered for inlining in a directory.
Definition: fs_api.h:49
Entries in the directory (builder).
Definition: fs_directory.c:319
static struct GNUNET_FS_Uri * uri
Value of URI provided on command-line (when not publishing a file but just creating UBlocks to refer ...
struct GNUNET_CONTAINER_MetaData * meta
Meta-data for the directory itself.
Definition: fs_directory.c:338
#define GNUNET_log_strerror(level, cmd)
Log an error message at log-level &#39;level&#39; that indicates a failure of the command &#39;cmd&#39; with the mess...
Meta data to associate with a file, directory or namespace.
void GNUNET_FS_directory_builder_add(struct GNUNET_FS_DirectoryBuilder *bld, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *md, const void *data)
Add an entry to a directory.
Definition: fs_directory.c:384
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_create(void)
Create a fresh meta data container.
#define GNUNET_new_array(n, type)
Allocate a size n array with structs or unions of the given type.
static char * filename
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
static char * plugin_name
Solver plugin name as string.
void(* GNUNET_FS_DirectoryEntryProcessor)(void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, size_t length, const void *data)
Function used to process entries in a directory.
void GNUNET_FS_uri_destroy(struct GNUNET_FS_Uri *uri)
Free URI.
Definition: fs_uri.c:675
int GNUNET_CONTAINER_meta_data_insert(struct GNUNET_CONTAINER_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
int GNUNET_FS_uri_test_chk(const struct GNUNET_FS_Uri *uri)
Is this a file (or directory) URI?
Definition: fs_uri.c:1354
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:40
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
unsigned int count
Number of entires in the directory.
Definition: fs_directory.c:348
int GNUNET_FS_meta_data_test_for_directory(const struct GNUNET_CONTAINER_MetaData *md)
Does the meta-data claim that this is a directory? Checks if the mime-type is that of a GNUnet direct...
Definition: fs_directory.c:54
int GNUNET_FS_directory_list_contents(size_t size, const void *data, uint64_t offset, GNUNET_FS_DirectoryEntryProcessor dep, void *dep_cls)
Iterate over all entries in a directory.
Definition: fs_directory.c:177
#define GNUNET_MAX_MALLOC_CHECKED
Maximum allocation with GNUNET_malloc macro.
static int find_full_data(void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_len)
Type of a function that libextractor calls for each meta data item found.
Definition: fs_directory.c:131
uint64_t GNUNET_FS_uri_chk_get_file_size(const struct GNUNET_FS_Uri *uri)
What is the size of the file that this URI refers to?
Definition: fs_uri.c:1368
shared definitions for the FS library
A Universal Resource Identifier (URI), opaque.
Definition: fs_api.h:162
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
int GNUNET_CONTAINER_meta_data_iterate(const struct GNUNET_CONTAINER_MetaData *md, EXTRACTOR_MetaDataProcessor iter, void *iter_cls)
Iterate over MD entries.
#define GNUNET_log(kind,...)
struct GNUNET_FS_Uri * GNUNET_FS_uri_loc_get_uri(const struct GNUNET_FS_Uri *uri)
Obtain the URI of the content itself.
Definition: fs_uri.c:850
struct BuilderEntry * next
This is a linked list.
Definition: fs_directory.c:323
void * data
Extracted binary meta data.
Definition: fs_directory.c:104
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
#define GNUNET_YES
Definition: gnunet_common.h:77
struct GNUNET_FS_DirectoryBuilder * GNUNET_FS_directory_builder_create(const struct GNUNET_CONTAINER_MetaData *mdir)
Create a directory builder.
Definition: fs_directory.c:358
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_duplicate(const struct GNUNET_CONTAINER_MetaData *md)
Duplicate a MetaData token.