GNUnet  0.19.2
fs_directory.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2003, 2004, 2006, 2009 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
35 #include "platform.h"
36 
37 #include "gnunet_fs_service.h"
38 #include "fs_api.h"
39 
44 #define GNUNET_DIRECTORY_MAGIC "\211GND\r\n\032\n"
45 
46 
54 int
57 {
58  char *mime;
59  int ret;
60 
61  if (NULL == md)
62  return GNUNET_SYSERR;
64  EXTRACTOR_METATYPE_MIMETYPE);
65  if (NULL == mime)
66  return GNUNET_SYSERR;
67  ret = (0 == strcasecmp (mime, GNUNET_FS_DIRECTORY_MIME)) ? GNUNET_YES :
68  GNUNET_NO;
69  GNUNET_free (mime);
70  return ret;
71 }
72 
73 
80 void
82 {
83  char *mime;
84 
85  mime =
86  GNUNET_FS_meta_data_get_by_type (md, EXTRACTOR_METATYPE_MIMETYPE);
87  if (mime != NULL)
88  {
89  GNUNET_break (0 == strcmp (mime, GNUNET_FS_DIRECTORY_MIME));
90  GNUNET_free (mime);
91  return;
92  }
93  GNUNET_FS_meta_data_insert (md, "<gnunet>",
94  EXTRACTOR_METATYPE_MIMETYPE,
95  EXTRACTOR_METAFORMAT_UTF8, "text/plain",
97  strlen (GNUNET_FS_DIRECTORY_MIME) + 1);
98 }
99 
100 
105 {
109  void *data;
110 
114  size_t size;
115 };
116 
117 
135 static int
136 find_full_data (void *cls, const char *plugin_name,
137  enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format,
138  const char *data_mime_type, const char *data, size_t data_len)
139 {
140  struct GetFullDataClosure *gfdc = cls;
141 
142  if (type == EXTRACTOR_METATYPE_GNUNET_FULL_DATA)
143  {
144  gfdc->size = data_len;
145  if (data_len > 0)
146  {
147  gfdc->data = GNUNET_malloc (data_len);
148  GNUNET_memcpy (gfdc->data, data, data_len);
149  }
150  return 1;
151  }
152  return 0;
153 }
154 
155 
181 int
183  const void *data,
184  uint64_t offset,
186  void *dep_cls)
187 {
188  struct GetFullDataClosure full_data;
189  const char *cdata = data;
190  char *emsg;
191  uint64_t pos;
192  uint64_t align;
193  uint32_t mdSize;
194  uint64_t epos;
195  struct GNUNET_FS_Uri *uri;
196  struct GNUNET_FS_MetaData *md;
197  char *filename;
198 
199  if ((offset == 0) &&
200  ((size < 8 + sizeof(uint32_t)) ||
201  (0 != memcmp (cdata,
203  8))))
204  return GNUNET_SYSERR;
205  pos = offset;
206  if (offset == 0)
207  {
208  GNUNET_memcpy (&mdSize,
209  &cdata[8],
210  sizeof(uint32_t));
211  mdSize = ntohl (mdSize);
212  if (mdSize > size - 8 - sizeof(uint32_t))
213  {
214  /* invalid size */
216  _ ("MAGIC mismatch. This is not a GNUnet directory.\n"));
217  return GNUNET_SYSERR;
218  }
219  md = GNUNET_FS_meta_data_deserialize (&cdata[8 + sizeof(uint32_t)],
220  mdSize);
221  if (md == NULL)
222  {
223  GNUNET_break (0);
224  return GNUNET_SYSERR; /* malformed ! */
225  }
226  dep (dep_cls,
227  NULL,
228  NULL,
229  md,
230  0,
231  NULL);
233  pos = 8 + sizeof(uint32_t) + mdSize;
234  }
235  while (pos < size)
236  {
237  /* find end of URI */
238  if (cdata[pos] == '\0')
239  {
240  /* URI is never empty, must be end of block,
241  * skip to next alignment */
242  align = ((pos / DBLOCK_SIZE) + 1) * DBLOCK_SIZE;
243  if (align == pos)
244  {
245  /* if we were already aligned, still skip a block! */
246  align += DBLOCK_SIZE;
247  }
248  pos = align;
249  if (pos >= size)
250  {
251  /* malformed - or partial download... */
252  break;
253  }
254  }
255  epos = pos;
256  while ((epos < size) && (cdata[epos] != '\0'))
257  epos++;
258  if (epos >= size)
259  return GNUNET_NO; /* malformed - or partial download */
260 
261  uri = GNUNET_FS_uri_parse (&cdata[pos], &emsg);
262  pos = epos + 1;
263  if (NULL == uri)
264  {
265  GNUNET_free (emsg);
266  pos--; /* go back to '\0' to force going to next alignment */
267  continue;
268  }
270  {
272  GNUNET_break (0);
273  return GNUNET_NO; /* illegal in directory! */
274  }
275 
276  GNUNET_memcpy (&mdSize,
277  &cdata[pos],
278  sizeof(uint32_t));
279  mdSize = ntohl (mdSize);
280  pos += sizeof(uint32_t);
281  if (pos + mdSize > size)
282  {
284  return GNUNET_NO; /* malformed - or partial download */
285  }
286 
287  md = GNUNET_FS_meta_data_deserialize (&cdata[pos],
288  mdSize);
289  if (NULL == md)
290  {
292  GNUNET_break (0);
293  return GNUNET_NO; /* malformed ! */
294  }
295  pos += mdSize;
296  filename =
299  full_data.size = 0;
300  full_data.data = NULL;
303  &full_data);
304  if (NULL != dep)
305  {
306  dep (dep_cls,
307  filename,
308  uri,
309  md,
310  full_data.size,
311  full_data.data);
312  }
313  GNUNET_free (full_data.data);
317  }
318  return GNUNET_OK;
319 }
320 
321 
326 {
331 
335  size_t len;
336 };
337 
342 {
347 
352 
356  unsigned int count;
357 };
358 
359 
367  *mdir)
368 {
370 
372  if (mdir != NULL)
373  ret->meta = GNUNET_FS_meta_data_duplicate (mdir);
374  else
375  ret->meta = GNUNET_FS_meta_data_create ();
377  return ret;
378 }
379 
380 
391 void
393  const struct GNUNET_FS_Uri *uri,
394  const struct GNUNET_FS_MetaData *md,
395  const void *data)
396 {
397  struct GNUNET_FS_Uri *curi;
398  struct BuilderEntry *e;
399  uint64_t fsize;
400  uint32_t big;
401  ssize_t ret;
402  size_t mds;
403  size_t mdxs;
404  char *uris;
405  char *serialized;
406  char *sptr;
407  size_t slen;
408  struct GNUNET_FS_MetaData *meta;
409  const struct GNUNET_FS_MetaData *meta_use;
410 
412  if (NULL != data)
413  {
416  {
418  }
419  else
420  {
422  GNUNET_assert (NULL != curi);
423  fsize = GNUNET_FS_uri_chk_get_file_size (curi);
424  GNUNET_FS_uri_destroy (curi);
425  }
426  }
427  else
428  {
429  fsize = 0; /* not given */
430  }
431  if (fsize > MAX_INLINE_SIZE)
432  fsize = 0; /* too large */
433  uris = GNUNET_FS_uri_to_string (uri);
434  slen = strlen (uris) + 1;
436  meta_use = md;
437  meta = NULL;
438  if (fsize > 0)
439  {
441  GNUNET_FS_meta_data_insert (meta, "<gnunet>",
442  EXTRACTOR_METATYPE_GNUNET_FULL_DATA,
443  EXTRACTOR_METAFORMAT_BINARY, NULL, data,
444  fsize);
446  if ((slen + sizeof(uint32_t) + mdxs - 1) / DBLOCK_SIZE ==
447  (slen + sizeof(uint32_t) + mds - 1) / DBLOCK_SIZE)
448  {
449  /* adding full data would not cause us to cross
450  * additional blocks, so add it! */
451  meta_use = meta;
452  mds = mdxs;
453  }
454  }
455 
456  if (mds > GNUNET_MAX_MALLOC_CHECKED / 2)
457  mds = GNUNET_MAX_MALLOC_CHECKED / 2;
458  e = GNUNET_malloc (sizeof(struct BuilderEntry) + slen + mds
459  + sizeof(uint32_t));
460  serialized = (char *) &e[1];
461  GNUNET_memcpy (serialized, uris, slen);
462  GNUNET_free (uris);
463  sptr = &serialized[slen + sizeof(uint32_t)];
464  ret =
465  GNUNET_FS_meta_data_serialize (meta_use, &sptr, mds,
467  if (NULL != meta)
469  if (ret == -1)
470  mds = 0;
471  else
472  mds = ret;
473  big = htonl (mds);
474  GNUNET_memcpy (&serialized[slen], &big, sizeof(uint32_t));
475  e->len = slen + sizeof(uint32_t) + mds;
476  e->next = bld->head;
477  bld->head = e;
478  bld->count++;
479 }
480 
481 
487 static size_t
488 do_align (size_t start_position, size_t end_position)
489 {
490  size_t align;
491 
492  align = (end_position / DBLOCK_SIZE) * DBLOCK_SIZE;
493  if ((start_position < align) && (end_position > align))
494  return align + end_position - start_position;
495  return end_position;
496 }
497 
498 
508 static void
509 block_align (size_t start, unsigned int count, const size_t *sizes,
510  unsigned int *perm)
511 {
512  unsigned int i;
513  unsigned int j;
514  unsigned int tmp;
515  unsigned int best;
516  ssize_t badness;
517  size_t cpos;
518  size_t cend;
519  ssize_t cbad;
520  unsigned int cval;
521 
522  cpos = start;
523  for (i = 0; i < count; i++)
524  {
525  start = cpos;
526  badness = 0x7FFFFFFF;
527  best = -1;
528  for (j = i; j < count; j++)
529  {
530  cval = perm[j];
531  cend = cpos + sizes[cval];
532  if (cpos % DBLOCK_SIZE == 0)
533  {
534  /* prefer placing the largest blocks first */
535  cbad = -(cend % DBLOCK_SIZE);
536  }
537  else
538  {
539  if (cpos / DBLOCK_SIZE == cend / DBLOCK_SIZE)
540  {
541  /* Data fits into the same block! Prefer small left-overs! */
542  cbad = DBLOCK_SIZE - cend % DBLOCK_SIZE;
543  }
544  else
545  {
546  /* Would have to waste space to re-align, add big factor, this
547  * case is a real loss (proportional to space wasted)! */
548  cbad = DBLOCK_SIZE * (DBLOCK_SIZE - cpos % DBLOCK_SIZE);
549  }
550  }
551  if (cbad < badness)
552  {
553  best = j;
554  badness = cbad;
555  }
556  }
557  GNUNET_assert (best != -1);
558  tmp = perm[i];
559  perm[i] = perm[best];
560  perm[best] = tmp;
561  cpos += sizes[perm[i]];
562  cpos = do_align (start, cpos);
563  }
564 }
565 
566 
577 int
579  size_t *rsize,
580  void **rdata)
581 {
582  char *data;
583  char *sptr;
584  size_t *sizes;
585  unsigned int *perm;
586  unsigned int i;
587  unsigned int j;
588  struct BuilderEntry *pos;
589  struct BuilderEntry **bes;
590  size_t size;
591  size_t psize;
592  size_t off;
593  ssize_t ret;
594  uint32_t big;
595 
596  size = strlen (GNUNET_DIRECTORY_MAGIC) + sizeof(uint32_t);
598  sizes = NULL;
599  perm = NULL;
600  bes = NULL;
601  if (0 < bld->count)
602  {
603  sizes = GNUNET_new_array (bld->count,
604  size_t);
605  perm = GNUNET_new_array (bld->count,
606  unsigned int);
607  bes = GNUNET_new_array (bld->count,
608  struct BuilderEntry *);
609  pos = bld->head;
610  for (i = 0; i < bld->count; i++)
611  {
612  perm[i] = i;
613  bes[i] = pos;
614  sizes[i] = pos->len;
615  pos = pos->next;
616  }
617  block_align (size, bld->count, sizes, perm);
618  /* compute final size with alignment */
619  for (i = 0; i < bld->count; i++)
620  {
621  psize = size;
622  size += sizes[perm[i]];
623  size = do_align (psize, size);
624  }
625  }
626  *rsize = size;
628  if (data == NULL)
629  {
631  "malloc");
632  *rsize = 0;
633  *rdata = NULL;
634  GNUNET_free (sizes);
635  GNUNET_free (perm);
636  GNUNET_free (bes);
637  return GNUNET_SYSERR;
638  }
639  *rdata = data;
642  strlen (GNUNET_DIRECTORY_MAGIC));
643  off = strlen (GNUNET_DIRECTORY_MAGIC);
644 
645  sptr = &data[off + sizeof(uint32_t)];
646  ret =
648  &sptr,
649  size - off - sizeof(uint32_t),
651  GNUNET_assert (ret != -1);
652  big = htonl (ret);
653  GNUNET_memcpy (&data[off],
654  &big,
655  sizeof(uint32_t));
656  off += sizeof(uint32_t) + ret;
657  for (j = 0; j < bld->count; j++)
658  {
659  i = perm[j];
660  psize = off;
661  off += sizes[i];
662  off = do_align (psize, off);
663  GNUNET_memcpy (&data[off - sizes[i]], &(bes[i])[1], sizes[i]);
664  GNUNET_free (bes[i]);
665  }
666  GNUNET_free (sizes);
667  GNUNET_free (perm);
668  GNUNET_free (bes);
669  GNUNET_assert (off == size);
671  GNUNET_free (bld);
672  return GNUNET_OK;
673 }
674 
675 
676 /* end of fs_directory.c */
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:41
shared definitions for the FS library
#define MAX_INLINE_SIZE
Maximum size for a file to be considered for inlining in a directory.
Definition: fs_api.h:50
static int find_full_data(void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_len)
Type of a function that libextractor calls for each meta data item found.
Definition: fs_directory.c:136
static size_t do_align(size_t start_position, size_t end_position)
Given the start and end position of a block of data, return the end position of that data after align...
Definition: fs_directory.c:488
#define GNUNET_DIRECTORY_MAGIC
String that is used to indicate that a file is a GNUnet directory.
Definition: fs_directory.c:44
static void block_align(size_t start, unsigned int count, const size_t *sizes, unsigned int *perm)
Compute a permutation of the blocks to minimize the cost of alignment.
Definition: fs_directory.c:509
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static int start
Set if we are to start default services (including ARM).
Definition: gnunet-arm.c:39
static struct Experiment * e
static char * filename
uint32_t data
The data value.
static struct GNUNET_FS_Uri * uri
Value of URI provided on command-line (when not publishing a file but just creating UBlocks to refer ...
static struct GNUNET_FS_MetaData * meta
Meta-data provided via command-line option.
static char * plugin_name
Name of our plugin.
API for file sharing via GNUnet.
#define EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME
struct GNUNET_FS_Uri * GNUNET_FS_uri_parse(const char *uri, char **emsg)
Convert a UTF-8 String to a URI.
Definition: fs_uri.c:637
void GNUNET_FS_meta_data_make_directory(struct GNUNET_FS_MetaData *md)
Set the MIMETYPE information for the given metadata to "application/gnunet-directory".
Definition: fs_directory.c:81
int GNUNET_FS_directory_list_contents(size_t size, const void *data, uint64_t offset, GNUNET_FS_DirectoryEntryProcessor dep, void *dep_cls)
Iterate over all entries in a directory.
Definition: fs_directory.c:182
int GNUNET_FS_directory_builder_finish(struct GNUNET_FS_DirectoryBuilder *bld, size_t *rsize, void **rdata)
Finish building the directory.
Definition: fs_directory.c:578
uint64_t GNUNET_FS_uri_chk_get_file_size(const struct GNUNET_FS_Uri *uri)
What is the size of the file that this URI refers to?
Definition: fs_uri.c:1360
int GNUNET_FS_uri_test_ksk(const struct GNUNET_FS_Uri *uri)
Is this a keyword URI?
Definition: fs_uri.c:1324
#define GNUNET_FS_DIRECTORY_MAGIC
#define GNUNET_FS_DIRECTORY_MIME
struct GNUNET_FS_DirectoryBuilder * GNUNET_FS_directory_builder_create(const struct GNUNET_FS_MetaData *mdir)
Create a directory builder.
Definition: fs_directory.c:366
void(* GNUNET_FS_DirectoryEntryProcessor)(void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, const struct GNUNET_FS_MetaData *meta, size_t length, const void *data)
Function used to process entries in a directory.
char * GNUNET_FS_uri_to_string(const struct GNUNET_FS_Uri *uri)
Convert a URI to a UTF-8 String.
Definition: fs_uri.c:2017
void GNUNET_FS_uri_destroy(struct GNUNET_FS_Uri *uri)
Free URI.
Definition: fs_uri.c:677
int GNUNET_FS_uri_test_sks(const struct GNUNET_FS_Uri *uri)
Is this a namespace URI?
Definition: fs_uri.c:1271
int GNUNET_FS_meta_data_test_for_directory(const struct GNUNET_FS_MetaData *md)
Does the meta-data claim that this is a directory? Checks if the mime-type is that of a GNUnet direct...
Definition: fs_directory.c:55
int GNUNET_FS_uri_test_chk(const struct GNUNET_FS_Uri *uri)
Is this a file (or directory) URI?
Definition: fs_uri.c:1346
struct GNUNET_FS_Uri * GNUNET_FS_uri_loc_get_uri(const struct GNUNET_FS_Uri *uri)
Obtain the URI of the content itself.
Definition: fs_uri.c:843
void GNUNET_FS_directory_builder_add(struct GNUNET_FS_DirectoryBuilder *bld, const struct GNUNET_FS_Uri *uri, const struct GNUNET_FS_MetaData *md, const void *data)
Add an entry to a directory.
Definition: fs_directory.c:392
#define GNUNET_log(kind,...)
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
#define GNUNET_log_strerror(level, cmd)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
#define GNUNET_MAX_MALLOC_CHECKED
Maximum allocation with GNUNET_malloc macro.
#define GNUNET_malloc_large(size)
Wrapper around malloc.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_new_array(n, type)
Allocate a size n array with structs or unions of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
int GNUNET_FS_meta_data_iterate(const struct GNUNET_FS_MetaData *md, EXTRACTOR_MetaDataProcessor iter, void *iter_cls)
Iterate over MD entries.
Definition: meta_data.c:422
ssize_t GNUNET_FS_meta_data_get_serialized_size(const struct GNUNET_FS_MetaData *md)
Get the size of the full meta-data in serialized form.
Definition: meta_data.c:862
ssize_t GNUNET_FS_meta_data_serialize(const struct GNUNET_FS_MetaData *md, char **target, size_t max, enum GNUNET_FS_MetaDataSerializationOptions opt)
Serialize meta-data to target.
Definition: meta_data.c:641
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_create()
Create a fresh struct FS_MetaData token.
Definition: meta_data.c:131
char * GNUNET_FS_meta_data_get_by_type(const struct GNUNET_FS_MetaData *md, enum EXTRACTOR_MetaType type)
Get the first MD entry of the given type.
Definition: meta_data.c:442
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_duplicate(const struct GNUNET_FS_MetaData *md)
Duplicate a struct GNUNET_FS_MetaData.
Definition: meta_data.c:531
int GNUNET_FS_meta_data_insert(struct GNUNET_FS_MetaData *md, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_size)
Extend metadata.
Definition: meta_data.c:258
void GNUNET_FS_meta_data_destroy(struct GNUNET_FS_MetaData *md)
Free meta data.
Definition: meta_data.c:170
struct GNUNET_FS_MetaData * GNUNET_FS_meta_data_deserialize(const char *input, size_t size)
Deserialize meta-data.
Definition: meta_data.c:889
@ GNUNET_FS_META_DATA_SERIALIZE_FULL
Serialize all of the data.
@ GNUNET_FS_META_DATA_SERIALIZE_PART
If not enough space is available, it is acceptable to only serialize some of the metadata.
static unsigned int size
Size of the "table".
Definition: peer.c:68
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
Entries in the directory (builder).
Definition: fs_directory.c:326
size_t len
Length of this entry.
Definition: fs_directory.c:335
struct BuilderEntry * next
This is a linked list.
Definition: fs_directory.c:330
Internal state of a directory builder.
Definition: fs_directory.c:342
struct GNUNET_FS_MetaData * meta
Meta-data for the directory itself.
Definition: fs_directory.c:346
struct BuilderEntry * head
Head of linked list of entries.
Definition: fs_directory.c:351
unsigned int count
Number of entries in the directory.
Definition: fs_directory.c:356
Meta data to associate with a file, directory or namespace.
Definition: meta_data.c:96
A Universal Resource Identifier (URI), opaque.
Definition: fs_api.h:167
Closure for 'find_full_data'.
Definition: fs_directory.c:105
size_t size
Number of bytes stored in data.
Definition: fs_directory.c:114
void * data
Extracted binary meta data.
Definition: fs_directory.c:109
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model