GNUnet  0.11.x
fs_download.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2001-2012 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
25 #include "platform.h"
26 #include "gnunet_constants.h"
27 #include "gnunet_fs_service.h"
28 #include "fs_api.h"
29 #include "fs_tree.h"
30 
31 
36 static int
38 {
39  return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
41  ((NULL == dc->meta) &&
42  ((NULL == dc->filename) ||
43  ((strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
44  (NULL != strstr (dc->filename + strlen (dc->filename)
45  - strlen (GNUNET_FS_DIRECTORY_EXT),
47 }
48 
49 
66 static uint64_t
67 compute_disk_offset (uint64_t fsize, uint64_t off, unsigned int depth)
68 {
69  unsigned int i;
70  uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
71  uint64_t loff; /* where do IBlocks for depth "i" start? */
72  unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
73 
74  if (0 == depth)
75  return off;
76  /* first IBlocks start at the end of file, rounded up
77  * to full DBLOCK_SIZE */
78  loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
79  lsize =
80  ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof(struct ContentHashKey);
81  GNUNET_assert (0 == (off % DBLOCK_SIZE));
82  ioff = (off / DBLOCK_SIZE);
83  for (i = 1; i < depth; i++)
84  {
85  loff += lsize;
86  lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
87  GNUNET_assert (lsize > 0);
88  GNUNET_assert (0 == (ioff % CHK_PER_INODE));
89  ioff /= CHK_PER_INODE;
90  }
91  return loff + ioff * sizeof(struct ContentHashKey);
92 }
93 
94 
102 void
105 {
106  pi->value.download.dc = dc;
107  pi->value.download.cctx = dc->client_info;
108  pi->value.download.pctx =
109  (NULL == dc->parent) ? NULL : dc->parent->client_info;
110  pi->value.download.sctx =
111  (NULL == dc->search) ? NULL : dc->search->client_info;
112  pi->value.download.uri = dc->uri;
113  pi->value.download.filename = dc->filename;
114  pi->value.download.size = dc->length;
115  /* FIXME: Fix duration calculation to account for pauses */
116  pi->value.download.duration =
118  pi->value.download.completed = dc->completed;
119  pi->value.download.anonymity = dc->anonymity;
120  pi->value.download.eta =
122  pi->value.download.is_active = (NULL == dc->mq) ? GNUNET_NO : GNUNET_YES;
123  pi->fsh = dc->h;
124  if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
125  dc->client_info = dc->h->upcb (dc->h->upcb_cls, pi);
126  else
128 }
129 
130 
135 {
140 
144  const void *data;
145 
150 
155 
159  size_t size;
160 
165 
169  int do_store;
170 
174  uint32_t respect_offered;
175 
180 };
181 
182 
192 static int
193 process_result_with_request (void *cls,
194  const struct GNUNET_HashCode *key,
195  void *value);
196 
197 
211 static int
213  const struct ContentHashKey *chk,
214  struct DownloadRequest *dr,
215  const char *block,
216  size_t len,
217  int do_store)
218 {
219  struct ProcessResultClosure prc;
220  char enc[len];
223  struct GNUNET_HashCode query;
224 
225  GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
226  if (-1 == GNUNET_CRYPTO_symmetric_encrypt (block, len, &sk, &iv, enc))
227  {
228  GNUNET_break (0);
229  return GNUNET_SYSERR;
230  }
231  GNUNET_CRYPTO_hash (enc, len, &query);
232  if (0 != memcmp (&query, &chk->query, sizeof(struct GNUNET_HashCode)))
233  {
234  GNUNET_break_op (0);
235  return GNUNET_SYSERR;
236  }
237  GNUNET_log (
239  "Matching %u byte block for `%s' at offset %llu already present, no need for download!\n",
240  (unsigned int) len,
241  dc->filename,
242  (unsigned long long) dr->offset);
243  /* already got it! */
244  prc.dc = dc;
245  prc.data = enc;
246  prc.size = len;
247  prc.type = (0 == dr->depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK
249  prc.query = chk->query;
250  prc.do_store = do_store;
252  process_result_with_request (&prc, &chk->key, dr);
253  return GNUNET_OK;
254 }
255 
256 
264 static void
266 
267 
279 static void
280 trigger_recursive_download (void *cls,
281  const char *filename,
282  const struct GNUNET_FS_Uri *uri,
283  const struct GNUNET_CONTAINER_MetaData *meta,
284  size_t length,
285  const void *data);
286 
287 
294 static void
296 {
297  size_t size;
298  uint64_t size64;
299  void *data;
300  struct GNUNET_DISK_FileHandle *h;
301  struct GNUNET_DISK_MapHandle *m;
302 
303  size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
304  size = (size_t) size64;
305  if (size64 != (uint64_t) size)
306  {
307  GNUNET_log (
309  _ (
310  "Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
311  return;
312  }
313  if (NULL != dc->filename)
314  {
318  }
319  else
320  {
321  GNUNET_assert (NULL != dc->temp_filename);
325  }
326  if (NULL == h)
327  return; /* oops */
328  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
329  if (NULL == data)
330  {
332  _ ("Directory too large for system address space\n"));
333  }
334  else
335  {
336  if (GNUNET_OK !=
338  data,
339  0,
341  dc))
342  {
343  GNUNET_log (
345  _ (
346  "Failed to access full directroy contents of `%s' for recursive download\n"),
347  dc->filename);
348  }
350  }
352  if (NULL == dc->filename)
353  {
354  if (0 != unlink (dc->temp_filename))
356  "unlink",
357  dc->temp_filename);
359  dc->temp_filename = NULL;
360  }
361 }
362 
363 
374 static void
376 {
377  struct GNUNET_FS_ProgressInfo pi;
378  struct GNUNET_FS_DownloadContext *pos;
379 
380  /* first, check if we need to download children */
381  if (is_recursive_download (dc))
383  /* then, check if children are done already */
384  for (pos = dc->child_head; NULL != pos; pos = pos->next)
385  {
386  if ((NULL == pos->emsg) && (pos->completed < pos->length))
387  return; /* not done yet */
388  if ((NULL != pos->child_head) && (pos->has_finished != GNUNET_YES))
389  return; /* not transitively done yet */
390  }
391  /* All of our children are done, so mark this download done */
392  dc->has_finished = GNUNET_YES;
393  if (NULL != dc->job_queue)
394  {
396  dc->job_queue = NULL;
397  }
398  if (NULL != dc->task)
399  {
401  dc->task = NULL;
402  }
403  if (NULL != dc->rfh)
404  {
406  dc->rfh = NULL;
407  }
409 
410  /* signal completion */
413 
414  /* let parent know */
415  if (NULL != dc->parent)
416  check_completed (dc->parent);
417 }
418 
419 
430 static void
432  struct DownloadRequest *dr,
433  const char *data,
434  size_t data_len)
435 {
436  struct GNUNET_FS_ProgressInfo pi;
437  unsigned int i;
438  char enc[DBLOCK_SIZE];
439  struct ContentHashKey chks[CHK_PER_INODE];
440  struct ContentHashKey in_chk;
443  size_t dlen;
444  struct DownloadRequest *drc;
445  struct GNUNET_DISK_FileHandle *fh;
446  int complete;
447  const char *fn;
448  const char *odata;
449  size_t odata_len;
450 
451  odata = data;
452  odata_len = data_len;
453  if (BRS_DOWNLOAD_UP == dr->state)
454  return;
455  if (dr->depth > 0)
456  {
457  if ((dc->offset > 0) ||
458  (dc->length < GNUNET_ntohll (dc->uri->data.chk.file_length)))
459  {
460  /* NOTE: this test is not tight, but should suffice; the issue
461  here is that 'dr->num_children' may inherently only specify a
462  smaller range than what is in the original file;
463  thus, reconstruction of (some) inner blocks will fail.
464  FIXME: we might eventually want to write a tighter test to
465  maximize the circumstances under which we do succeed with
466  IBlock reconstruction. (need good tests though). */return;
467  }
468  complete = GNUNET_YES;
469  for (i = 0; i < dr->num_children; i++)
470  {
471  drc = dr->children[i];
472  try_match_block (dc, drc, data, data_len);
473  if (drc->state != BRS_RECONSTRUCT_META_UP)
474  complete = GNUNET_NO;
475  else
476  chks[i] = drc->chk;
477  }
478  if (GNUNET_YES != complete)
479  return;
480  data = (const char *) chks;
481  dlen = dr->num_children * sizeof(struct ContentHashKey);
482  }
483  else
484  {
485  if (dr->offset > data_len)
486  return; /* oops */
487  dlen = GNUNET_MIN (data_len - dr->offset, DBLOCK_SIZE);
488  }
489  GNUNET_CRYPTO_hash (&data[dr->offset], dlen, &in_chk.key);
490  GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
491  if (-1 ==
492  GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset], dlen, &sk, &iv, enc))
493  {
494  GNUNET_break (0);
495  return;
496  }
497  GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query);
498  switch (dr->state)
499  {
500  case BRS_INIT:
501  dr->chk = in_chk;
503  break;
504 
505  case BRS_CHK_SET:
506  if (0 != memcmp (&in_chk, &dr->chk, sizeof(struct ContentHashKey)))
507  {
508  /* other peer provided bogus meta data */
509  GNUNET_break_op (0);
510  break;
511  }
512  /* write block to disk */
513  fn = (NULL != dc->filename) ? dc->filename : dc->temp_filename;
514  if (NULL != fn)
515  {
516  fh = GNUNET_DISK_file_open (fn,
524  if (NULL == fh)
525  {
527  GNUNET_asprintf (&dc->emsg,
528  _ ("Failed to open file `%s' for writing"),
529  fn);
531  dr->state = BRS_ERROR;
533  pi.value.download.specifics.error.message = dc->emsg;
535  return;
536  }
537  if (data_len != GNUNET_DISK_file_write (fh, odata, odata_len))
538  {
540  GNUNET_asprintf (&dc->emsg,
541  _ ("Failed to open file `%s' for writing"),
542  fn);
544  dr->state = BRS_ERROR;
546  pi.value.download.specifics.error.message = dc->emsg;
548  return;
549  }
551  }
552  /* signal success */
553  dr->state = BRS_DOWNLOAD_UP;
554  dc->completed = dc->length;
557  pi.value.download.specifics.progress.data = data;
558  pi.value.download.specifics.progress.offset = 0;
559  pi.value.download.specifics.progress.data_len = dlen;
560  pi.value.download.specifics.progress.depth = 0;
561  pi.value.download.specifics.progress.respect_offered = 0;
562  pi.value.download.specifics.progress.block_download_duration =
565  if ((NULL != dc->filename) &&
566  (0 != truncate (dc->filename,
569  "truncate",
570  dc->filename);
571  check_completed (dc);
572  break;
573 
574  default:
575  /* how did we get here? */
576  GNUNET_break (0);
577  break;
578  }
579 }
580 
581 
600 static int
601 match_full_data (void *cls,
602  const char *plugin_name,
604  enum EXTRACTOR_MetaFormat format,
605  const char *data_mime_type,
606  const char *data,
607  size_t data_len)
608 {
609  struct GNUNET_FS_DownloadContext *dc = cls;
610 
612  return 0;
614  "Found %u bytes of FD!\n",
615  (unsigned int) data_len);
616  if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len)
617  {
618  GNUNET_break_op (0);
619  return 1; /* bogus meta data */
620  }
621  try_match_block (dc, dc->top_request, data, data_len);
622  return 1;
623 }
624 
625 
632 static void
634 {
635  unsigned int i;
636 
637  do
638  {
639  dr->state = BRS_DOWNLOAD_UP;
640  dr = dr->parent;
641  if (NULL == dr)
642  break;
643  for (i = 0; i < dr->num_children; i++)
644  if (dr->children[i]->state != BRS_DOWNLOAD_UP)
645  break;
646  }
647  while (i == dr->num_children);
648 }
649 
650 
661 static void
663  struct DownloadRequest *dr)
664 {
665  uint64_t off;
666  char block[DBLOCK_SIZE];
667  struct GNUNET_HashCode key;
668  uint64_t total;
669  size_t len;
670  unsigned int i;
671  struct DownloadRequest *drc;
672  uint64_t child_block_size;
673  const struct ContentHashKey *chks;
674  int up_done;
675 
676  GNUNET_assert (NULL != dc->rfh);
678  total = GNUNET_FS_uri_chk_get_file_size (dc->uri);
679  GNUNET_assert (dr->depth < dc->treedepth);
680  len = GNUNET_FS_tree_calculate_block_size (total, dr->offset, dr->depth);
681  GNUNET_assert (len <= DBLOCK_SIZE);
682  off = compute_disk_offset (total, dr->offset, dr->depth);
683  if (dc->old_file_size < off + len)
684  return; /* failure */
685  if (off != GNUNET_DISK_file_seek (dc->rfh, off, GNUNET_DISK_SEEK_SET))
686  {
688  return; /* failure */
689  }
690  if (len != GNUNET_DISK_file_read (dc->rfh, block, len))
691  {
693  return; /* failure */
694  }
695  GNUNET_CRYPTO_hash (block, len, &key);
696  if (0 != memcmp (&key, &dr->chk.key, sizeof(struct GNUNET_HashCode)))
697  return; /* mismatch */
698  if (GNUNET_OK !=
699  encrypt_existing_match (dc, &dr->chk, dr, block, len, GNUNET_NO))
700  {
701  /* hash matches but encrypted block does not, really bad */
702  dr->state = BRS_ERROR;
703  /* propagate up */
704  while (NULL != dr->parent)
705  {
706  dr = dr->parent;
707  dr->state = BRS_ERROR;
708  }
709  return;
710  }
711  /* block matches */
712  dr->state = BRS_DOWNLOAD_DOWN;
713 
714  /* set CHKs for children */
715  up_done = GNUNET_YES;
716  chks = (const struct ContentHashKey *) block;
717  for (i = 0; i < dr->num_children; i++)
718  {
719  drc = dr->children[i];
720  GNUNET_assert (drc->offset >= dr->offset);
721  child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
722  GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
723  if (BRS_INIT == drc->state)
724  {
725  drc->state = BRS_CHK_SET;
726  drc->chk = chks[drc->chk_idx];
727  try_top_down_reconstruction (dc, drc);
728  }
729  if (BRS_DOWNLOAD_UP != drc->state)
730  up_done = GNUNET_NO; /* children not all done */
731  }
732  if (GNUNET_YES == up_done)
733  propagate_up (dr); /* children all done (or no children...) */
734 }
735 
736 
745 static int
746 retry_entry (void *cls, const struct GNUNET_HashCode *key, void *entry)
747 {
748  struct GNUNET_FS_DownloadContext *dc = cls;
749  struct DownloadRequest *dr = entry;
750  struct SearchMessage *sm;
751  struct GNUNET_MQ_Envelope *env;
752 
756  else
758  if (0 == dr->depth)
759  sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK);
760  else
761  sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK);
762  sm->anonymity_level = htonl (dc->anonymity);
763  sm->target = dc->target;
764  sm->query = dr->chk.query;
765  GNUNET_MQ_send (dc->mq, env);
766  return GNUNET_OK;
767 }
768 
769 
776 static void
778  struct DownloadRequest *dr)
779 {
780  unsigned int i;
781 
782  switch (dr->state)
783  {
784  case BRS_INIT:
785  GNUNET_assert (0);
786  break;
787 
789  GNUNET_assert (0);
790  break;
791 
793  GNUNET_assert (0);
794  break;
795 
796  case BRS_RECONSTRUCT_UP:
797  GNUNET_assert (0);
798  break;
799 
800  case BRS_CHK_SET:
801  /* normal case, start download */
802  break;
803 
804  case BRS_DOWNLOAD_DOWN:
805  for (i = 0; i < dr->num_children; i++)
806  schedule_block_download (dc, dr->children[i]);
807  return;
808 
809  case BRS_DOWNLOAD_UP:
810  /* We're done! */
811  return;
812 
813  case BRS_ERROR:
814  GNUNET_break (0);
815  return;
816  }
818  "Scheduling download at offset %llu and depth %u for `%s'\n",
819  (unsigned long long) dr->offset,
820  dr->depth,
821  GNUNET_h2s (&dr->chk.query));
823  &dr->chk.query,
824  dr))
825  return; /* already active */
827  &dr->chk.query,
828  dr,
830  if (NULL == dc->mq)
831  return; /* download not active */
832  retry_entry (dc, &dr->chk.query, dr);
833 }
834 
835 
836 #define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX
837 
849 static void
851  const char *filename,
852  const struct GNUNET_FS_Uri *uri,
853  const struct GNUNET_CONTAINER_MetaData *meta,
854  size_t length,
855  const void *data)
856 {
857  struct GNUNET_FS_DownloadContext *dc = cls;
858  struct GNUNET_FS_DownloadContext *cpos;
859  char *temp_name;
860  char *fn;
861  char *us;
862  char *ext;
863  char *dn;
864  char *pos;
865  char *full_name;
866  char *sfn;
867 
868  if (NULL == uri)
869  return; /* entry for the directory itself */
870  cpos = dc->child_head;
871  while (NULL != cpos)
872  {
873  if ((GNUNET_FS_uri_test_equal (uri, cpos->uri)) ||
874  ((NULL != filename) && (0 == strcmp (cpos->filename, filename))))
875  break;
876  cpos = cpos->next;
877  }
878  if (NULL != cpos)
879  return; /* already exists */
880  fn = NULL;
881  if (NULL == filename)
882  {
884  if (NULL == fn)
885  {
886  us = GNUNET_FS_uri_to_string (uri);
887  fn = GNUNET_strdup (&us[strlen (GNUNET_FS_URI_CHK_PREFIX)]);
888  GNUNET_free (us);
889  }
890  else if ('.' == fn[0])
891  {
892  ext = fn;
893  us = GNUNET_FS_uri_to_string (uri);
894  GNUNET_asprintf (&fn,
895  "%s%s",
896  &us[strlen (GNUNET_FS_URI_CHK_PREFIX)],
897  ext);
898  GNUNET_free (ext);
899  GNUNET_free (us);
900  }
901  /* change '\' to '/' (this should have happened
902  * during insertion, but malicious peers may
903  * not have done this) */
904  while (NULL != (pos = strstr (fn, "\\")))
905  *pos = '/';
906  /* remove '../' everywhere (again, well-behaved
907  * peers don't do this, but don't trust that
908  * we did not get something nasty) */
909  while (NULL != (pos = strstr (fn, "../")))
910  {
911  pos[0] = '_';
912  pos[1] = '_';
913  pos[2] = '_';
914  }
915  filename = fn;
916  }
917  if (NULL == dc->filename)
918  {
919  full_name = NULL;
920  }
921  else
922  {
923  dn = GNUNET_strdup (dc->filename);
924  GNUNET_break (
925  (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
926  (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT),
928  sfn = GNUNET_strdup (filename);
929  while ((strlen (sfn) > 0) && ('/' == filename[strlen (sfn) - 1]))
930  sfn[strlen (sfn) - 1] = '\0';
931  if ((strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
932  (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT),
934  dn[strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
936  ((strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
937  (NULL == strstr (filename + strlen (filename)
938  - strlen (GNUNET_FS_DIRECTORY_EXT),
940  {
941  GNUNET_asprintf (&full_name,
942  "%s%s%s%s",
943  dn,
945  sfn,
947  }
948  else
949  {
950  GNUNET_asprintf (&full_name, "%s%s%s", dn, DIR_SEPARATOR_STR, sfn);
951  }
952  GNUNET_free (sfn);
953  GNUNET_free (dn);
954  }
955  if ((NULL != full_name) &&
957  {
959  _ (
960  "Failed to create directory for recursive download of `%s'\n"),
961  full_name);
962  GNUNET_free (full_name);
964  return;
965  }
966 
967  temp_name = NULL;
969  "Triggering recursive download of size %llu with %u bytes MD\n",
970  (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
972  meta));
974  uri,
975  meta,
976  full_name,
977  temp_name,
978  0,
980  dc->anonymity,
981  dc->options,
982  NULL,
983  dc);
984  GNUNET_free_non_null (full_name);
985  GNUNET_free_non_null (temp_name);
987 }
988 
989 
995 void
997 {
998  if (NULL == dr)
999  return;
1000  for (unsigned int i = 0; i < dr->num_children; i++)
1003  GNUNET_free (dr);
1004 }
1005 
1006 
1016 static int
1018  const struct GNUNET_HashCode *key,
1019  void *value)
1020 {
1021  struct ProcessResultClosure *prc = cls;
1022  struct DownloadRequest *dr = value;
1023  struct GNUNET_FS_DownloadContext *dc = prc->dc;
1024  struct DownloadRequest *drc;
1025  struct GNUNET_DISK_FileHandle *fh = NULL;
1028  char pt[prc->size];
1029  struct GNUNET_FS_ProgressInfo pi;
1030  uint64_t off;
1031  size_t bs;
1032  size_t app;
1033  int i;
1034  struct ContentHashKey *chkarr;
1035 
1036  GNUNET_log (
1038  "Received %u byte block `%s' matching pending request at depth %u and offset %llu/%llu\n",
1039  (unsigned int) prc->size,
1040  GNUNET_h2s (key),
1041  dr->depth,
1042  (unsigned long long) dr->offset,
1043  (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
1045  dc->uri->data.chk.file_length),
1046  dr->offset,
1047  dr->depth);
1048  if (prc->size != bs)
1049  {
1050  GNUNET_asprintf (
1051  &dc->emsg,
1052  _ (
1053  "Internal error or bogus download URI (expected %u bytes at depth %u and offset %llu/%llu, got %u bytes)"),
1054  bs,
1055  dr->depth,
1056  (unsigned long long) dr->offset,
1057  (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length),
1058  prc->size);
1059  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "%s\n", dc->emsg);
1060  while (NULL != dr->parent)
1061  {
1062  dr->state = BRS_ERROR;
1063  dr = dr->parent;
1064  }
1065  dr->state = BRS_ERROR;
1066  goto signal_error;
1067  }
1068 
1069  (void) GNUNET_CONTAINER_multihashmap_remove (dc->active, &prc->query, dr);
1070  GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv);
1071  if (-1 ==
1072  GNUNET_CRYPTO_symmetric_decrypt (prc->data, prc->size, &skey, &iv, pt))
1073  {
1074  GNUNET_break (0);
1075  dc->emsg = GNUNET_strdup (_ ("internal error decrypting content"));
1076  goto signal_error;
1077  }
1079  dr->offset,
1080  dr->depth);
1081  /* save to disk */
1082  if ((GNUNET_YES == prc->do_store) &&
1083  ((NULL != dc->filename) || (is_recursive_download (dc))) &&
1084  ((dr->depth == dc->treedepth) ||
1086  {
1087  fh = GNUNET_DISK_file_open (NULL != dc->filename ? dc->filename
1088  : dc->temp_filename,
1095  if (NULL == fh)
1096  {
1097  GNUNET_asprintf (&dc->emsg,
1098  _ ("Download failed: could not open file `%s': %s"),
1099  dc->filename,
1100  strerror (errno));
1101  goto signal_error;
1102  }
1104  "Saving decrypted block to disk at offset %llu\n",
1105  (unsigned long long) off);
1106  if ((off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)))
1107  {
1108  GNUNET_asprintf (&dc->emsg,
1109  _ ("Failed to seek to offset %llu in file `%s': %s"),
1110  (unsigned long long) off,
1111  dc->filename,
1112  strerror (errno));
1113  goto signal_error;
1114  }
1115  if (prc->size != GNUNET_DISK_file_write (fh, pt, prc->size))
1116  {
1117  GNUNET_asprintf (
1118  &dc->emsg,
1119  _ ("Failed to write block of %u bytes at offset %llu in file `%s': %s"),
1120  (unsigned int) prc->size,
1121  (unsigned long long) off,
1122  dc->filename,
1123  strerror (errno));
1124  goto signal_error;
1125  }
1127  fh = NULL;
1128  }
1129 
1130  if (0 == dr->depth)
1131  {
1132  /* DBLOCK, update progress and try recursion if applicable */
1133  app = prc->size;
1134  if (dr->offset < dc->offset)
1135  {
1136  /* starting offset begins in the middle of pt,
1137  * do not count first bytes as progress */
1138  GNUNET_assert (app > (dc->offset - dr->offset));
1139  app -= (dc->offset - dr->offset);
1140  }
1141  if (dr->offset + prc->size > dc->offset + dc->length)
1142  {
1143  /* end of block is after relevant range,
1144  * do not count last bytes as progress */
1145  GNUNET_assert (app >
1146  (dr->offset + prc->size) - (dc->offset + dc->length));
1147  app -= (dr->offset + prc->size) - (dc->offset + dc->length);
1148  }
1149  dc->completed += app;
1150 
1151  /* do recursive download if option is set and either meta data
1152  * says it is a directory or if no meta data is given AND filename
1153  * ends in '.gnd' (top-level case) */
1154  if (is_recursive_download (dc))
1156  pt,
1157  off,
1159  dc);
1160  }
1161  GNUNET_assert (dc->completed <= dc->length);
1162  dr->state = BRS_DOWNLOAD_DOWN;
1164  pi.value.download.specifics.progress.data = pt;
1165  pi.value.download.specifics.progress.offset = dr->offset;
1166  pi.value.download.specifics.progress.data_len = prc->size;
1167  pi.value.download.specifics.progress.depth = dr->depth;
1168  pi.value.download.specifics.progress.respect_offered = prc->respect_offered;
1169  pi.value.download.specifics.progress.num_transmissions =
1170  prc->num_transmissions;
1171  if (prc->last_transmission.abs_value_us !=
1172  GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us)
1173  pi.value.download.specifics.progress.block_download_duration =
1175  else
1176  pi.value.download.specifics.progress.block_download_duration =
1177  GNUNET_TIME_UNIT_ZERO; /* found locally */
1179  if (0 == dr->depth)
1180  propagate_up (dr);
1181 
1182  if (dc->completed == dc->length)
1183  {
1184  /* download completed, signal */
1186  "Download completed, truncating file to desired length %llu\n",
1187  (unsigned long long) GNUNET_ntohll (
1188  dc->uri->data.chk.file_length));
1189  /* truncate file to size (since we store IBlocks at the end) */
1190  if (NULL != dc->filename)
1191  {
1192  if (0 != truncate (dc->filename,
1195  "truncate",
1196  dc->filename);
1197  }
1198  GNUNET_assert (0 == dr->depth);
1199  check_completed (dc);
1200  }
1201  if (0 == dr->depth)
1202  {
1203  /* bottom of the tree, no child downloads possible, just sync */
1205  return GNUNET_YES;
1206  }
1207 
1208  GNUNET_log (
1210  "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
1211  dr->depth,
1212  (unsigned long long) dr->offset);
1213  GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
1214  chkarr = (struct ContentHashKey *) pt;
1215  for (i = dr->num_children - 1; i >= 0; i--)
1216  {
1217  drc = dr->children[i];
1218  switch (drc->state)
1219  {
1220  case BRS_INIT:
1221  if ((drc->chk_idx + 1) * sizeof(struct ContentHashKey) > prc->size)
1222  {
1223  /* 'chkarr' does not have enough space for this chk_idx;
1224  internal error! */
1225  GNUNET_break (0);
1226  GNUNET_assert (0);
1227  dc->emsg = GNUNET_strdup (_ ("internal error decoding tree"));
1228  goto signal_error;
1229  }
1230  drc->chk = chkarr[drc->chk_idx];
1231  drc->state = BRS_CHK_SET;
1232  if (GNUNET_YES == dc->issue_requests)
1233  schedule_block_download (dc, drc);
1234  break;
1235 
1236  case BRS_RECONSTRUCT_DOWN:
1237  GNUNET_assert (0);
1238  break;
1239 
1241  GNUNET_assert (0);
1242  break;
1243 
1244  case BRS_RECONSTRUCT_UP:
1245  GNUNET_assert (0);
1246  break;
1247 
1248  case BRS_CHK_SET:
1249  GNUNET_assert (0);
1250  break;
1251 
1252  case BRS_DOWNLOAD_DOWN:
1253  GNUNET_assert (0);
1254  break;
1255 
1256  case BRS_DOWNLOAD_UP:
1257  GNUNET_assert (0);
1258  break;
1259 
1260  case BRS_ERROR:
1261  GNUNET_assert (0);
1262  break;
1263 
1264  default:
1265  GNUNET_assert (0);
1266  break;
1267  }
1268  }
1270  return GNUNET_YES;
1271 
1272 signal_error:
1273  if (NULL != fh)
1276  pi.value.download.specifics.error.message = dc->emsg;
1278  GNUNET_MQ_destroy (dc->mq);
1279  dc->mq = NULL;
1281  dc->top_request = NULL;
1282  if (NULL != dc->job_queue)
1283  {
1285  dc->job_queue = NULL;
1286  }
1288  return GNUNET_NO;
1289 }
1290 
1291 
1299 static int
1300 check_put (void *cls, const struct ClientPutMessage *cm)
1301 {
1302  /* any varsize length is OK */
1303  return GNUNET_OK;
1304 }
1305 
1306 
1314 static void
1315 handle_put (void *cls, const struct ClientPutMessage *cm)
1316 {
1317  struct GNUNET_FS_DownloadContext *dc = cls;
1318  uint16_t msize = ntohs (cm->header.size) - sizeof(*cm);
1319  struct ProcessResultClosure prc;
1320 
1321  prc.dc = dc;
1322  prc.data = &cm[1];
1324  prc.size = msize;
1325  prc.type = ntohl (cm->type);
1326  prc.do_store = GNUNET_YES;
1327  prc.respect_offered = ntohl (cm->respect_offered);
1328  prc.num_transmissions = ntohl (cm->num_transmissions);
1329  GNUNET_CRYPTO_hash (prc.data, msize, &prc.query);
1331  "Received result for query `%s' from FS service\n",
1332  GNUNET_h2s (&prc.query));
1334  &prc.query,
1336  &prc);
1337 }
1338 
1339 
1348 static void
1350 {
1351  struct GNUNET_FS_DownloadContext *dc = cls;
1352 
1353  if (NULL != dc->mq)
1354  {
1355  GNUNET_MQ_destroy (dc->mq);
1356  dc->mq = NULL;
1357  }
1359  "Transmitting download request failed, trying to reconnect\n");
1360  try_reconnect (dc);
1361 }
1362 
1363 
1369 static void
1370 do_reconnect (void *cls)
1371 {
1372  struct GNUNET_FS_DownloadContext *dc = cls;
1373  struct GNUNET_MQ_MessageHandler handlers[] =
1374  { GNUNET_MQ_hd_var_size (put,
1376  struct ClientPutMessage,
1377  dc),
1378  GNUNET_MQ_handler_end () };
1379 
1380  dc->task = NULL;
1381  dc->mq = GNUNET_CLIENT_connect (dc->h->cfg,
1382  "fs",
1383  handlers,
1385  dc);
1386  if (NULL == dc->mq)
1387  {
1389  "Connecting to `%s'-service failed, will try again.\n",
1390  "FS");
1391  try_reconnect (dc);
1392  return;
1393  }
1395 }
1396 
1397 
1405 static void
1407 {
1408  if (NULL != dc->mq)
1409  {
1411  "Moving all requests back to pending list\n");
1412  GNUNET_MQ_destroy (dc->mq);
1413  dc->mq = NULL;
1414  }
1415  if (0 == dc->reconnect_backoff.rel_value_us)
1417  else
1419 
1421  "Will try to reconnect in %s\n",
1423  GNUNET_YES));
1424  GNUNET_break (NULL != dc->job_queue);
1425  dc->task =
1427 }
1428 
1429 
1436 static void
1438 {
1439  struct GNUNET_FS_DownloadContext *dc = cls;
1440  struct GNUNET_FS_ProgressInfo pi;
1441 
1442  GNUNET_assert (NULL == dc->mq);
1443  GNUNET_assert (NULL != dc->active);
1444  do_reconnect (dc);
1445  if (NULL != dc->mq)
1446  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download activated\n");
1449 }
1450 
1451 
1457 static void
1459 {
1460  struct GNUNET_FS_DownloadContext *dc = cls;
1461  struct GNUNET_FS_ProgressInfo pi;
1462 
1463  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download deactivated\n");
1464  if (NULL != dc->mq)
1465  {
1466  GNUNET_MQ_destroy (dc->mq);
1467  dc->mq = NULL;
1468  }
1471 }
1472 
1473 
1493 static struct DownloadRequest *
1495  unsigned int chk_idx,
1496  unsigned int depth,
1497  uint64_t dr_offset,
1498  uint64_t file_start_offset,
1499  uint64_t desired_length)
1500 {
1501  struct DownloadRequest *dr;
1502  unsigned int i;
1503  unsigned int head_skip;
1504  uint64_t child_block_size;
1505 
1506  dr = GNUNET_new (struct DownloadRequest);
1507  dr->parent = parent;
1508  dr->depth = depth;
1509  dr->offset = dr_offset;
1510  dr->chk_idx = chk_idx;
1511  if (0 == depth)
1512  return dr;
1513  child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1);
1514 
1515  /* calculate how many blocks at this level are not interesting
1516  * from the start (rounded down), either because of the requested
1517  * file offset or because this IBlock is further along */
1518  if (dr_offset < file_start_offset)
1519  {
1520  head_skip = (file_start_offset - dr_offset) / child_block_size;
1521  }
1522  else
1523  {
1524  head_skip = 0;
1525  }
1526 
1527  /* calculate index of last block at this level that is interesting (rounded up) */
1528  dr->num_children =
1529  (file_start_offset + desired_length - dr_offset) / child_block_size;
1530  if (dr->num_children * child_block_size <
1531  file_start_offset + desired_length - dr_offset)
1532  dr->num_children++; /* round up */
1533  GNUNET_assert (dr->num_children > head_skip);
1534  dr->num_children -= head_skip;
1535  if (dr->num_children > CHK_PER_INODE)
1536  dr->num_children = CHK_PER_INODE; /* cap at max */
1538  "Block at offset %llu and depth %u has %u children\n",
1539  (unsigned long long) dr_offset,
1540  depth,
1541  dr->num_children);
1542 
1543  /* now we can get the total number of *interesting* children for this block */
1544 
1545  /* why else would we have gotten here to begin with? (that'd be a bad logic error) */
1546  GNUNET_assert (dr->num_children > 0);
1547 
1548  dr->children = GNUNET_new_array (dr->num_children, struct DownloadRequest *);
1549  for (i = 0; i < dr->num_children; i++)
1550  {
1551  dr->children[i] =
1553  i + head_skip,
1554  depth - 1,
1555  dr_offset + (i + head_skip) * child_block_size,
1556  file_start_offset,
1557  desired_length);
1558  }
1559  return dr;
1560 }
1561 
1562 
1569 static void
1570 reconstruct_cont (void *cls)
1571 {
1572  struct GNUNET_FS_DownloadContext *dc = cls;
1573 
1574  /* clean up state from tree encoder */
1575  if (NULL != dc->task)
1576  {
1578  dc->task = NULL;
1579  }
1580  if (NULL != dc->rfh)
1581  {
1583  dc->rfh = NULL;
1584  }
1585  /* start "normal" download */
1586  dc->issue_requests = GNUNET_YES;
1587  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting normal download\n");
1589 }
1590 
1591 
1597 static void
1598 get_next_block (void *cls)
1599 {
1600  struct GNUNET_FS_DownloadContext *dc = cls;
1601 
1602  dc->task = NULL;
1604 }
1605 
1606 
1624 static void
1625 reconstruct_cb (void *cls,
1626  const struct ContentHashKey *chk,
1627  uint64_t offset,
1628  unsigned int depth,
1629  enum GNUNET_BLOCK_Type type,
1630  const void *block,
1631  uint16_t block_size)
1632 {
1633  struct GNUNET_FS_DownloadContext *dc = cls;
1634  struct GNUNET_FS_ProgressInfo pi;
1635  struct DownloadRequest *dr;
1636  uint64_t blen;
1637  unsigned int chld;
1638 
1639  /* find corresponding request entry */
1640  dr = dc->top_request;
1641  while (dr->depth > depth)
1642  {
1643  GNUNET_assert (dr->num_children > 0);
1644  blen = GNUNET_FS_tree_compute_tree_size (dr->depth - 1);
1645  chld = (offset - dr->offset) / blen;
1646  if (chld < dr->children[0]->chk_idx)
1647  {
1649  "Block %u < %u irrelevant for our range\n",
1650  chld,
1651  dr->children[0]->chk_idx);
1653  return; /* irrelevant block */
1654  }
1655  if (chld > dr->children[dr->num_children - 1]->chk_idx)
1656  {
1658  "Block %u > %u irrelevant for our range\n",
1659  chld,
1660  dr->children[dr->num_children - 1]->chk_idx);
1662  return; /* irrelevant block */
1663  }
1664  dr = dr->children[chld - dr->children[0]->chk_idx];
1665  }
1666  GNUNET_log (
1668  "Matched TE block with request at offset %llu and depth %u in state %d\n",
1669  (unsigned long long) dr->offset,
1670  dr->depth,
1671  dr->state);
1672  /* FIXME: this code needs more testing and might
1673  need to handle more states... */
1674  switch (dr->state)
1675  {
1676  case BRS_INIT:
1677  break;
1678 
1679  case BRS_RECONSTRUCT_DOWN:
1680  break;
1681 
1683  break;
1684 
1685  case BRS_RECONSTRUCT_UP:
1686  break;
1687 
1688  case BRS_CHK_SET:
1689  if (0 == memcmp (chk, &dr->chk, sizeof(struct ContentHashKey)))
1690  {
1691  GNUNET_log (
1693  "Reconstruction succeeded, can use block at offset %llu, depth %u\n",
1694  (unsigned long long) offset,
1695  depth);
1696  /* block matches, hence tree below matches;
1697  * this request is done! */
1698  dr->state = BRS_DOWNLOAD_UP;
1700  &dr->chk.query,
1701  dr);
1702  /* calculate how many bytes of payload this block
1703  * corresponds to */
1705  /* how many of those bytes are in the requested range? */
1706  blen = GNUNET_MIN (blen, dc->length + dc->offset - dr->offset);
1707  /* signal progress */
1708  dc->completed += blen;
1710  pi.value.download.specifics.progress.data = NULL;
1711  pi.value.download.specifics.progress.offset = offset;
1712  pi.value.download.specifics.progress.data_len = 0;
1713  pi.value.download.specifics.progress.depth = 0;
1714  pi.value.download.specifics.progress.respect_offered = 0;
1715  pi.value.download.specifics.progress.block_download_duration =
1718  /* FIXME: duplicated code from 'process_result_with_request - refactor */
1719  if (dc->completed == dc->length)
1720  {
1721  /* download completed, signal */
1723  "Download completed, truncating file to desired length %llu\n",
1724  (unsigned long long) GNUNET_ntohll (
1725  dc->uri->data.chk.file_length));
1726  /* truncate file to size (since we store IBlocks at the end) */
1727  if (NULL != dc->filename)
1728  {
1729  if (0 != truncate (dc->filename,
1732  "truncate",
1733  dc->filename);
1734  }
1735  }
1736  }
1737  else
1738  GNUNET_log (
1740  "Reconstruction failed, need to download block at offset %llu, depth %u\n",
1741  (unsigned long long) offset,
1742  depth);
1743  break;
1744 
1745  case BRS_DOWNLOAD_DOWN:
1746  break;
1747 
1748  case BRS_DOWNLOAD_UP:
1749  break;
1750 
1751  case BRS_ERROR:
1752  break;
1753 
1754  default:
1755  GNUNET_assert (0);
1756  break;
1757  }
1759  if ((dr == dc->top_request) && (dr->state == BRS_DOWNLOAD_UP))
1760  check_completed (dc);
1761 }
1762 
1763 
1776 static size_t
1777 fh_reader (void *cls, uint64_t offset, size_t max, void *buf, char **emsg)
1778 {
1779  struct GNUNET_FS_DownloadContext *dc = cls;
1780  struct GNUNET_DISK_FileHandle *fh = dc->rfh;
1781  ssize_t ret;
1782 
1783  if (NULL != emsg)
1784  *emsg = NULL;
1785  if (offset != GNUNET_DISK_file_seek (fh, offset, GNUNET_DISK_SEEK_SET))
1786  {
1787  if (NULL != emsg)
1788  *emsg = GNUNET_strdup (strerror (errno));
1789  return 0;
1790  }
1791  ret = GNUNET_DISK_file_read (fh, buf, max);
1792  if (ret < 0)
1793  {
1794  if (NULL != emsg)
1795  *emsg = GNUNET_strdup (strerror (errno));
1796  return 0;
1797  }
1798  return ret;
1799 }
1800 
1801 
1808 void
1810 {
1811  struct GNUNET_FS_DownloadContext *dc = cls;
1812  struct GNUNET_FS_ProgressInfo pi;
1813  struct GNUNET_DISK_FileHandle *fh;
1814 
1815  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start task running...\n");
1816  dc->task = NULL;
1817  if (0 == dc->length)
1818  {
1819  /* no bytes required! */
1820  if (NULL != dc->filename)
1821  {
1822  fh = GNUNET_DISK_file_open (dc->filename,
1825  | ((0 ==
1828  : 0),
1834  }
1837  pi.value.download.specifics.start.meta = dc->meta;
1839  check_completed (dc);
1840  return;
1841  }
1842  if (NULL != dc->emsg)
1843  return;
1844  if (NULL == dc->top_request)
1845  {
1847  0,
1848  dc->treedepth - 1,
1849  0,
1850  dc->offset,
1851  dc->length);
1852  dc->top_request->state = BRS_CHK_SET;
1853  dc->top_request->chk = (dc->uri->type == GNUNET_FS_URI_CHK)
1854  ? dc->uri->data.chk.chk
1855  : dc->uri->data.loc.fi.chk;
1856  /* signal start */
1858  if (NULL != dc->search)
1861  pi.value.download.specifics.start.meta = dc->meta;
1863  }
1865  /* attempt reconstruction from disk */
1867  dc->rfh = GNUNET_DISK_file_open (dc->filename,
1870  if (dc->top_request->state == BRS_CHK_SET)
1871  {
1872  if (NULL != dc->rfh)
1873  {
1874  /* first, try top-down */
1876  "Trying top-down reconstruction for `%s'\n",
1877  dc->filename);
1879  switch (dc->top_request->state)
1880  {
1881  case BRS_CHK_SET:
1882  break; /* normal */
1883 
1884  case BRS_DOWNLOAD_DOWN:
1885  break; /* normal, some blocks already down */
1886 
1887  case BRS_DOWNLOAD_UP:
1888  /* already done entirely, party! */
1889  if (NULL != dc->rfh)
1890  {
1891  /* avoid hanging on to file handle longer than
1892  * necessary */
1894  dc->rfh = NULL;
1895  }
1896  return;
1897 
1898  case BRS_ERROR:
1899  GNUNET_asprintf (&dc->emsg, _ ("Invalid URI"));
1902  pi.value.download.specifics.error.message = dc->emsg;
1904  return;
1905 
1906  default:
1907  GNUNET_assert (0);
1908  break;
1909  }
1910  }
1911  }
1912  /* attempt reconstruction from meta data */
1914  (NULL != dc->meta))
1915  {
1916  GNUNET_log (
1918  "Trying to find embedded meta data for download of size %llu with %u bytes MD\n",
1919  (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri),
1922  if (BRS_DOWNLOAD_UP == dc->top_request->state)
1923  {
1924  if (NULL != dc->rfh)
1925  {
1926  /* avoid hanging on to file handle longer than
1927  * necessary */
1929  dc->rfh = NULL;
1930  }
1931  return; /* finished, status update was already done for us */
1932  }
1933  }
1934  if (NULL != dc->rfh)
1935  {
1936  /* finally, actually run bottom-up */
1938  "Trying bottom-up reconstruction of file `%s'\n",
1939  dc->filename);
1940  dc->te =
1943  dc,
1944  &fh_reader,
1945  &reconstruct_cb,
1946  NULL,
1947  &reconstruct_cont);
1949  }
1950  else
1951  {
1952  /* simple, top-level download */
1953  dc->issue_requests = GNUNET_YES;
1955  }
1956  if (BRS_DOWNLOAD_UP == dc->top_request->state)
1957  check_completed (dc);
1958 }
1959 
1960 
1967 void
1969 {
1970  struct GNUNET_FS_DownloadContext *dc = cls;
1971  struct GNUNET_FS_ProgressInfo pi;
1972 
1973  if (NULL != dc->top)
1974  GNUNET_FS_end_top (dc->h, dc->top);
1975  while (NULL != dc->child_head)
1977  if (NULL != dc->search)
1978  {
1979  dc->search->download = NULL;
1980  dc->search = NULL;
1981  }
1982  if (NULL != dc->job_queue)
1983  {
1985  dc->job_queue = NULL;
1986  }
1987  if (NULL != dc->parent)
1989  dc->parent->child_tail,
1990  dc);
1991  if (NULL != dc->task)
1992  {
1994  dc->task = NULL;
1995  }
1998  if (NULL != dc->te)
1999  {
2000  GNUNET_FS_tree_encoder_finish (dc->te, NULL);
2001  dc->te = NULL;
2002  }
2003  if (NULL != dc->rfh)
2004  {
2006  dc->rfh = NULL;
2007  }
2009  if (NULL != dc->active)
2010  {
2012  dc->active = NULL;
2013  }
2016  GNUNET_FS_uri_destroy (dc->uri);
2019  GNUNET_assert (NULL == dc->job_queue);
2020  GNUNET_free (dc);
2021 }
2022 
2023 
2045  const struct GNUNET_FS_Uri *uri,
2046  const struct GNUNET_CONTAINER_MetaData *meta,
2047  const char *filename,
2048  const char *tempname,
2049  uint64_t offset,
2050  uint64_t length,
2051  uint32_t anonymity,
2053  void *cctx)
2054 {
2055  struct GNUNET_FS_DownloadContext *dc;
2056 
2058  if ((offset + length < offset) ||
2059  (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)))
2060  {
2061  GNUNET_break (0);
2062  return NULL;
2063  }
2064  dc = GNUNET_new (struct GNUNET_FS_DownloadContext);
2066  "Starting download %p, %u bytes at offset %llu\n",
2067  dc,
2068  (unsigned int) length,
2069  (unsigned long long) offset);
2070  dc->h = h;
2071  dc->uri = GNUNET_FS_uri_dup (uri);
2073  dc->client_info = cctx;
2075  if (NULL != filename)
2076  {
2077  dc->filename = GNUNET_strdup (filename);
2078  if (GNUNET_YES == GNUNET_DISK_file_test (filename))
2080  &dc->old_file_size,
2081  GNUNET_YES,
2082  GNUNET_YES));
2083  }
2084  if (GNUNET_FS_uri_test_loc (dc->uri))
2087  dc->offset = offset;
2088  dc->length = length;
2089  dc->anonymity = anonymity;
2090  dc->options = options;
2091  dc->active =
2093  GNUNET_NO);
2094  dc->treedepth =
2096  if ((NULL == filename) && (is_recursive_download (dc)))
2097  {
2098  if (NULL != tempname)
2099  dc->temp_filename = GNUNET_strdup (tempname);
2100  else
2101  dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
2102  }
2104  "Starting download `%s' of %llu bytes with tree depth %u\n",
2105  filename,
2106  (unsigned long long) length,
2107  dc->treedepth);
2108  GNUNET_assert (NULL == dc->job_queue);
2110  return dc;
2111 }
2112 
2113 
2146  const struct GNUNET_FS_Uri *uri,
2147  const struct GNUNET_CONTAINER_MetaData *meta,
2148  const char *filename,
2149  const char *tempname,
2150  uint64_t offset,
2151  uint64_t length,
2152  uint32_t anonymity,
2154  void *cctx,
2156 {
2157  struct GNUNET_FS_DownloadContext *dc;
2158 
2159  dc = create_download_context (h,
2160  uri,
2161  meta,
2162  filename,
2163  tempname,
2164  offset,
2165  length,
2166  anonymity,
2167  options,
2168  cctx);
2169  if (NULL == dc)
2170  return NULL;
2171  dc->parent = parent;
2172  if (NULL != parent)
2173  GNUNET_CONTAINER_DLL_insert (parent->child_head, parent->child_tail, dc);
2174  else if (0 == (GNUNET_FS_DOWNLOAD_IS_PROBE & options))
2175  dc->top =
2177  return dc;
2178 }
2179 
2180 
2217  struct GNUNET_FS_SearchResult *sr,
2218  const char *filename,
2219  const char *tempname,
2220  uint64_t offset,
2221  uint64_t length,
2222  uint32_t anonymity,
2224  void *cctx)
2225 {
2226  struct GNUNET_FS_DownloadContext *dc;
2227 
2228  if ((NULL == sr) || (NULL != sr->download))
2229  {
2230  GNUNET_break (0);
2231  return NULL;
2232  }
2233  dc = create_download_context (h,
2234  sr->uri,
2235  sr->meta,
2236  filename,
2237  tempname,
2238  offset,
2239  length,
2240  anonymity,
2241  options,
2242  cctx);
2243  if (NULL == dc)
2244  return NULL;
2245  dc->search = sr;
2246  sr->download = dc;
2247  if (NULL != sr->probe_ctx)
2248  {
2250  sr->probe_ctx = NULL;
2252  }
2253  return dc;
2254 }
2255 
2256 
2262 void
2264 {
2265  if (dc->completed == dc->length)
2266  return;
2267  if (NULL != dc->mq)
2268  return; /* already running */
2269  GNUNET_assert (NULL == dc->job_queue);
2270  GNUNET_assert (NULL == dc->task);
2271  GNUNET_assert (NULL != dc->active);
2272  dc->job_queue =
2273  GNUNET_FS_queue_ (dc->h,
2276  dc,
2277  (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
2278  (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
2282  "Download %p put into queue as job %p\n",
2283  dc,
2284  dc->job_queue);
2285 }
2286 
2287 
2293 void
2295 {
2297 }
2298 
2299 
2305 void
2307 {
2308  struct GNUNET_FS_ProgressInfo pi;
2309 
2312 
2313  GNUNET_assert (NULL == dc->task);
2314  dc->job_queue =
2315  GNUNET_FS_queue_ (dc->h,
2318  dc,
2319  (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
2320  (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
2323 }
2324 
2325 
2332 void
2334 {
2335  struct GNUNET_FS_ProgressInfo pi;
2336  int have_children;
2337  int search_was_null;
2338 
2339  if (NULL != dc->top)
2340  GNUNET_FS_end_top (dc->h, dc->top);
2341  if (NULL != dc->task)
2342  {
2344  dc->task = NULL;
2345  }
2346  search_was_null = (NULL == dc->search);
2347  if (NULL != dc->search)
2348  {
2349  dc->search->download = NULL;
2351  dc->search = NULL;
2352  }
2353  if (NULL != dc->job_queue)
2354  {
2356  dc->job_queue = NULL;
2357  }
2358  if (NULL != dc->te)
2359  {
2360  GNUNET_FS_tree_encoder_finish (dc->te, NULL);
2361  dc->te = NULL;
2362  }
2363  have_children = (NULL != dc->child_head) ? GNUNET_YES : GNUNET_NO;
2364  while (NULL != dc->child_head)
2365  GNUNET_FS_download_stop (dc->child_head, do_delete);
2366  if (NULL != dc->parent)
2368  dc->parent->child_tail,
2369  dc);
2370  if (NULL != dc->serialization)
2372  ((NULL != dc->parent) || (! search_was_null))
2375  dc->serialization);
2376  if ((GNUNET_YES == have_children) && (NULL == dc->parent))
2378  (! search_was_null)
2380  : GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD,
2381  dc->serialization);
2385  dc->top_request = NULL;
2386  if (NULL != dc->active)
2387  {
2389  dc->active = NULL;
2390  }
2391  if (NULL != dc->filename)
2392  {
2393  if ((dc->completed != dc->length) && (GNUNET_YES == do_delete))
2394  {
2395  if ((0 != unlink (dc->filename)) && (ENOENT != errno))
2397  "unlink",
2398  dc->filename);
2399  }
2400  GNUNET_free (dc->filename);
2401  }
2403  GNUNET_FS_uri_destroy (dc->uri);
2404  if (NULL != dc->temp_filename)
2405  {
2406  if (0 != unlink (dc->temp_filename))
2408  "unlink",
2409  dc->temp_filename);
2410  GNUNET_free (dc->temp_filename);
2411  }
2413  GNUNET_assert (NULL == dc->job_queue);
2414  GNUNET_free (dc);
2415 }
2416 
2417 
2418 /* end of fs_download.c */
int GNUNET_FS_uri_loc_get_peer_identity(const struct GNUNET_FS_Uri *uri, struct GNUNET_PeerIdentity *peer)
Obtain the identity of the peer offering the data.
Definition: fs_uri.c:822
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
void * client_info
Client info for this search result.
Definition: fs_api.h:533
struct GNUNET_HashCode query
Hash of data.
Definition: fs_download.c:139
This is a probe (low priority).
Definition: fs_api.h:407
ssize_t GNUNET_CONTAINER_meta_data_get_serialized_size(const struct GNUNET_CONTAINER_MetaData *md)
Get the size of the full meta-data in serialized form.
int GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory)...
Definition: disk.c:544
Notification that we have started this download.
Open the file for reading.
char * GNUNET_FS_meta_data_suggest_filename(const struct GNUNET_CONTAINER_MetaData *md)
Suggest a filename based on given metadata.
Definition: fs_misc.c:38
void * GNUNET_FS_search_probe_progress_(void *cls, const struct GNUNET_FS_ProgressInfo *info)
Notification of FS that a search probe has made progress.
Definition: fs_search.c:280
static void try_match_block(struct GNUNET_FS_DownloadContext *dc, struct DownloadRequest *dr, const char *data, size_t data_len)
We got a block of plaintext data (from the meta data).
Definition: fs_download.c:431
static void do_reconnect(void *cls)
Reconnect to the FS service and transmit our queries NOW.
Definition: fs_download.c:1370
Closure for iterator processing results.
Definition: fs_download.c:134
struct ContentHashKey chk
Query and key of the top GNUNET_EC_IBlock.
Definition: fs_api.h:103
No options (use defaults for everything).
Create file if it doesn&#39;t exist.
struct GNUNET_FS_DownloadContext * create_download_context(struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, const char *filename, const char *tempname, uint64_t offset, uint64_t length, uint32_t anonymity, enum GNUNET_FS_DownloadOptions options, void *cctx)
Helper function to setup the download context.
Definition: fs_download.c:2044
EXTRACTOR_MetaFormat
Format in which the extracted meta data is presented.
static int retry_entry(void *cls, const struct GNUNET_HashCode *key, void *entry)
Add entries to the message queue.
Definition: fs_download.c:746
Context for controlling a download.
Definition: fs_api.h:1742
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh(struct GNUNET_TIME_AbsoluteNBO a)
Convert absolute time from network byte order.
Definition: time.c:673
static void reconstruct_cb(void *cls, const struct ContentHashKey *chk, uint64_t offset, unsigned int depth, enum GNUNET_BLOCK_Type type, const void *block, uint16_t block_size)
Function called asking for the current (encoded) block to be processed.
Definition: fs_download.c:1625
struct GNUNET_FS_DownloadContext * download
ID of an associated download based on this search result (or NULL for none).
Definition: fs_api.h:545
int GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1345
struct FileIdentifier chk
Information needed to retrieve a file (content-hash-key plus file size).
Definition: fs_api.h:211
uint64_t rel_value_us
The actual value.
struct GNUNET_FS_DownloadContext * dc
Our download context.
Definition: fs_download.c:149
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
struct GNUNET_FS_ProgressInfo::@25::@27 download
Values for all "GNUNET_FS_STATUS_DOWNLOAD_*" events.
enum GNUNET_FS_DownloadOptions options
Options for the download.
Definition: fs_api.h:1922
struct GNUNET_FS_Handle * h
Global FS context.
Definition: fs_api.h:1747
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
static int process_result_with_request(void *cls, const struct GNUNET_HashCode *key, void *value)
Iterator over entries in the pending requests in the &#39;active&#39; map for the reply that we just got...
Definition: fs_download.c:1017
struct GNUNET_MQ_Handle * GNUNET_CLIENT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *service_name, const struct GNUNET_MQ_MessageHandler *handlers, GNUNET_MQ_ErrorHandler error_handler, void *error_handler_cls)
Create a message queue to connect to a GNUnet service.
Definition: client.c:1057
struct GNUNET_MessageHeader header
Message type will be GNUNET_MESSAGE_TYPE_FS_PUT.
Definition: fs.h:359
Notification that this download was suspended.
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
content hash key
Definition: fs.h:53
#define GNUNET_FS_SYNC_PATH_CHILD_DOWNLOAD
Name of the directory with downloads that are part of another download or a search.
Definition: fs_api.h:71
struct DownloadRequest ** children
Array (!) of child-requests, or NULL for the bottom of the tree.
Definition: fs_api.h:1684
static void try_top_down_reconstruction(struct GNUNET_FS_DownloadContext *dc, struct DownloadRequest *dr)
Try top-down reconstruction.
Definition: fs_download.c:662
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:732
GNUNET_MQ_Error
Error codes for the queue.
void GNUNET_FS_download_make_status_(struct GNUNET_FS_ProgressInfo *pi, struct GNUNET_FS_DownloadContext *dc)
Fill in all of the generic fields for a download event and call the callback.
Definition: fs_download.c:103
struct GNUNET_FS_DownloadContext * child_head
Head of list of child downloads.
Definition: fs_api.h:1774
void GNUNET_FS_download_sync_(struct GNUNET_FS_DownloadContext *dc)
Synchronize this download struct with its mirror on disk.
Definition: fs_api.c:1972
Master context for most FS operations.
Definition: fs_api.h:1068
static struct DownloadRequest * create_download_request(struct DownloadRequest *parent, unsigned int chk_idx, unsigned int depth, uint64_t dr_offset, uint64_t file_start_offset, uint64_t desired_length)
(recursively) Create a download request structure.
Definition: fs_download.c:1494
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
enum GNUNET_BLOCK_Type type
Type of data.
Definition: fs_download.c:164
void * GNUNET_DISK_file_map(const struct GNUNET_DISK_FileHandle *h, struct GNUNET_DISK_MapHandle **m, enum GNUNET_DISK_MapType access, size_t len)
Map a file into memory.
Definition: disk.c:1440
uint32_t options
Bitmask with options.
Definition: fs.h:284
static void trigger_recursive_download(void *cls, const char *filename, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, size_t length, const void *data)
We found an entry in a directory.
Definition: fs_download.c:850
static void download_mq_error_handler(void *cls, enum GNUNET_MQ_Error error)
Generic error handler, called with the appropriate error code and the same closure specified at the c...
Definition: fs_download.c:1349
struct DownloadRequest * top_request
Top-level download request.
Definition: fs_api.h:1855
struct GNUNET_FS_DownloadContext * parent
Parent download (used when downloading files in directories).
Definition: fs_api.h:1763
uint64_t offset
Offset of the corresponding block.
Definition: fs_api.h:1697
Nobody is allowed to do anything to the file.
void GNUNET_FS_tree_encoder_next(struct GNUNET_FS_TreeEncoder *te)
Encrypt the next block of the file (and call proc and progress accordingly; or of course "cont" if we...
Definition: fs_tree.c:331
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
#define GNUNET_MQ_msg(mvar, type)
Allocate a GNUNET_MQ_Envelope.
Definition: gnunet_mq_lib.h:67
struct GNUNET_FS_Uri * uri
URI to which this search result refers to.
Definition: fs_api.h:523
int GNUNET_DISK_directory_create_for_file(const char *filename)
Create the directory structure for storing a file.
Definition: disk.c:684
uint64_t GNUNET_FS_tree_compute_tree_size(unsigned int depth)
Calculate how many bytes of payload a block tree of the given depth MAY correspond to at most (this f...
Definition: fs_tree.c:156
#define GNUNET_NO
Definition: gnunet_common.h:78
char * filename
Where are we writing the data (name of the file, can be NULL!).
Definition: fs_api.h:1821
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
struct GNUNET_TIME_AbsoluteNBO last_transmission
When was the last time we&#39;ve tried to download this block? (FOREVER if unknown/not relevant) ...
Definition: fs.h:375
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
int GNUNET_FS_uri_test_equal(const struct GNUNET_FS_Uri *u1, const struct GNUNET_FS_Uri *u2)
Test if two URIs are equal.
Definition: fs_uri.c:1212
#define GNUNET_new(type)
Allocate a struct or union of the given type.
struct ContentHashKey chk
CHK for the request for this block (set during reconstruction to what we have on disk, later to what we want to have).
Definition: fs_api.h:1690
struct GNUNET_FS_DownloadContext * probe_ctx
ID of a job that is currently probing this results&#39; availability (NULL if we are not currently probin...
Definition: fs_api.h:539
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format...
off_t GNUNET_DISK_file_seek(const struct GNUNET_DISK_FileHandle *h, off_t offset, enum GNUNET_DISK_Seek whence)
Move the read/write pointer in a file.
Definition: disk.c:226
We&#39;ve calculated the CHK bottom-up based on what we have on disk, which may not be what the desired C...
Definition: fs_api.h:1636
enum BlockRequestState state
State in the FSM.
Definition: fs_api.h:1717
static struct GNUNET_PEERINFO_Handle * pi
Handle to peerinfo service.
EXTRACTOR_MetaType
Enumeration defining various sources of keywords.
unsigned int num_children
Number of entries in children array.
Definition: fs_api.h:1702
char * temp_filename
Where are we writing the data temporarily (name of the file, can be NULL!); used if we do not have a ...
Definition: fs_api.h:1828
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
struct GNUNET_PeerIdentity target
If the request is for a DBLOCK or IBLOCK, this is the identity of the peer that is known to have a re...
Definition: fs.h:306
struct GNUNET_FS_DownloadContext * GNUNET_FS_download_start(struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *meta, const char *filename, const char *tempname, uint64_t offset, uint64_t length, uint32_t anonymity, enum GNUNET_FS_DownloadOptions options, void *cctx, struct GNUNET_FS_DownloadContext *parent)
Download parts of a file.
Definition: fs_download.c:2145
uint64_t abs_value_us
The actual value.
struct GNUNET_FS_Uri * GNUNET_FS_uri_dup(const struct GNUNET_FS_Uri *uri)
Duplicate URI.
Definition: fs_uri.c:998
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
void * client_info
Context kept for the client.
Definition: fs_api.h:1794
#define GNUNET_FS_SYNC_PATH_MASTER_DOWNLOAD
Name of the directory with master downloads (not associated with search or part of another download)...
Definition: fs_api.h:65
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
Initial state, block has only been allocated (since it is relevant to the overall download request)...
Definition: fs_api.h:1612
void GNUNET_FS_dequeue_(struct GNUNET_FS_QueueEntry *qe)
Dequeue a job from the queue.
Definition: fs_api.c:366
uint64_t old_file_size
What was the size of the file on disk that we&#39;re downloading before we started? Used to detect if the...
Definition: fs_api.h:1897
#define GNUNET_TIME_UNIT_FOREVER_ABS
Constant used to specify "forever".
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
int GNUNET_FS_uri_test_loc(const struct GNUNET_FS_Uri *uri)
Is this a location URI?
Definition: fs_uri.c:1395
char * GNUNET_FS_uri_to_string(const struct GNUNET_FS_Uri *uri)
Convert a URI to a UTF-8 String.
Definition: fs_uri.c:2028
static void activate_fs_download(void *cls)
We&#39;re allowed to ask the FS service for our blocks.
Definition: fs_download.c:1437
struct GNUNET_TIME_Absolute last_transmission
When did we last transmit the request?
Definition: fs_download.c:154
struct GNUNET_HashCode query
Hash of the public key for UBLOCKs; Hash of the CHK-encoded block for DBLOCKS and IBLOCKS...
Definition: fs.h:312
#define MAX_INLINE_SIZE
Maximum size for a file to be considered for inlining in a directory.
Definition: fs_api.h:49
Do not append temporary data to the target file (for the IBlocks).
uint64_t length
How many bytes starting from offset are desired? This is NOT the overall length of the file! ...
Definition: fs_api.h:1883
static struct GNUNET_FS_Uri * uri
Value of URI provided on command-line (when not publishing a file but just creating UBlocks to refer ...
static struct GNUNET_ARM_MonitorHandle * m
Monitor connection with ARM.
Definition: gnunet-arm.c:104
unsigned int GNUNET_FS_compute_depth(uint64_t flen)
Compute the depth of the CHK tree.
Definition: fs_tree.c:125
int GNUNET_asprintf(char **buf, const char *format,...)
Like asprintf, just portable.
Meta data to associate with a file, directory or namespace.
Notification that this download completed.
union GNUNET_FS_ProgressInfo::@25 value
Values that depend on the event type.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_delayed(struct GNUNET_TIME_Relative delay, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified delay.
Definition: scheduler.c:1253
char * GNUNET_DISK_mktemp(const char *t)
Create an (empty) temporary file on disk.
Definition: disk.c:468
struct GNUNET_FS_DownloadContext * next
Next download belonging to the same parent.
Definition: fs_api.h:1789
#define GNUNET_FS_DIRECTORY_EXT
void * cls
Closure for mv and cb.
char * serialization
Random portion of filename we use for syncing state of this download.
Definition: fs_api.h:1815
static char * value
Value of the record to add/remove.
unsigned int chk_idx
Offset of the CHK for this block in the parent block.
Definition: fs_api.h:1712
static uint64_t compute_disk_offset(uint64_t fsize, uint64_t off, unsigned int depth)
We&#39;re storing the IBLOCKS after the DBLOCKS on disk (so that we only have to truncate the file once w...
Definition: fs_download.c:67
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level &#39;level&#39; that indicates a failure of the command &#39;cmd&#39; with the mess...
#define GNUNET_MQ_hd_var_size(name, code, str, ctx)
static char * fn
Filename of the unique file.
#define GNUNET_break_op(cond)
Use this for assertion violations caused by other peers (i.e.
void GNUNET_CRYPTO_hash(const void *block, size_t size, struct GNUNET_HashCode *ret)
Compute hash of a given block.
Definition: crypto_hash.c:48
ssize_t GNUNET_CRYPTO_symmetric_encrypt(const void *block, size_t size, const struct GNUNET_CRYPTO_SymmetricSessionKey *sessionkey, const struct GNUNET_CRYPTO_SymmetricInitializationVector *iv, void *result)
Encrypt a block using a symmetric sessionkey.
Information we store for each search result.
Definition: fs_api.h:497
Truncate file if it exists.
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:820
void GNUNET_FS_end_top(struct GNUNET_FS_Handle *h, struct TopLevelActivity *top)
Destroy a top-level activity entry.
Definition: fs_api.c:412
void GNUNET_CONTAINER_multihashmap_destroy(struct GNUNET_CONTAINER_MultiHashMap *map)
Destroy a hash map.
Only download from the local host, do not access remote systems (no P2P)
static int encrypt_existing_match(struct GNUNET_FS_DownloadContext *dc, const struct ContentHashKey *chk, struct DownloadRequest *dr, const char *block, size_t len, int do_store)
We&#39;ve found a matching block without downloading it.
Definition: fs_download.c:212
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_now(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run as soon as possible.
Definition: scheduler.c:1280
#define DIR_SEPARATOR_STR
Definition: platform.h:168
int do_store
Flag to indicate if this block should be stored on disk.
Definition: fs_download.c:169
enum GNUNET_FS_Status status
Specific status code (determines the event type).
Internal option used to flag this download as a &#39;probe&#39; for a search result.
struct GNUNET_HashCode query
Hash of the encrypted content, used for querying.
Definition: fs.h:63
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:80
Inner block in the CHK tree.
void GNUNET_FS_search_result_sync_(struct GNUNET_FS_SearchResult *sr)
Synchronize this search result with its mirror on disk.
Definition: fs_api.c:2076
Read-only memory map.
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition: strings.c:687
We&#39;ve checked the block on the path down the tree, and the content on disk did match the desired CHK...
Definition: fs_api.h:1620
static void check_completed(struct GNUNET_FS_DownloadContext *dc)
Check if all child-downloads have completed (or trigger them if necessary) and once we&#39;re completely ...
Definition: fs_download.c:375
void GNUNET_FS_download_stop(struct GNUNET_FS_DownloadContext *dc, int do_delete)
Stop a download (aborts if download is incomplete).
Definition: fs_download.c:2333
struct GNUNET_FS_QueueEntry * GNUNET_FS_queue_(struct GNUNET_FS_Handle *h, GNUNET_SCHEDULER_TaskCallback start, GNUNET_SCHEDULER_TaskCallback stop, void *cls, unsigned int blocks, enum GNUNET_FS_QueuePriority priority)
Add a job to the queue.
Definition: fs_api.c:331
static char buf[2048]
#define GNUNET_new_array(n, type)
Allocate a size n array with structs or unions of the given type.
static char * filename
#define CHK_PER_INODE
Pick a multiple of 2 here to achive 8-byte alignment! We also probably want DBlocks to have (roughly)...
Definition: fs_api.h:43
const void * data
Data found in P2P network.
Definition: fs_download.c:144
int GNUNET_CONTAINER_multihashmap_remove(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, const void *value)
Remove the given key-value pair from the map.
static void try_reconnect(struct GNUNET_FS_DownloadContext *dc)
We&#39;ve lost our connection with the FS service.
Definition: fs_download.c:1406
static void reconstruct_cont(void *cls)
Continuation after a possible attempt to reconstruct the current IBlock from the existing file...
Definition: fs_download.c:1570
uint32_t type
Type of the content that we&#39;re looking for.
Definition: fs.h:289
void GNUNET_CONTAINER_meta_data_destroy(struct GNUNET_CONTAINER_MetaData *md)
Free meta data.
int issue_requests
Are we ready to issue requests (reconstructions are finished)?
Definition: fs_api.h:1934
struct GNUNET_CONTAINER_MultiHashMap * active
Map of active requests (those waiting for a response).
Definition: fs_api.h:1850
static int fh
Handle to the unique file.
struct GNUNET_FS_TreeEncoder * te
Tree encoder used for the reconstruction.
Definition: fs_api.h:1838
struct GNUNET_FS_TreeEncoder * GNUNET_FS_tree_encoder_create(struct GNUNET_FS_Handle *h, uint64_t size, void *cls, GNUNET_FS_DataReader reader, GNUNET_FS_TreeBlockProcessor proc, GNUNET_FS_TreeProgressCallback progress, GNUNET_SCHEDULER_TaskCallback cont)
Initialize a tree encoder.
Definition: fs_tree.c:269
We&#39;ve calculated the CHK bottom-up based on the meta data.
Definition: fs_api.h:1628
uint64_t offset
What is the first offset that we&#39;re interested in?
Definition: fs_api.h:1877
static int check_put(void *cls, const struct ClientPutMessage *cm)
Type of a function to call when we check the PUT message from the service.
Definition: fs_download.c:1300
struct TopLevelActivity * top
Our top-level activity entry (if we are top-level, otherwise NULL).
Definition: fs_api.h:1752
void GNUNET_FS_uri_destroy(struct GNUNET_FS_Uri *uri)
Free URI.
Definition: fs_uri.c:678
A 512-bit hashcode.
static size_t fh_reader(void *cls, uint64_t offset, size_t max, void *buf, char **emsg)
Function called by the tree encoder to obtain a block of plaintext data (for the lowest level of the ...
Definition: fs_download.c:1777
#define GNUNET_TIME_UNIT_MILLISECONDS
One millisecond.
We&#39;ve successfully downloaded this block, but the children still need to be either downloaded or veri...
Definition: fs_api.h:1654
Message handler for a specific message type.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
struct GNUNET_SCHEDULER_Task * task
ID of a task that is using this struct and that must be cancelled when the download is being stopped ...
Definition: fs_api.h:1871
void GNUNET_CRYPTO_hash_to_aes_key(const struct GNUNET_HashCode *hc, struct GNUNET_CRYPTO_SymmetricSessionKey *skey, struct GNUNET_CRYPTO_SymmetricInitializationVector *iv)
Convert a hashcode into a key.
Definition: crypto_hash.c:222
struct GNUNET_DISK_FileHandle * rfh
File handle for reading data from an existing file (to pass to tree encoder).
Definition: fs_api.h:1844
int GNUNET_FS_uri_test_chk(const struct GNUNET_FS_Uri *uri)
Is this a file (or directory) URI?
Definition: fs_uri.c:1357
This block and all of its children have been downloaded successfully (full completion propagates up)...
Definition: fs_api.h:1660
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:40
Notification that this download is no longer actively being pursued (back in the queue).
Content-hash-key (simple file).
Definition: fs_api.h:143
struct GNUNET_FS_SearchResult * search
Associated search (used when downloading files based on search results), or NULL for none...
Definition: fs_api.h:1769
uint32_t respect_offered
how much respect did we offer to get this reply?
Definition: fs_download.c:174
struct GNUNET_HashCode key
The key used in the DHT.
char * emsg
Error message, NULL if we&#39;re doing OK.
Definition: fs_api.h:1809
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static void deactivate_fs_download(void *cls)
We must stop to ask the FS service for our blocks.
Definition: fs_download.c:1458
Notification about progress with this download.
int GNUNET_DISK_file_unmap(struct GNUNET_DISK_MapHandle *h)
Unmap a file.
Definition: disk.c:1477
GNUNET_FS_DownloadOptions
Options for downloading.
int has_finished
Flag set upon transitive completion (includes child downloads).
Definition: fs_api.h:1929
int GNUNET_CONTAINER_multihashmap_contains_value(const struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, const void *value)
Check if the map contains the given value under the given key.
Notification that this download is now actively being pursued (as opposed to waiting in the queue)...
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
#define GNUNET_TIME_STD_BACKOFF(r)
Perform our standard exponential back-off calculation, starting at 1 ms and then going by a factor of...
struct GNUNET_CONTAINER_MetaData * meta
Metadata for the search result.
Definition: fs_api.h:528
enum GNUNET_FS_UriType type
Type of the URI.
Definition: fs_api.h:170
int GNUNET_CONTAINER_multihashmap_put(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, void *value, enum GNUNET_CONTAINER_MultiHashMapOption opt)
Store a key-value pair in the map.
int GNUNET_FS_meta_data_test_for_directory(const struct GNUNET_CONTAINER_MetaData *md)
Does the meta-data claim that this is a directory? Checks if the mime-type is that of a GNUnet direct...
Definition: fs_directory.c:54
int GNUNET_FS_directory_list_contents(size_t size, const void *data, uint64_t offset, GNUNET_FS_DirectoryEntryProcessor dep, void *dep_cls)
Iterate over all entries in a directory.
Definition: fs_directory.c:181
uint32_t respect_offered
How much respect did we offer (in total) before getting an answer (estimate).
Definition: fs.h:387
#define GNUNET_TIME_UNIT_ZERO
Relative time zero.
int GNUNET_DISK_file_size(const char *filename, uint64_t *size, int include_symbolic_links, int single_file_mode)
Get the size of the file (or directory) of the given file (in bytes).
Definition: disk.c:257
union GNUNET_FS_Uri::@13 data
#define GNUNET_MESSAGE_TYPE_FS_START_SEARCH
Client asks FS service to start a (keyword) search.
struct GNUNET_TIME_Relative reconnect_backoff
How long to wait before we try to reconnect to FS service?
Definition: fs_api.h:1907
struct GNUNET_FS_Handle * fsh
File-sharing handle that generated the event.
static void full_recursive_download(struct GNUNET_FS_DownloadContext *dc)
We&#39;re done downloading a directory.
Definition: fs_download.c:295
struct TopLevelActivity * GNUNET_FS_make_top(struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, void *ssf_cls)
Create a top-level activity entry.
Definition: fs_api.c:391
We got a block back that matched the query but did not hash to the key (malicious publisher or hash c...
Definition: fs_api.h:1667
Response from FS service with a result for a previous FS search.
Definition: fs.h:354
static void get_next_block(void *cls)
Task requesting the next block from the tree encoder.
Definition: fs_download.c:1598
Allow multiple values with the same key.
uint32_t type
Type of the block (in big endian).
Definition: fs.h:364
struct GNUNET_HashCode key
Hash of the original content, used for encryption.
Definition: fs.h:58
uint32_t num_transmissions
how often did we transmit the query?
Definition: fs_download.c:179
void GNUNET_FS_free_download_request_(struct DownloadRequest *dr)
(recursively) free download request structure
Definition: fs_download.c:996
Everybody can read.
struct GNUNET_FS_DownloadContext * child_tail
Tail of list of child downloads.
Definition: fs_api.h:1779
static void propagate_up(struct DownloadRequest *dr)
Set the state of the given download request to BRS_DOWNLOAD_UP and propagate it up the tree...
Definition: fs_download.c:633
GNUNET_FS_ProgressCallback upcb
Function to call with updates on our progress.
Definition: fs_api.h:1083
int GNUNET_CONTAINER_multihashmap_get_multiple(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, GNUNET_CONTAINER_MulitHashMapIteratorCallback it, void *it_cls)
Iterate over all entries in the map that match a particular key.
static int is_recursive_download(struct GNUNET_FS_DownloadContext *dc)
Determine if the given download (options and meta data) should cause use to try to do a recursive dow...
Definition: fs_download.c:37
Seek an absolute position (from the start of the file).
Do a recursive download (that is, automatically trigger the download of files in directories).
static int match_full_data(void *cls, const char *plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char *data_mime_type, const char *data, size_t data_len)
Type of a function that libextractor calls for each meta data item found.
Definition: fs_download.c:601
uint64_t GNUNET_FS_uri_chk_get_file_size(const struct GNUNET_FS_Uri *uri)
What is the size of the file that this URI refers to?
Definition: fs_uri.c:1371
shared definitions for the FS library
struct GNUNET_PeerIdentity target
Identity of the peer having the content, or all-zeros if we don&#39;t know of such a peer.
Definition: fs_api.h:1861
A Universal Resource Identifier (URI), opaque.
Definition: fs_api.h:165
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:375
uint64_t file_length
Total size of the file in bytes.
Definition: fs_api.h:98
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
static OpusEncoder * enc
OPUS encoder.
void * upcb_cls
Closure for upcb.
Definition: fs_api.h:1088
struct FileIdentifier fi
Information about the shared file.
Definition: fs_api.h:116
int GNUNET_CONTAINER_meta_data_iterate(const struct GNUNET_CONTAINER_MetaData *md, EXTRACTOR_MetaDataProcessor iter, void *iter_cls)
Iterate over MD entries.
#define GNUNET_log(kind,...)
#define GNUNET_FS_URI_CHK_PREFIX
Definition: fs_download.c:836
Open the file for both reading and writing.
uint32_t anonymity
Desired level of anonymity.
Definition: fs_api.h:1912
unsigned int treedepth
The depth of the file-tree.
Definition: fs_api.h:1917
Only search the local host, do not search remote systems (no P2P)
Data block (leaf) in the CHK tree.
struct GNUNET_CONTAINER_MultiHashMap * GNUNET_CONTAINER_multihashmap_create(unsigned int len, int do_not_copy_keys)
Create a multi hash map.
struct GNUNET_MQ_Handle * mq
Connection to the FS service.
Definition: fs_api.h:1757
struct GNUNET_CONTAINER_MetaData * meta
Known meta-data for the file (can be NULL).
Definition: fs_api.h:1804
void GNUNET_FS_download_signal_suspend_(void *cls)
Create SUSPEND event for the given download operation and then clean up our state (without stop signa...
Definition: fs_download.c:1968
struct GNUNET_TIME_Relative GNUNET_TIME_calculate_eta(struct GNUNET_TIME_Absolute start, uint64_t finished, uint64_t total)
Calculate the estimate time of arrival/completion for an operation.
Definition: time.c:551
void GNUNET_FS_download_suspend(struct GNUNET_FS_DownloadContext *dc)
Suspend a download.
Definition: fs_download.c:2294
struct Location loc
Information needed to retrieve a file including signed location (identity of a peer) of the content...
Definition: fs_api.h:217
void GNUNET_FS_tree_encoder_finish(struct GNUNET_FS_TreeEncoder *te, char **emsg)
Clean up a tree encoder and return information about possible errors.
Definition: fs_tree.c:443
Time for absolute times used by GNUnet, in microseconds.
void GNUNET_MQ_destroy(struct GNUNET_MQ_Handle *mq)
Destroy the message queue.
Definition: mq.c:824
#define GNUNET_YES
Definition: gnunet_common.h:77
static unsigned int anonymity
void GNUNET_MQ_send(struct GNUNET_MQ_Handle *mq, struct GNUNET_MQ_Envelope *ev)
Send a message with the given message queue.
Definition: mq.c:353
static void handle_put(void *cls, const struct ClientPutMessage *cm)
Type of a function to call when we receive a message from the service.
Definition: fs_download.c:1315
struct GNUNET_TIME_Absolute start_time
Time download was started.
Definition: fs_api.h:1902
void GNUNET_FS_download_start_task_(void *cls)
Task that creates the initial (top-level) download request for the file.
Definition: fs_download.c:1809
uint64_t completed
How many bytes have we already received within the specified range (DBlocks only).
Definition: fs_api.h:1889
struct GNUNET_FS_DownloadContext * GNUNET_FS_download_start_from_search(struct GNUNET_FS_Handle *h, struct GNUNET_FS_SearchResult *sr, const char *filename, const char *tempname, uint64_t offset, uint64_t length, uint32_t anonymity, enum GNUNET_FS_DownloadOptions options, void *cctx)
Download parts of a file based on a search result.
Definition: fs_download.c:2216
struct GNUNET_FS_Uri * uri
URI that identifies the file that we are downloading.
Definition: fs_api.h:1799
size_t size
Number of bytes in data.
Definition: fs_download.c:159
void GNUNET_FS_download_resume(struct GNUNET_FS_DownloadContext *dc)
Resume a suspended download.
Definition: fs_download.c:2306
ssize_t GNUNET_CRYPTO_symmetric_decrypt(const void *block, size_t size, const struct GNUNET_CRYPTO_SymmetricSessionKey *sessionkey, const struct GNUNET_CRYPTO_SymmetricInitializationVector *iv, void *result)
Decrypt a given block using a symmetric sessionkey.
int GNUNET_CONTAINER_multihashmap_iterate(struct GNUNET_CONTAINER_MultiHashMap *map, GNUNET_CONTAINER_MulitHashMapIteratorCallback it, void *it_cls)
Iterate over all entries in the map.
Notification that this download was stopped (final event with respect to this action).
Message sent from a GNUnet (fs) search activity to the gnunet-service-fs to start a search...
Definition: fs.h:268
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1268
struct DownloadRequest * parent
Parent in the CHK-tree.
Definition: fs_api.h:1679
void GNUNET_FS_download_start_downloading_(struct GNUNET_FS_DownloadContext *dc)
Start the downloading process (by entering the queue).
Definition: fs_download.c:2263
#define GNUNET_MESSAGE_TYPE_FS_PUT
P2P response with content or active migration of content.
Information about an active download request.
Definition: fs_api.h:1674
Handle used to access files (and pipes).
uint32_t anonymity_level
Desired anonymity level, big-endian.
Definition: fs.h:294
unsigned int depth
Depth of the corresponding block in the tree.
Definition: fs_api.h:1707
#define GNUNET_MQ_handler_end()
End-marker for the handlers array.
static char * plugin_name
Name of our plugin.
static void schedule_block_download(struct GNUNET_FS_DownloadContext *dc, struct DownloadRequest *dr)
Schedule the download of the specified block in the tree.
Definition: fs_download.c:777
We&#39;ve determined the real, desired CHK for this block (full tree reconstruction failed), request is now pending.
Definition: fs_api.h:1645
void GNUNET_FS_remove_sync_file_(struct GNUNET_FS_Handle *h, const char *ext, const char *ent)
Remove serialization/deserialization file from disk.
Definition: fs_api.c:744
size_t GNUNET_FS_tree_calculate_block_size(uint64_t fsize, uint64_t offset, unsigned int depth)
Compute how many bytes of data should be stored in the specified block.
Definition: fs_tree.c:222
Notification that this download encountered an error.
static struct GNUNET_FS_DownloadContext * dc
Handle for a memory-mapping operation.
Definition: disk.c:1412
uint64_t GNUNET_ntohll(uint64_t n)
Convert unsigned 64-bit integer to host byte order.
Definition: common_endian.c:48
Default priority.
Definition: fs_api.h:412
void GNUNET_FS_stop_probe_ping_task_(struct GNUNET_FS_SearchResult *sr)
Stop the ping task for this search result.
Definition: fs_search.c:409
uint32_t num_transmissions
How often did we transmit this query before getting an answer (estimate).
Definition: fs.h:381
#define GNUNET_free(ptr)
Wrapper around free.
Argument given to the progress callback with information about what is going on.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
void GNUNET_FS_remove_sync_dir_(struct GNUNET_FS_Handle *h, const char *ext, const char *uni)
Remove serialization/deserialization directory from disk.
Definition: fs_api.c:803
struct GNUNET_FS_QueueEntry * job_queue
Our entry in the job queue.
Definition: fs_api.h:1833
Merkle-tree-ish-CHK file encoding for GNUnet.
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:966
struct GNUNET_CONTAINER_MetaData * GNUNET_CONTAINER_meta_data_duplicate(const struct GNUNET_CONTAINER_MetaData *md)
Duplicate a MetaData token.
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration to use.
Definition: fs_api.h:1073