GNUnet  0.10.x
fs_publish.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2009, 2010 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
27 #include "platform.h"
28 #include "gnunet_constants.h"
29 #include "gnunet_signatures.h"
30 #include "gnunet_util_lib.h"
31 #include "gnunet_fs_service.h"
32 #include "fs_api.h"
33 #include "fs_tree.h"
34 
35 
46 void *
49  const struct GNUNET_FS_FileInformation *p,
50  uint64_t offset)
51 {
52  pi->value.publish.pc = pc;
53  pi->value.publish.fi = p;
54  pi->value.publish.cctx = p->client_info;
55  pi->value.publish.pctx = (NULL == p->dir) ? NULL : p->dir->client_info;
56  pi->value.publish.filename = p->filename;
57  pi->value.publish.size =
58  (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
59  pi->value.publish.eta =
61  pi->value.publish.size);
62  pi->value.publish.completed = offset;
63  pi->value.publish.duration =
65  pi->value.publish.anonymity = p->bo.anonymity_level;
66  pi->fsh = pc->h;
67  return pc->h->upcb(pc->h->upcb_cls, pi);
68 }
69 
70 
76 static void
78 {
80  "Cleaning up publish context (done!)\n");
81  if (NULL != pc->fhc)
82  {
84  pc->fhc = NULL;
85  }
86  GNUNET_FS_file_information_destroy(pc->fi, NULL, NULL);
90  if (NULL != pc->dsh)
91  {
93  pc->dsh = NULL;
94  }
95  if (NULL != pc->mq)
96  {
97  GNUNET_MQ_destroy(pc->mq);
98  pc->mq = NULL;
99  }
100  GNUNET_assert(NULL == pc->upload_task);
101  GNUNET_free(pc);
102 }
103 
104 
114 static void
115 ds_put_cont(void *cls,
116  int success,
118  const char *msg)
119 {
120  struct GNUNET_FS_PublishContext *pc = cls;
121  struct GNUNET_FS_ProgressInfo pi;
122 
123  pc->qre = NULL;
124  if (GNUNET_SYSERR == success)
125  {
127  _("Publishing failed: %s"),
128  msg);
131  pi.value.publish.specifics.error.message = pc->fi_pos->emsg;
132  pc->fi_pos->client_info =
133  GNUNET_FS_publish_make_status_(&pi, pc, pc->fi_pos, 0);
134  if ((GNUNET_YES != pc->fi_pos->is_directory) &&
135  (NULL != pc->fi_pos->filename) &&
136  (GNUNET_YES == pc->any_done) &&
137  (GNUNET_YES == pc->fi_pos->data.file.do_index))
138  {
139  /* run unindex to clean up */
141  pc->fi_pos->filename,
142  NULL);
143  }
144  return;
145  }
146  pc->any_done = GNUNET_YES;
147  GNUNET_assert(NULL == pc->upload_task);
148  pc->upload_task =
151 }
152 
153 
162 static void
165 {
166  struct GNUNET_FS_ProgressInfo pi;
167 
170  pi.value.publish.specifics.completed.chk_uri = p->chk_uri;
171  pi.value.publish.specifics.completed.sks_uri = p->sks_uri;
172  p->client_info =
174  p->data.file.file_size);
175 }
176 
177 
187 static void
190  const char *emsg)
191 {
192  struct GNUNET_FS_ProgressInfo pi;
193 
194  p->emsg = GNUNET_strdup(emsg);
197  pi.value.publish.specifics.error.message = emsg;
198  p->client_info = GNUNET_FS_publish_make_status_(&pi, pc, p, 0);
199  if ((p->is_directory != GNUNET_YES) &&
200  (NULL != p->filename) &&
201  (GNUNET_YES == pc->any_done) &&
202  (p->data.file.do_index == GNUNET_YES))
203  {
204  /* run unindex to clean up */
206  p->filename,
207  NULL);
208  }
209 }
210 
211 
220 static void
221 finish_release_reserve(void *cls, int success,
223  const char *msg)
224 {
225  struct GNUNET_FS_PublishContext *pc = cls;
226 
227  pc->qre = NULL;
229  "Releasing reserve done!\n");
230  signal_publish_completion(pc->fi, pc);
231  pc->all_done = GNUNET_YES;
233 }
234 
235 
244 static void
246  const struct GNUNET_FS_Uri *uri,
247  const char *emsg)
248 {
249  struct GNUNET_FS_PublishContext *pc = cls;
250 
251  pc->sks_pc = NULL;
252  if (NULL != emsg)
253  {
254  signal_publish_error(pc->fi, pc, emsg);
256  return;
257  }
258  if (NULL != uri)
259  {
260  /* sks publication, remember namespace URI */
261  pc->fi->sks_uri = GNUNET_FS_uri_dup(uri);
262  }
263  GNUNET_assert(pc->qre == NULL);
264  if ((pc->dsh != NULL) && (pc->rid != 0))
265  {
266  pc->qre =
267  GNUNET_DATASTORE_release_reserve(pc->dsh, pc->rid, UINT_MAX, UINT_MAX,
269  }
270  else
271  {
273  }
274 }
275 
276 
283 static void
285 {
286  if (NULL != pc->ns)
287  pc->sks_pc = GNUNET_FS_publish_sks(pc->h,
288  pc->ns,
289  pc->nid,
290  pc->nuid,
291  pc->fi->meta,
292  pc->fi->chk_uri,
293  &pc->fi->bo,
294  pc->options,
295  &publish_sblocks_cont, pc);
296  else
297  publish_sblocks_cont(pc, NULL, NULL);
298 }
299 
300 
310 static void
312  const struct GNUNET_FS_Uri *uri,
313  const char *emsg)
314 {
315  struct GNUNET_FS_PublishContext *pc = cls;
316  struct GNUNET_FS_FileInformation *p = pc->fi_pos;
317 
318  pc->ksk_pc = NULL;
319  if (NULL != emsg)
320  {
322  "Error uploading KSK blocks: %s\n",
323  emsg);
324  signal_publish_error(p, pc, emsg);
327  GNUNET_assert(NULL == pc->upload_task);
328  pc->upload_task =
332  pc);
333  return;
334  }
336  "KSK blocks published, moving on to next file\n");
337  if (NULL != p->dir)
339  /* move on to next file */
340  if (NULL != p->next)
341  pc->fi_pos = p->next;
342  else
343  pc->fi_pos = p->dir;
345  GNUNET_assert(NULL == pc->upload_task);
346  pc->upload_task =
349 }
350 
351 
365 static size_t
366 block_reader(void *cls,
367  uint64_t offset,
368  size_t max,
369  void *buf,
370  char **emsg)
371 {
372  struct GNUNET_FS_PublishContext *pc = cls;
374  const char *dd;
375  size_t pt_size;
376 
377  p = pc->fi_pos;
378  if (GNUNET_YES == p->is_directory)
379  {
380  pt_size = GNUNET_MIN(max, p->data.dir.dir_size - offset);
381  dd = p->data.dir.dir_data;
382  GNUNET_memcpy(buf, &dd[offset], pt_size);
383  }
384  else
385  {
386  if (UINT64_MAX == offset)
387  {
388  if (&GNUNET_FS_data_reader_file_ == p->data.file.reader)
389  {
390  /* force closing the file to avoid keeping too many files open */
391  p->data.file.reader(p->data.file.reader_cls, offset, 0, NULL, NULL);
392  }
393  return 0;
394  }
395  pt_size = GNUNET_MIN(max, p->data.file.file_size - offset);
396  if (0 == pt_size)
397  return 0; /* calling reader with pt_size==0
398  * might free buf, so don't! */
399  if (pt_size !=
400  p->data.file.reader(p->data.file.reader_cls, offset, pt_size, buf,
401  emsg))
402  return 0;
403  }
404  return pt_size;
405 }
406 
407 
415 static void
416 encode_cont(void *cls)
417 {
418  struct GNUNET_FS_PublishContext *pc = cls;
420  struct GNUNET_FS_ProgressInfo pi;
421  char *emsg;
422  uint64_t flen;
423 
425  "Finished with tree encoder\n");
426  p = pc->fi_pos;
430  p->te = NULL;
431  if (NULL != emsg)
432  {
434  "Error during tree walk: %s\n",
435  emsg);
436  GNUNET_asprintf(&p->emsg,
437  _("Publishing failed: %s"),
438  emsg);
439  GNUNET_free(emsg);
442  pi.value.publish.specifics.error.message = p->emsg;
443  p->client_info = GNUNET_FS_publish_make_status_(&pi, pc, p, 0);
444  }
445  else
446  {
447  /* final progress event */
448  GNUNET_assert(NULL != p->chk_uri);
451  pi.value.publish.specifics.progress.data = NULL;
452  pi.value.publish.specifics.progress.offset = flen;
453  pi.value.publish.specifics.progress.data_len = 0;
454  pi.value.publish.specifics.progress.depth = GNUNET_FS_compute_depth(flen);
455  p->client_info = GNUNET_FS_publish_make_status_(&pi, pc, p, flen);
456  }
457  /* continue with main */ /* continue with main */
458  GNUNET_assert(NULL == pc->upload_task);
459  pc->upload_task =
462 }
463 
464 
479 static void
480 block_proc(void *cls,
481  const struct ContentHashKey *chk,
482  uint64_t offset,
483  unsigned int depth,
484  enum GNUNET_BLOCK_Type type,
485  const void *block,
486  uint16_t block_size)
487 {
488  struct GNUNET_FS_PublishContext *pc = cls;
490  struct OnDemandBlock odb;
491 
492  p = pc->fi_pos;
493  if (NULL == pc->dsh)
494  {
496  "Waiting for datastore connection\n");
497  GNUNET_assert(NULL == pc->upload_task);
498  pc->upload_task =
501  return;
502  }
503 
504  if ((GNUNET_YES != p->is_directory) &&
505  (GNUNET_YES == p->data.file.do_index) &&
506  (GNUNET_BLOCK_TYPE_FS_DBLOCK == type))
507  {
509  "Indexing block `%s' for offset %llu with index size %u\n",
510  GNUNET_h2s(&chk->query),
511  (unsigned long long)offset,
512  (unsigned int)sizeof(struct OnDemandBlock));
513  odb.offset = GNUNET_htonll(offset);
514  odb.file_id = p->data.file.file_id;
515  GNUNET_assert(pc->qre == NULL);
516  pc->qre =
518  (p->is_directory == GNUNET_YES) ? 0 : pc->rid,
519  &chk->query,
520  sizeof(struct OnDemandBlock),
521  &odb,
523  p->bo.content_priority,
524  p->bo.anonymity_level,
525  p->bo.replication_level,
526  p->bo.expiration_time,
527  -2, 1,
528  &ds_put_cont, pc);
529  return;
530  }
532  "Publishing block `%s' for offset %llu with size %u\n",
533  GNUNET_h2s(&chk->query),
534  (unsigned long long)offset,
535  (unsigned int)block_size);
536  GNUNET_assert(pc->qre == NULL);
537  pc->qre =
538  GNUNET_DATASTORE_put(pc->dsh, (p->is_directory == GNUNET_YES) ? 0 : pc->rid,
539  &chk->query,
540  block_size,
541  block,
542  type,
543  p->bo.content_priority,
544  p->bo.anonymity_level,
545  p->bo.replication_level,
546  p->bo.expiration_time,
547  -2, 1,
548  &ds_put_cont,
549  pc);
550 }
551 
552 
563 static void
564 progress_proc(void *cls, uint64_t offset,
565  const void *pt_block,
566  size_t pt_size,
567  unsigned int depth)
568 {
569  struct GNUNET_FS_PublishContext *pc = cls;
571  struct GNUNET_FS_FileInformation *par;
572  struct GNUNET_FS_ProgressInfo pi;
573 
574  p = pc->fi_pos;
576  pi.value.publish.specifics.progress.data = pt_block;
577  pi.value.publish.specifics.progress.offset = offset;
578  pi.value.publish.specifics.progress.data_len = pt_size;
579  pi.value.publish.specifics.progress.depth = depth;
580  p->client_info = GNUNET_FS_publish_make_status_(&pi, pc, p, offset);
581  if ((0 != depth) ||
582  (GNUNET_YES == p->is_directory))
583  return;
584  while (NULL != (par = p->dir))
585  {
586  p = par;
588  p->data.dir.contents_completed += pt_size;
590  pi.value.publish.specifics.progress_directory.completed = p->data.dir.contents_completed;
591  pi.value.publish.specifics.progress_directory.total = p->data.dir.contents_size;
592  pi.value.publish.specifics.progress_directory.eta = GNUNET_TIME_calculate_eta(p->start_time,
593  p->data.dir.contents_completed,
594  p->data.dir.contents_size);
595  p->client_info = GNUNET_FS_publish_make_status_(&pi, pc, p, 0);
596  }
597 }
598 
599 
607 static void
609 {
611  char *emsg;
613  struct GNUNET_FS_FileInformation *dirpos;
614  void *raw_data;
615  uint64_t size;
616 
617  p = pc->fi_pos;
618  GNUNET_assert(NULL != p);
619  if (NULL == p->te)
620  {
621  if (GNUNET_YES == p->is_directory)
622  {
623  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Creating directory\n");
625  dirpos = p->data.dir.entries;
626  while (NULL != dirpos)
627  {
628  if (GNUNET_YES == dirpos->is_directory)
629  {
630  raw_data = dirpos->data.dir.dir_data;
631  dirpos->data.dir.dir_data = NULL;
632  }
633  else
634  {
635  raw_data = NULL;
636  if ((dirpos->data.file.file_size < MAX_INLINE_SIZE) &&
637  (dirpos->data.file.file_size > 0))
638  {
639  raw_data = GNUNET_malloc(dirpos->data.file.file_size);
640  emsg = NULL;
641  if (dirpos->data.file.file_size !=
642  dirpos->data.file.reader(dirpos->data.file.reader_cls, 0,
643  dirpos->data.file.file_size, raw_data,
644  &emsg))
645  {
646  GNUNET_free_non_null(emsg);
647  GNUNET_free(raw_data);
648  raw_data = NULL;
649  }
650  dirpos->data.file.reader(dirpos->data.file.reader_cls, UINT64_MAX, 0, 0, NULL);
651  }
652  }
653  GNUNET_FS_directory_builder_add(db, dirpos->chk_uri, dirpos->meta,
654  raw_data);
655  GNUNET_free_non_null(raw_data);
656  dirpos = dirpos->next;
657  }
659  p->data.dir.dir_data = NULL;
660  p->data.dir.dir_size = 0;
662  &p->data.dir.dir_data);
664  }
665  size = (GNUNET_YES == p->is_directory) ? p->data.dir.dir_size : p->data.file.file_size;
667  "Creating tree encoder\n");
668  p->te =
671  &encode_cont);
672  }
674  "Processing next block from tree\n");
676 }
677 
678 
686 static int
688  const struct GNUNET_MessageHeader *msg)
689 {
690  size_t msize = ntohs(msg->size) - sizeof(*msg);
691  const char *emsg = (const char *)&msg[1];
692 
693  if (emsg[msize - sizeof(struct GNUNET_MessageHeader) - 1] != '\0')
694  {
695  GNUNET_break(0);
696  return GNUNET_SYSERR;
697  }
698  return GNUNET_OK;
699 }
700 
701 
709 static void
711  const struct GNUNET_MessageHeader *msg)
712 {
713  struct GNUNET_FS_PublishContext *pc = cls;
715  const char *emsg = (const char *)&msg[1];
716  char *msgtxt;
717 
718  GNUNET_MQ_destroy(pc->mq);
719  pc->mq = NULL;
720  p = pc->fi_pos;
721  GNUNET_asprintf(&msgtxt,
722  _("Can not index file `%s': %s.\n"),
723  p->filename,
724  gettext(emsg));
726  pc,
727  msgtxt);
728  GNUNET_free(msgtxt);
731 }
732 
733 
741 static void
743  const struct GNUNET_MessageHeader *msg)
744 {
745  struct GNUNET_FS_PublishContext *pc = cls;
747 
748  GNUNET_MQ_destroy(pc->mq);
749  pc->mq = NULL;
750  p = pc->fi_pos;
751  p->data.file.index_start_confirmed = GNUNET_YES;
753  publish_content(pc);
754 }
755 
756 
765 static void
767  enum GNUNET_MQ_Error error)
768 {
769  struct GNUNET_FS_PublishContext *pc = cls;
771 
772  if (NULL != pc->mq)
773  {
774  GNUNET_MQ_destroy(pc->mq);
775  pc->mq = NULL;
776  }
777  p = pc->fi_pos;
779  _("Can not index file `%s': %s. Will try to insert instead.\n"),
780  p->filename,
781  _("error on index-start request to `fs' service"));
782  p->data.file.do_index = GNUNET_NO;
784  publish_content(pc);
785 }
786 
787 
795 static void
797  const struct GNUNET_HashCode *res)
798 {
799  struct GNUNET_FS_PublishContext *pc = cls;
800  struct GNUNET_MQ_MessageHandler handlers[] = {
801  GNUNET_MQ_hd_fixed_size(index_start_ok,
803  struct GNUNET_MessageHeader,
804  pc),
805  GNUNET_MQ_hd_var_size(index_start_failed,
807  struct GNUNET_MessageHeader,
808  pc),
810  };
812  struct GNUNET_MQ_Envelope *env;
813  struct IndexStartMessage *ism;
814  size_t slen;
815  uint64_t dev;
816  uint64_t ino;
817  char *fn;
818 
819  pc->fhc = NULL;
820  p = pc->fi_pos;
821  if (NULL == res)
822  {
824  _("Can not index file `%s': %s. Will try to insert instead.\n"),
825  p->filename,
826  _("failed to compute hash"));
827  p->data.file.do_index = GNUNET_NO;
829  publish_content(pc);
830  return;
831  }
832  if (GNUNET_YES == p->data.file.index_start_confirmed)
833  {
834  publish_content(pc);
835  return;
836  }
838  GNUNET_assert(fn != NULL);
839  slen = strlen(fn) + 1;
840  if (slen >=
842  {
844  _
845  ("Can not index file `%s': %s. Will try to insert instead.\n"),
846  fn, _("filename too long"));
847  GNUNET_free(fn);
848  p->data.file.do_index = GNUNET_NO;
850  publish_content(pc);
851  return;
852  }
854  "Hash of indexed file `%s' is `%s'\n",
855  p->filename,
856  GNUNET_h2s(res));
858  {
859  p->data.file.file_id = *res;
860  p->data.file.have_hash = GNUNET_YES;
861  p->data.file.index_start_confirmed = GNUNET_YES;
863  publish_content(pc);
864  GNUNET_free(fn);
865  return;
866  }
867  pc->mq = GNUNET_CLIENT_connect(pc->h->cfg,
868  "fs",
869  handlers,
871  pc);
872  if (NULL == pc->mq)
873  {
875  _("Can not index file `%s': %s. Will try to insert instead.\n"),
876  p->filename,
877  _("could not connect to `fs' service"));
878  p->data.file.do_index = GNUNET_NO;
879  publish_content(pc);
880  GNUNET_free(fn);
881  return;
882  }
883  if (p->data.file.have_hash != GNUNET_YES)
884  {
885  p->data.file.file_id = *res;
886  p->data.file.have_hash = GNUNET_YES;
888  }
889  env = GNUNET_MQ_msg_extra(ism,
890  slen,
892  if (GNUNET_OK ==
894  &dev,
895  &ino))
896  {
897  ism->device = GNUNET_htonll(dev);
898  ism->inode = GNUNET_htonll(ino);
899  }
900  else
901  {
903  _("Failed to get file identifiers for `%s'\n"),
904  p->filename);
905  }
906  ism->file_id = *res;
907  GNUNET_memcpy(&ism[1],
908  fn,
909  slen);
910  GNUNET_free(fn);
911  GNUNET_MQ_send(pc->mq,
912  env);
913 }
914 
915 
921 static void
923 {
925 
926  p = pc->fi_pos;
927  /* upload of "p" complete, publish KBlocks! */
928  if (NULL != p->keywords)
929  {
930  pc->ksk_pc = GNUNET_FS_publish_ksk(pc->h,
931  p->keywords,
932  p->meta,
933  p->chk_uri,
934  &p->bo,
935  pc->options,
937  pc);
938  }
939  else
940  {
941  publish_kblocks_cont(pc, p->chk_uri, NULL);
942  }
943 }
944 
945 
952 static void
954  const struct ResponseLocSignatureMessage *sig)
955 {
956  struct GNUNET_FS_PublishContext *pc = cls;
958 
959  p = pc->fi_pos;
961  /* p->data.loc.fi kept from CHK before */
962  p->chk_uri->data.loc.peer = sig->peer;
968  publish_kblocks(pc);
969 }
970 
971 
980 static void
982  enum GNUNET_MQ_Error error)
983 {
984  struct GNUNET_FS_PublishContext *pc = cls;
985 
986  if (NULL != pc->mq)
987  {
988  GNUNET_MQ_destroy(pc->mq);
989  pc->mq = NULL;
990  }
992  _("Can not create LOC URI. Will continue with CHK instead.\n"));
993  publish_kblocks(pc);
994 }
995 
996 
1004 static void
1006 {
1007  struct GNUNET_MQ_MessageHandler handlers[] = {
1008  GNUNET_MQ_hd_fixed_size(signature_response,
1011  pc),
1013  };
1014  struct GNUNET_MQ_Envelope *env;
1015  struct RequestLocSignatureMessage *req;
1016  struct GNUNET_FS_FileInformation *p;
1017 
1018  if (NULL != pc->mq)
1019  GNUNET_MQ_destroy(pc->mq);
1020  pc->mq = GNUNET_CLIENT_connect(pc->h->cfg,
1021  "fs",
1022  handlers,
1024  pc);
1025  if (NULL == pc->mq)
1026  {
1028  _("Can not create LOC URI. Will continue with CHK instead.\n"));
1029  publish_kblocks(pc);
1030  return;
1031  }
1032  p = pc->fi_pos;
1033  env = GNUNET_MQ_msg(req,
1037  req->chk = p->chk_uri->data.chk.chk;
1039  GNUNET_MQ_send(pc->mq,
1040  env);
1041 }
1042 
1043 
1049 void
1051 {
1052  struct GNUNET_FS_PublishContext *pc = cls;
1053  struct GNUNET_FS_ProgressInfo pi;
1054  struct GNUNET_FS_FileInformation *p;
1055  char *fn;
1056 
1057  pc->upload_task = NULL;
1058  p = pc->fi_pos;
1059  if (NULL == p)
1060  {
1062  "Publishing complete, now publishing SKS and KSK blocks.\n");
1063  /* upload of entire hierarchy complete,
1064  * publish namespace entries */
1066  publish_sblock(pc);
1067  return;
1068  }
1069  /* find starting position */
1070  while ((GNUNET_YES == p->is_directory) &&
1071  (NULL != p->data.dir.entries) &&
1072  (NULL == p->emsg) &&
1073  (NULL == p->data.dir.entries->chk_uri))
1074  {
1075  p = p->data.dir.entries;
1076  pc->fi_pos = p;
1078  }
1079  /* abort on error */
1080  if (NULL != p->emsg)
1081  {
1083  "Error uploading: %s\n",
1084  p->emsg);
1085  /* error with current file, abort all
1086  * related files as well! */
1087  while (NULL != p->dir)
1088  {
1091  p = p->dir;
1092  if (fn != NULL)
1093  {
1094  GNUNET_asprintf(&p->emsg,
1095  _("Recursive upload failed at `%s': %s"),
1096  fn,
1097  p->emsg);
1098  GNUNET_free(fn);
1099  }
1100  else
1101  {
1102  GNUNET_asprintf(&p->emsg,
1103  _("Recursive upload failed: %s"),
1104  p->emsg);
1105  }
1108  pi.value.publish.specifics.error.message = p->emsg;
1109  p->client_info = GNUNET_FS_publish_make_status_(&pi, pc, p, 0);
1110  }
1111  pc->all_done = GNUNET_YES;
1113  return;
1114  }
1115  /* handle completion */
1116  if (NULL != p->chk_uri)
1117  {
1119  "File upload complete, now publishing KSK blocks.\n");
1121 
1122  if ((0 == p->bo.anonymity_level) &&
1123  (GNUNET_YES !=
1125  {
1126  /* zero anonymity, box CHK URI in LOC URI */
1127  create_loc_uri(pc);
1128  }
1129  else
1130  {
1131  publish_kblocks(pc);
1132  }
1133  return;
1134  }
1135  if ((GNUNET_YES != p->is_directory) && (p->data.file.do_index))
1136  {
1137  if (NULL == p->filename)
1138  {
1139  p->data.file.do_index = GNUNET_NO;
1141  _("Can not index file `%s': %s. Will try to insert instead.\n"),
1142  "<no-name>",
1143  _("needs to be an actual file"));
1145  publish_content(pc);
1146  return;
1147  }
1148  if (p->data.file.have_hash)
1149  {
1150  hash_for_index_cb(pc, &p->data.file.file_id);
1151  }
1152  else
1153  {
1155  pc->fhc =
1158  }
1159  return;
1160  }
1161  publish_content(pc);
1162 }
1163 
1164 
1179 static int
1181  struct GNUNET_FS_FileInformation *fi,
1182  uint64_t length,
1184  struct GNUNET_FS_Uri **uri,
1185  struct GNUNET_FS_BlockOptions *bo,
1186  int *do_index,
1187  void **client_info)
1188 {
1189  struct GNUNET_FS_PublishContext *pc = cls;
1190  struct GNUNET_FS_ProgressInfo pi;
1191  unsigned int kc;
1192  uint64_t left;
1193 
1194  if (GNUNET_YES == pc->skip_next_fi_callback)
1195  {
1197  return GNUNET_OK;
1198  }
1200  "Starting publish operation\n");
1201  if (*do_index)
1202  {
1203  /* space for on-demand blocks */
1204  pc->reserve_space +=
1205  ((length + DBLOCK_SIZE -
1206  1) / DBLOCK_SIZE) * sizeof(struct OnDemandBlock);
1207  }
1208  else
1209  {
1210  /* space for DBlocks */
1211  pc->reserve_space += length;
1212  }
1213  /* entries for IBlocks and DBlocks, space for IBlocks */
1214  left = length;
1215  while (1)
1216  {
1217  left = (left + DBLOCK_SIZE - 1) / DBLOCK_SIZE;
1218  pc->reserve_entries += left;
1219  if (left <= 1)
1220  break;
1221  left = left * sizeof(struct ContentHashKey);
1222  pc->reserve_space += left;
1223  }
1224  pc->reserve_entries++;
1225  /* entries and space for keywords */
1226  if (NULL != *uri)
1227  {
1229  pc->reserve_entries += kc;
1231  }
1233  *client_info = GNUNET_FS_publish_make_status_(&pi, pc, fi, 0);
1235  if ((fi->is_directory) && (fi->dir != NULL))
1236  {
1237  /* We are a directory, and we are not top-level; process entries in directory */
1240  }
1241  return GNUNET_OK;
1242 }
1243 
1244 
1252 static void
1254  struct GNUNET_FS_PublishContext *pc)
1255 {
1256  struct GNUNET_FS_ProgressInfo pi;
1257  uint64_t off;
1258 
1259  if (NULL != pc->ksk_pc)
1260  {
1262  pc->ksk_pc = NULL;
1263  }
1264  if (NULL != pc->sks_pc)
1265  {
1267  pc->sks_pc = NULL;
1268  }
1270  "Suspending publish operation\n");
1272  fi->serialization = NULL;
1273  off = (NULL == fi->chk_uri) ? 0 : (GNUNET_YES == fi->is_directory) ? fi->data.dir.dir_size : fi->data.file.file_size;
1275  GNUNET_break(NULL == GNUNET_FS_publish_make_status_(&pi, pc, fi, off));
1276  if (NULL != pc->qre)
1277  {
1279  pc->qre = NULL;
1280  }
1281  if (NULL != pc->dsh)
1282  {
1284  pc->dsh = NULL;
1285  }
1286  pc->rid = 0;
1287 }
1288 
1289 
1304 static int
1306  struct GNUNET_FS_FileInformation *fi,
1307  uint64_t length,
1309  struct GNUNET_FS_Uri **uri,
1310  struct GNUNET_FS_BlockOptions *bo,
1311  int *do_index,
1312  void **client_info)
1313 {
1314  struct GNUNET_FS_PublishContext *pc = cls;
1315 
1316  if (GNUNET_YES == pc->skip_next_fi_callback)
1317  {
1319  return GNUNET_OK;
1320  }
1322  {
1323  /* process entries in directory */
1326  }
1327  suspend_operation(fi, pc);
1328  *client_info = NULL;
1329  return GNUNET_OK;
1330 }
1331 
1332 
1339 void
1341 {
1342  struct GNUNET_FS_PublishContext *pc = cls;
1343 
1344  if (NULL != pc->upload_task)
1345  {
1347  pc->upload_task = NULL;
1348  }
1351  suspend_operation(pc->fi, pc);
1352  GNUNET_FS_end_top(pc->h, pc->top);
1353  pc->top = NULL;
1354  publish_cleanup(pc);
1355 }
1356 
1357 
1367 static void
1368 finish_reserve(void *cls,
1369  int success,
1371  const char *msg)
1372 {
1373  struct GNUNET_FS_PublishContext *pc = cls;
1374 
1375  pc->qre = NULL;
1377  "Reservation complete (%d)!\n",
1378  success);
1379  if ((msg != NULL) || (success <= 0))
1380  {
1381  GNUNET_asprintf(&pc->fi->emsg,
1382  _("Datastore failure: %s"),
1383  msg);
1384  signal_publish_error(pc->fi, pc, pc->fi->emsg);
1385  return;
1386  }
1387  pc->rid = success;
1388  GNUNET_assert(NULL == pc->upload_task);
1389  pc->upload_task =
1392 }
1393 
1394 
1400 static uint64_t
1402 {
1403  struct GNUNET_FS_FileInformation *ent;
1404 
1405  if (GNUNET_YES != fi->is_directory)
1406  return fi->data.file.file_size;
1407  fi->data.dir.contents_size = 0;
1408  for (ent = fi->data.dir.entries; NULL != ent; ent = ent->next)
1410  return fi->data.dir.contents_size;
1411 }
1412 
1413 
1427 struct GNUNET_FS_PublishContext *
1429  struct GNUNET_FS_FileInformation *fi,
1430  const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns,
1431  const char *nid,
1432  const char *nuid,
1434 {
1435  struct GNUNET_FS_PublishContext *ret;
1436  struct GNUNET_DATASTORE_Handle *dsh;
1437 
1438  GNUNET_assert(NULL != h);
1440  if (0 == (options & GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY))
1441  {
1442  dsh = GNUNET_DATASTORE_connect(h->cfg);
1443  if (NULL == dsh)
1444  return NULL;
1445  }
1446  else
1447  {
1448  dsh = NULL;
1449  }
1450  ret = GNUNET_new(struct GNUNET_FS_PublishContext);
1451  ret->dsh = dsh;
1452  ret->h = h;
1453  ret->fi = fi;
1454  if (NULL != ns)
1455  {
1457  *ret->ns = *ns;
1458  GNUNET_assert(NULL != nid);
1459  ret->nid = GNUNET_strdup(nid);
1460  if (NULL != nuid)
1461  ret->nuid = GNUNET_strdup(nuid);
1462  }
1463  ret->options = options;
1464  /* signal start */
1466  ret->fi_pos = ret->fi;
1469  if (NULL != ret->dsh)
1470  {
1471  GNUNET_assert(NULL == ret->qre);
1473  _("Reserving space for %u entries and %llu bytes for publication\n"),
1474  (unsigned int)ret->reserve_entries,
1475  (unsigned long long)ret->reserve_space);
1476  ret->qre =
1478  ret->reserve_entries,
1479  &finish_reserve,
1480  ret);
1481  }
1482  else
1483  {
1484  GNUNET_assert(NULL == ret->upload_task);
1485  ret->upload_task =
1488  }
1489  return ret;
1490 }
1491 
1492 
1507 static int
1509  struct GNUNET_FS_FileInformation *fi,
1510  uint64_t length,
1512  struct GNUNET_FS_Uri **uri,
1513  struct GNUNET_FS_BlockOptions *bo,
1514  int *do_index, void **client_info)
1515 {
1516  struct GNUNET_FS_PublishContext *pc = cls;
1517  struct GNUNET_FS_ProgressInfo pi;
1518  uint64_t off;
1519 
1520  if (GNUNET_YES == pc->skip_next_fi_callback)
1521  {
1523  return GNUNET_OK;
1524  }
1526  {
1527  /* process entries in directory first */
1530  }
1531  if (NULL != fi->serialization)
1532  {
1534  fi->serialization);
1536  fi->serialization = NULL;
1537  }
1538  off = (fi->chk_uri == NULL) ? 0 : length;
1540  GNUNET_break(NULL == GNUNET_FS_publish_make_status_(&pi, pc, fi, off));
1541  *client_info = NULL;
1542  return GNUNET_OK;
1543 }
1544 
1545 
1554 void
1556 {
1557  struct GNUNET_FS_ProgressInfo pi;
1558  uint64_t off;
1559 
1561  "Publish stop called\n");
1562  GNUNET_FS_end_top(pc->h, pc->top);
1563  if (NULL != pc->ksk_pc)
1564  {
1566  pc->ksk_pc = NULL;
1567  }
1568  if (NULL != pc->sks_pc)
1569  {
1571  pc->sks_pc = NULL;
1572  }
1573  if (NULL != pc->upload_task)
1574  {
1576  pc->upload_task = NULL;
1577  }
1580 
1581  if (NULL != pc->fi->serialization)
1582  {
1584  pc->fi->serialization);
1586  pc->fi->serialization = NULL;
1587  }
1588  off = (NULL == pc->fi->chk_uri) ? 0 : GNUNET_ntohll(pc->fi->chk_uri->data.chk.file_length);
1589 
1590  if (NULL != pc->serialization)
1591  {
1593  pc->serialization);
1595  pc->serialization = NULL;
1596  }
1597  if (NULL != pc->qre)
1598  {
1600  pc->qre = NULL;
1601  }
1603  GNUNET_break(NULL == GNUNET_FS_publish_make_status_(&pi, pc, pc->fi, off));
1604  publish_cleanup(pc);
1605 }
1606 
1607 
1608 /* end of fs_publish.c */
struct GNUNET_TIME_Absolute expiration_time
At what time should the block expire? Data blocks (DBLOCKS and IBLOCKS) may still be used even if the...
struct GNUNET_FS_FileInformation * entries
Linked list of entries in the directory.
Definition: fs_api.h:350
unsigned int depth
Depth of the given block in the tree; 0 would be the lowest level (DBLOCKs).
static struct GNUNET_TIME_Absolute min_expiration
Minimum time that content should have to not be discarded instantly (time stamp of any content that w...
char * nid
ID of the content in the namespace, NULL if we have no namespace.
Definition: fs_api.h:1189
int GNUNET_FS_directory_builder_finish(struct GNUNET_FS_DirectoryBuilder *bld, size_t *rsize, void **rdata)
Finish building the directory.
Definition: fs_directory.c:570
static void publish_kblocks_cont(void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg)
We&#39;ve finished publishing a KBlock as part of a larger upload.
Definition: fs_publish.c:311
struct GNUNET_SCHEDULER_Task * upload_task
ID of the task performing the upload.
Definition: fs_api.h:1244
struct ContentHashKey chk
Query and key of the top GNUNET_EC_IBlock.
Definition: fs_api.h:102
struct GNUNET_FS_FileInformation * dir
If this is a file in a directory, "dir" refers to the directory; otherwise NULL.
Definition: fs_api.h:229
void GNUNET_FS_publish_stop(struct GNUNET_FS_PublishContext *pc)
Stop an upload.
Definition: fs_publish.c:1555
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh(struct GNUNET_TIME_AbsoluteNBO a)
Convert absolute time from network byte order.
Definition: time.c:671
int rid
Space reservation ID with datastore service for this upload.
Definition: fs_api.h:1266
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.
void GNUNET_FS_file_information_sync_(struct GNUNET_FS_FileInformation *fi)
Create a temporary file on disk to store the current state of fi in.
Definition: fs_api.c:1311
struct FileIdentifier chk
Information needed to retrieve a file (content-hash-key plus file size).
Definition: fs_api.h:204
struct GNUNET_CONTAINER_MetaData * meta
Metadata to use for the file.
Definition: fs_api.h:244
struct GNUNET_FS_FileInformation::@19::@20 file
Data for a file.
Notification that we are making progress sharing a file structure.
struct GNUNET_FS_PublishContext * GNUNET_FS_publish_start(struct GNUNET_FS_Handle *h, struct GNUNET_FS_FileInformation *fi, const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, const char *nid, const char *nuid, enum GNUNET_FS_PublishOptions options)
Publish a file or directory.
Definition: fs_publish.c:1428
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
struct GNUNET_MQ_Handle * GNUNET_CLIENT_connect(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *service_name, const struct GNUNET_MQ_MessageHandler *handlers, GNUNET_MQ_ErrorHandler error_handler, void *error_handler_cls)
Create a message queue to connect to a GNUnet service.
Definition: client.c:900
static void finish_release_reserve(void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Datastore returns from reservation cancel request.
Definition: fs_publish.c:221
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
content hash key
Definition: fs.h:53
void * client_info
Pointer kept for the client.
Definition: fs_api.h:239
uint32_t purpose
Requested signature purpose.
Definition: fs.h:83
char * emsg
Error message (non-NULL if this operation failed).
Definition: fs_api.h:288
GNUNET_MQ_Error
Error codes for the queue.
char * serialization
Under what filename is this struct serialized (for operational persistence).
Definition: fs_api.h:278
static void publish_content(struct GNUNET_FS_PublishContext *pc)
We are uploading a file or directory; load (if necessary) the next block into memory, encrypt it and send it to the FS service.
Definition: fs_publish.c:608
static void signal_publish_completion(struct GNUNET_FS_FileInformation *p, struct GNUNET_FS_PublishContext *pc)
Generate the callback that signals clients that a file (or directory) has been completely published...
Definition: fs_publish.c:163
Master context for most FS operations.
Definition: fs_api.h:1056
#define GNUNET_MESSAGE_TYPE_FS_INDEX_START_FAILED
Response to a request for start indexing that refuses.
static void ds_put_cont(void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Function called by the datastore API with the result from the PUT request.
Definition: fs_publish.c:115
#define gettext(Msgid)
Definition: gettext.h:45
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_put(struct GNUNET_DATASTORE_Handle *h, uint32_t rid, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Store an item in the datastore.
void GNUNET_FS_publish_ksk_cancel(struct GNUNET_FS_PublishKskContext *pkc)
Abort the KSK publishing operation.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
struct GNUNET_FS_PublishSksContext * GNUNET_FS_publish_sks(struct GNUNET_FS_Handle *h, const struct GNUNET_CRYPTO_EcdsaPrivateKey *ns, const char *identifier, const char *update, const struct GNUNET_CONTAINER_MetaData *meta, const struct GNUNET_FS_Uri *uri, const struct GNUNET_FS_BlockOptions *bo, enum GNUNET_FS_PublishOptions options, GNUNET_FS_PublishContinuation cont, void *cont_cls)
Publish an SBlock on GNUnet.
Definition: fs_namespace.c:430
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_reserve(struct GNUNET_DATASTORE_Handle *h, uint64_t amount, uint32_t entries, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Reserve space in the datastore.
struct GNUNET_TIME_Absolute start_time
At what time did we start this upload?
Definition: fs_api.h:271
uint64_t reserve_space
Storage space to reserve for the operation.
Definition: fs_api.h:1249
char * filename
Name of the file or directory (must be an absolute path).
Definition: fs_api.h:293
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
#define GNUNET_MQ_hd_fixed_size(name, code, str, ctx)
void GNUNET_FS_tree_encoder_next(struct GNUNET_FS_TreeEncoder *te)
Encrypt the next block of the file (and call proc and progress accordingly; or of course "cont" if we...
Definition: fs_tree.c:330
static void handle_index_start_ok(void *cls, const struct GNUNET_MessageHeader *msg)
Process the response from the "fs" service to our &#39;start index&#39; request.
Definition: fs_publish.c:742
#define GNUNET_MQ_msg(mvar, type)
Allocate a GNUNET_MQ_Envelope.
Definition: gnunet_mq_lib.h:67
uint64_t file_length
Size of the shared file (to be signed).
Definition: fs.h:98
Internal state of a directory builder.
Definition: fs_directory.c:334
enum GNUNET_FS_PublishOptions options
Options for publishing.
Definition: fs_api.h:1260
#define GNUNET_NO
Definition: gnunet_common.h:78
struct GNUNET_CRYPTO_FileHashContext * fhc
Non-null if we are currently hashing a file.
Definition: fs_api.h:1216
struct GNUNET_DATASTORE_Handle * dsh
Connection to the datastore service.
Definition: fs_api.h:1221
struct GNUNET_FS_Handle * h
Handle to the global fs context.
Definition: fs_api.h:1169
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
#define GNUNET_free_non_null(ptr)
Free the memory pointed to by ptr if ptr is not NULL.
int GNUNET_DISK_file_get_identifiers(const char *filename, uint64_t *dev, uint64_t *ino)
Obtain some unique identifiers for the given file that can be used to identify it in the local system...
Definition: disk.c:288
#define GNUNET_new(type)
Allocate a struct or union of the given type.
char * serialization
Filename used for serializing information about this operation (should be determined using &#39;mktemp&#39;)...
Definition: fs_api.h:1200
Private ECC key encoded for transmission.
struct GNUNET_PeerIdentity peer
Identity of the peer sharing the file.
Definition: fs_api.h:119
uint64_t offset
At what offset in the file is "data"?
size_t GNUNET_FS_data_reader_file_(void *cls, uint64_t offset, size_t max, void *buf, char **emsg)
Function that provides data by reading from a file.
Definition: fs_api.c:456
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_release_reserve(struct GNUNET_DATASTORE_Handle *h, uint32_t rid, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Signal that all of the data for which a reservation was made has been stored and that whatever excess...
Information for a file or directory that is about to be published.
Definition: fs_api.h:219
char * GNUNET_CONTAINER_meta_data_get_by_type(const struct GNUNET_CONTAINER_MetaData *md, enum EXTRACTOR_MetaType type)
Get the first MD entry of the given type.
uint16_t size
The length of the struct (in bytes, including the length field itself), in big-endian format...
Run as background job (higher than idle, lower than default).
struct GNUNET_CRYPTO_FileHashContext * GNUNET_CRYPTO_hash_file(enum GNUNET_SCHEDULER_Priority priority, const char *filename, size_t blocksize, GNUNET_CRYPTO_HashCompletedCallback callback, void *callback_cls)
Compute the hash of an entire file.
static struct GNUNET_PEERINFO_Handle * pi
Handle to peerinfo service.
static int ret
Final status code.
Definition: gnunet-arm.c:89
#define GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGNATURE
Reply sent by fs service with LOC signature.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
void * dir_data
Pointer to the data for the directory (or NULL if not available).
Definition: fs_api.h:362
struct GNUNET_FS_Uri * GNUNET_FS_tree_encoder_get_uri(struct GNUNET_FS_TreeEncoder *te)
Get the resulting URI from the encoding.
Definition: fs_tree.c:423
Message sent from a GNUnet (fs) publishing activity to the gnunet-fs-service to initiate indexing of ...
Definition: fs.h:148
static void hash_for_index_cb(void *cls, const struct GNUNET_HashCode *res)
Function called once the hash computation over an indexed file has completed.
Definition: fs_publish.c:796
static void signal_publish_error(struct GNUNET_FS_FileInformation *p, struct GNUNET_FS_PublishContext *pc, const char *emsg)
Generate the callback that signals clients that a file (or directory) has encountered a problem durin...
Definition: fs_publish.c:188
const struct GNUNET_FS_FileInformation * fi
Information about the file that is being publishd.
struct GNUNET_FS_Uri * GNUNET_FS_uri_dup(const struct GNUNET_FS_Uri *uri)
Duplicate URI.
Definition: fs_uri.c:995
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
Type of a block representing a block to be encoded on demand from disk.
static uint64_t compute_contents_size(struct GNUNET_FS_FileInformation *fi)
Calculate the total size of all of the files in the directory structure.
Definition: fs_publish.c:1401
static void handle_signature_response(void *cls, const struct ResponseLocSignatureMessage *sig)
Process the response from the "fs" service to our LOC sign request.
Definition: fs_publish.c:953
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:94
static void finish_reserve(void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
We have gotten a reply for our space reservation request.
Definition: fs_publish.c:1368
#define GNUNET_MESSAGE_TYPE_FS_INDEX_START_OK
Affirmative response to a request for start indexing.
void GNUNET_FS_publish_main_(void *cls)
Main function that performs the upload.
Definition: fs_publish.c:1050
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
Notification that we have stopped the process of uploading a file structure; no futher events will be...
int GNUNET_FS_uri_test_loc(const struct GNUNET_FS_Uri *uri)
Is this a location URI?
Definition: fs_uri.c:1392
#define MAX_INLINE_SIZE
Maximum size for a file to be considered for inlining in a directory.
Definition: fs_api.h:49
static void publish_kblocks(struct GNUNET_FS_PublishContext *pc)
We&#39;ve computed the CHK/LOC URI, now publish the KSKs (if applicable).
Definition: fs_publish.c:922
static struct GNUNET_FS_Uri * uri
Value of URI provided on command-line (when not publishing a file but just creating UBlocks to refer ...
static int fip_signal_stop(void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri **uri, struct GNUNET_FS_BlockOptions *bo, int *do_index, void **client_info)
Signal the FS&#39;s progress function that we are stopping an upload.
Definition: fs_publish.c:1508
uint32_t anonymity_level
At which anonymity level should the block be shared? (0: no anonymity, 1: normal GAP, >1: with cover traffic).
struct GNUNET_FS_PublishKskContext * GNUNET_FS_publish_ksk(struct GNUNET_FS_Handle *h, const struct GNUNET_FS_Uri *ksk_uri, const struct GNUNET_CONTAINER_MetaData *meta, const struct GNUNET_FS_Uri *uri, const struct GNUNET_FS_BlockOptions *bo, enum GNUNET_FS_PublishOptions options, GNUNET_FS_PublishContinuation cont, void *cont_cls)
Publish a CHK under various keywords on GNUnet.
unsigned int GNUNET_FS_compute_depth(uint64_t flen)
Compute the depth of the CHK tree.
Definition: fs_tree.c:124
static void publish_sblock(struct GNUNET_FS_PublishContext *pc)
We are almost done publishing the structure, add SBlocks (if needed).
Definition: fs_publish.c:284
Notification that we have started to publish a file structure.
int GNUNET_asprintf(char **buf, const char *format,...)
Like asprintf, just portable.
Meta data to associate with a file, directory or namespace.
#define GNUNET_SIGNATURE_PURPOSE_PEER_PLACEMENT
Signature by which a peer affirms that it is providing a certain bit of content (used in LOCation URI...
#define GNUNET_MQ_msg_extra(mvar, esize, type)
Allocate an envelope, with extra space allocated after the space needed by the message struct...
Definition: gnunet_mq_lib.h:52
void GNUNET_FS_directory_builder_add(struct GNUNET_FS_DirectoryBuilder *bld, const struct GNUNET_FS_Uri *uri, const struct GNUNET_CONTAINER_MetaData *md, const void *data)
Add an entry to a directory.
Definition: fs_directory.c:384
struct GNUNET_FS_TreeEncoder * te
Encoder being used to publish this file.
Definition: fs_api.h:283
union GNUNET_FS_ProgressInfo::@28 value
Values that depend on the event type.
static void create_loc_uri(struct GNUNET_FS_PublishContext *pc)
We&#39;re publishing without anonymity.
Definition: fs_publish.c:1005
void GNUNET_FS_publish_sync_(struct GNUNET_FS_PublishContext *pc)
Synchronize this publishing struct with its mirror on disk.
Definition: fs_api.c:1705
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_with_priority(enum GNUNET_SCHEDULER_Priority prio, GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run with a specified priority.
Definition: scheduler.c:1191
static struct GNUNET_OS_Process * p
Helper process we started.
Definition: gnunet-qr.c:59
static void progress_proc(void *cls, uint64_t offset, const void *pt_block, size_t pt_size, unsigned int depth)
Function called with information about our progress in computing the tree encoding.
Definition: fs_publish.c:564
Notification that we are making progress sharing a directory.
#define GNUNET_MQ_hd_var_size(name, code, str, ctx)
static char * fn
Filename of the unique file.
struct GNUNET_FS_UnindexContext * GNUNET_FS_unindex_start(struct GNUNET_FS_Handle *h, const char *filename, void *cctx)
Unindex a file.
Definition: fs_unindex.c:793
void GNUNET_FS_end_top(struct GNUNET_FS_Handle *h, struct TopLevelActivity *top)
Destroy a top-level activity entry.
Definition: fs_api.c:412
uint64_t device
ID of device containing the file, as seen by the client.
Definition: fs.h:166
char * nuid
ID for future updates, NULL if we have no namespace or no updates.
Definition: fs_api.h:1194
enum GNUNET_FS_Status status
Specific status code (determines the event type).
int is_directory
Is this struct for a file or directory?
Definition: fs_api.h:379
struct GNUNET_HashCode query
Hash of the encrypted content, used for querying.
Definition: fs.h:62
union GNUNET_FS_Uri::@16 data
void GNUNET_FS_publish_sks_cancel(struct GNUNET_FS_PublishSksContext *psc)
Abort the SKS publishing operation.
Definition: fs_namespace.c:493
#define GNUNET_MIN(a, b)
Definition: gnunet_common.h:80
struct GNUNET_FS_ProgressInfo::@28::@29 publish
Values for all "GNUNET_FS_STATUS_PUBLISH_*" events.
struct GNUNET_TIME_AbsoluteNBO expiration_time
Expiration time that was actually used (rounded!).
Definition: fs.h:121
static char buf[2048]
static void handle_index_start_failed(void *cls, const struct GNUNET_MessageHeader *msg)
Process the response from the "fs" service to our &#39;start index&#39; request.
Definition: fs_publish.c:710
#define GNUNET_TIME_UNIT_FOREVER_REL
Constant used to specify "forever".
struct GNUNET_FS_FileInformation * fi_pos
Current position in the file-tree for the upload.
Definition: fs_api.h:1211
GNUNET_FS_PublishOptions
Options for publishing.
static int check_index_start_failed(void *cls, const struct GNUNET_MessageHeader *msg)
Check the response from the "fs" service to our &#39;start index&#39; request.
Definition: fs_publish.c:687
struct GNUNET_FS_Uri * chk_uri
CHK for this file or directory.
Definition: fs_api.h:255
struct GNUNET_FS_TreeEncoder * GNUNET_FS_tree_encoder_create(struct GNUNET_FS_Handle *h, uint64_t size, void *cls, GNUNET_FS_DataReader reader, GNUNET_FS_TreeBlockProcessor proc, GNUNET_FS_TreeProgressCallback progress, GNUNET_SCHEDULER_TaskCallback cont)
Initialize a tree encoder.
Definition: fs_tree.c:268
void GNUNET_CRYPTO_hash_file_cancel(struct GNUNET_CRYPTO_FileHashContext *fhc)
Cancel a file hashing operation.
static void publish_cleanup(struct GNUNET_FS_PublishContext *pc)
Cleanup the publish context, we&#39;re done with it.
Definition: fs_publish.c:77
struct GNUNET_FS_PublishKskContext * ksk_pc
Context for KSK publishing operation that is part of this publishing operation (NULL if not active)...
Definition: fs_api.h:1238
Handle for controlling a publication process.
Definition: fs_api.h:1165
A 512-bit hashcode.
Message handler for a specific message type.
static int res
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:118
Run when otherwise idle.
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:40
uint64_t contents_size
Sum of all of the sizes of all of the files in the directory.
Definition: fs_api.h:372
static size_t block_reader(void *cls, uint64_t offset, size_t max, void *buf, char **emsg)
Function called by the tree encoder to obtain a block of plaintext data (for the lowest level of the ...
Definition: fs_publish.c:366
uint64_t GNUNET_htonll(uint64_t n)
Convert unsigned 64-bit integer to network byte order.
Definition: common_endian.c:35
static void suspend_operation(struct GNUNET_FS_FileInformation *fi, struct GNUNET_FS_PublishContext *pc)
Actually signal the FS&#39;s progress function that we are suspending an upload.
Definition: fs_publish.c:1253
#define GNUNET_MESSAGE_TYPE_FS_INDEX_START
Message sent by fs client to start indexing.
uint32_t reserve_entries
Overall number of entries to reserve for the publish operation.
Definition: fs_api.h:1255
#define GNUNET_FS_SYNC_PATH_FILE_INFO
Name of the directory with files that are being published.
Definition: fs_api.h:81
struct GNUNET_DATASTORE_QueueEntry * qre
Queue entry for reservation/unreservation.
Definition: fs_api.h:1226
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
static unsigned int size
Size of the "table".
Definition: peer.c:66
struct GNUNET_PeerIdentity peer
Identity of the peer sharing the file.
Definition: fs.h:131
struct GNUNET_HashCode file_id
Hash of the file that we would like to index.
Definition: fs.h:179
int all_done
Set to GNUNET_YES if all processing has completed.
Definition: fs_api.h:1277
char * GNUNET_STRINGS_filename_expand(const char *fil)
Complete filename (a la shell) from abbrevition.
Definition: strings.c:602
Settings for publishing a block (which may of course also apply to an entire directory or file)...
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
struct GNUNET_HashCode file_id
Hash code of the entire content of the file that was indexed (used to uniquely identify the plaintext...
Definition: block_fs.h:87
Notification that we completed sharing a file structure.
struct GNUNET_FS_PublishSksContext * sks_pc
Context for SKS publishing operation that is part of this publishing operation (NULL if not active)...
Definition: fs_api.h:1232
struct GNUNET_TIME_Absolute expirationTime
Time when this location URI expires.
Definition: fs_api.h:124
static struct GNUNET_NAMESTORE_Handle * ns
Handle to the namestore.
enum GNUNET_FS_UriType type
Type of the URI.
Definition: fs_api.h:166
struct GNUNET_FS_FileInformation * next
Files in a directory are kept as a linked list.
Definition: fs_api.h:223
int GNUNET_FS_meta_data_test_for_directory(const struct GNUNET_CONTAINER_MetaData *md)
Does the meta-data claim that this is a directory? Checks if the mime-type is that of a GNUnet direct...
Definition: fs_directory.c:54
struct GNUNET_MQ_Handle * mq
Our own message queue for the FS service; only briefly used when we start to index a file...
Definition: fs_api.h:1206
void * GNUNET_FS_publish_make_status_(struct GNUNET_FS_ProgressInfo *pi, struct GNUNET_FS_PublishContext *pc, const struct GNUNET_FS_FileInformation *p, uint64_t offset)
Fill in all of the generic fields for a publish event and call the callback.
Definition: fs_publish.c:47
#define GNUNET_TIME_UNIT_ZERO
Relative time zero.
#define GNUNET_MAX_MESSAGE_SIZE
Largest supported message (to be precise, one byte more than the largest possible message...
struct ContentHashKey chk
Information about the shared file (to be signed).
Definition: fs.h:93
static int fip_signal_start(void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri **uri, struct GNUNET_FS_BlockOptions *bo, int *do_index, void **client_info)
Signal the FS&#39;s progress function that we are starting an upload.
Definition: fs_publish.c:1180
struct GNUNET_FS_BlockOptions bo
Block options for the file.
Definition: fs_api.h:266
struct GNUNET_FS_Handle * fsh
File-sharing handle that generated the event.
struct TopLevelActivity * GNUNET_FS_make_top(struct GNUNET_FS_Handle *h, SuspendSignalFunction ssf, void *ssf_cls)
Create a top-level activity entry.
Definition: fs_api.c:391
Message sent from the service with the signed LOC URI.
Definition: fs.h:105
int do_index
Should the file be indexed or inserted?
Definition: fs_api.h:328
uint64_t inode
Inode of the file on the given device, as seen by the client ("st_ino" field from "struct stat")...
Definition: fs.h:174
struct TopLevelActivity * top
Our top-level activity entry (if we are top-level, otherwise NULL).
Definition: fs_api.h:1174
void GNUNET_DATASTORE_cancel(struct GNUNET_DATASTORE_QueueEntry *qe)
Cancel a datastore operation.
static struct GNUNET_FS_PublishContext * pc
Handle to FS-publishing operation.
struct GNUNET_FS_FileInformation * fi
File-structure that is being shared.
Definition: fs_api.h:1179
union GNUNET_FS_FileInformation::@19 data
Data describing either the file or the directory.
Notification that an error was encountered sharing a file structure.
GNUNET_FS_ProgressCallback upcb
Function to call with updates on our progress.
Definition: fs_api.h:1070
Notification that we have suspended sharing a file structure.
Message sent from a GNUnet (fs) publishing activity to sign a LOC URI.
Definition: fs.h:73
static struct GNUNET_FS_DirectoryBuilder * db
Definition: gnunet-search.c:41
#define HASHING_BLOCKSIZE
Blocksize to use when hashing files for indexing (blocksize for IO, not for the DBlocks).
Definition: fs.h:47
uint64_t GNUNET_FS_uri_chk_get_file_size(const struct GNUNET_FS_Uri *uri)
What is the size of the file that this URI refers to?
Definition: fs_uri.c:1368
shared definitions for the FS library
void GNUNET_DATASTORE_disconnect(struct GNUNET_DATASTORE_Handle *h, int drop)
Disconnect from the datastore service (and free associated resources).
struct GNUNET_CRYPTO_EcdsaPrivateKey * ns
Namespace that we are publishing in, NULL if we have no namespace.
Definition: fs_api.h:1184
void GNUNET_FS_file_information_inspect(struct GNUNET_FS_FileInformation *dir, GNUNET_FS_FileInformationProcessor proc, void *proc_cls)
Inspect a file or directory in a publish-structure.
A Universal Resource Identifier (URI), opaque.
Definition: fs_api.h:162
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:373
Handle to the datastore service.
uint64_t file_length
Total size of the file in bytes.
Definition: fs_api.h:97
static struct GNUNET_CONTAINER_MetaData * meta
Meta-data provided via command-line option.
static struct GNUNET_DATASTORE_Handle * dsh
Datastore handle.
struct GNUNET_CRYPTO_EddsaSignature signature
The requested signature.
Definition: fs.h:126
void * upcb_cls
Closure for upcb.
Definition: fs_api.h:1075
struct GNUNET_CRYPTO_EddsaSignature contentSignature
Signature over the GNUNET_EC_FileIdentifier, peer identity and expiration time.
Definition: fs_api.h:130
struct GNUNET_FS_Uri * keywords
Keywords to use for KBlocks.
Definition: fs_api.h:249
#define GNUNET_log(kind,...)
void GNUNET_FS_publish_signal_suspend_(void *cls)
Create SUSPEND event for the given publish operation and then clean up our state (without stop signal...
Definition: fs_publish.c:1340
uint64_t offset
At which offset should we be able to find this on-demand encoded block? (in NBO)
Definition: block_fs.h:93
void GNUNET_FS_file_information_destroy(struct GNUNET_FS_FileInformation *fi, GNUNET_FS_FileInformationProcessor cleaner, void *cleaner_cls)
Destroy publish-structure.
static void loc_mq_error_handler(void *cls, enum GNUNET_MQ_Error error)
Generic error handler, called with the appropriate error code and the same closure specified at the c...
Definition: fs_publish.c:981
#define GNUNET_MESSAGE_TYPE_FS_REQUEST_LOC_SIGN
Message sent by fs client to request LOC signature.
Data block (leaf) in the CHK tree.
unsigned int GNUNET_FS_uri_ksk_get_keyword_count(const struct GNUNET_FS_Uri *uri)
How many keywords are ANDed in this keyword URI?
Definition: fs_uri.c:709
#define GNUNET_FS_SYNC_PATH_MASTER_PUBLISH
Name of the directory with publishing operations.
Definition: fs_api.h:76
Location (chk with identity of hosting peer).
Definition: fs_api.h:155
struct GNUNET_TIME_AbsoluteNBO expiration_time
Requested expiration time.
Definition: fs.h:88
struct GNUNET_TIME_Relative GNUNET_TIME_calculate_eta(struct GNUNET_TIME_Absolute start, uint64_t finished, uint64_t total)
Calculate the estimate time of arrival/completion for an operation.
Definition: time.c:549
struct Location loc
Information needed to retrieve a file including signed location (identity of a peer) of the content...
Definition: fs_api.h:210
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Header for all communications.
void GNUNET_FS_tree_encoder_finish(struct GNUNET_FS_TreeEncoder *te, char **emsg)
Clean up a tree encoder and return information about possible errors.
Definition: fs_tree.c:442
Time for absolute times used by GNUnet, in microseconds.
void GNUNET_MQ_destroy(struct GNUNET_MQ_Handle *mq)
Destroy the message queue.
Definition: mq.c:821
#define GNUNET_YES
Definition: gnunet_common.h:77
void GNUNET_MQ_send(struct GNUNET_MQ_Handle *mq, struct GNUNET_MQ_Envelope *ev)
Send a message with the given message queue.
Definition: mq.c:351
static void block_proc(void *cls, const struct ContentHashKey *chk, uint64_t offset, unsigned int depth, enum GNUNET_BLOCK_Type type, const void *block, uint16_t block_size)
Function called asking for the current (encoded) block to be processed.
Definition: fs_publish.c:480
struct GNUNET_DATASTORE_Handle * GNUNET_DATASTORE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the datastore service.
struct GNUNET_FS_DirectoryBuilder * GNUNET_FS_directory_builder_create(const struct GNUNET_CONTAINER_MetaData *mdir)
Create a directory builder.
Definition: fs_directory.c:358
static int fip_signal_suspend(void *cls, struct GNUNET_FS_FileInformation *fi, uint64_t length, struct GNUNET_CONTAINER_MetaData *meta, struct GNUNET_FS_Uri **uri, struct GNUNET_FS_BlockOptions *bo, int *do_index, void **client_info)
Signal the FS&#39;s progress function that we are suspending an upload.
Definition: fs_publish.c:1305
struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton(struct GNUNET_TIME_Absolute a)
Convert absolute time to network byte order.
Definition: time.c:655
int any_done
Set to GNUNET_YES if we were able to publish any block.
Definition: fs_api.h:1272
int skip_next_fi_callback
Flag set to GNUNET_YES if the next callback from GNUNET_FS_file_information_inspect should be skipped...
Definition: fs_api.h:1284
static void encode_cont(void *cls)
The tree encoder has finished processing a file.
Definition: fs_publish.c:416
size_t dir_size
Size of the directory itself (in bytes); 0 if the size has not yet been calculated.
Definition: fs_api.h:356
struct GNUNET_FS_Uri * sks_uri
SKS URI for this file or directory.
Definition: fs_api.h:261
static void publish_sblocks_cont(void *cls, const struct GNUNET_FS_Uri *uri, const char *emsg)
We&#39;ve finished publishing the SBlock as part of a larger upload.
Definition: fs_publish.c:245
#define GNUNET_MQ_handler_end()
End-marker for the handlers array.
#define GNUNET_malloc(size)
Wrapper around malloc.
void GNUNET_FS_remove_sync_file_(struct GNUNET_FS_Handle *h, const char *ext, const char *ent)
Remove serialization/deserialization file from disk.
Definition: fs_api.c:743
uint64_t GNUNET_ntohll(uint64_t n)
Convert unsigned 64-bit integer to host byte order.
Definition: common_endian.c:48
index block (indexing a DBlock that can be obtained directly from reading the plaintext file) ...
Definition: block_fs.h:81
static void index_mq_error_handler(void *cls, enum GNUNET_MQ_Error error)
Generic error handler, called with the appropriate error code and the same closure specified at the c...
Definition: fs_publish.c:766
static struct GNUNET_FS_BlockOptions bo
Options we set for published blocks.
#define GNUNET_free(ptr)
Wrapper around free.
Argument given to the progress callback with information about what is going on.
Merkle-tree-ish-CHK file encoding for GNUnet.
void * GNUNET_SCHEDULER_cancel(struct GNUNET_SCHEDULER_Task *task)
Cancel the task with the specified identifier.
Definition: scheduler.c:956
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration to use.
Definition: fs_api.h:1060