GNUnet  0.10.x
gnunet-service-fs_indexing.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2009, 2010 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19 */
20 
26 #include "platform.h"
27 #include <float.h>
28 #include "gnunet_core_service.h"
30 #include "gnunet_peer_lib.h"
31 #include "gnunet_protocols.h"
32 #include "gnunet_signatures.h"
33 #include "gnunet_util_lib.h"
34 #include "gnunet-service-fs.h"
36 #include "fs.h"
37 
42 struct IndexInfo
43 {
44 
48  struct IndexInfo *next;
49 
53  struct IndexInfo *prev;
54 
59  const char *filename;
60 
65  struct GNUNET_SERVER_TransmitContext *tc;
66 
71 
76 
77 };
78 
79 
85 
90 
97 
101 static const struct GNUNET_CONFIGURATION_Handle *cfg;
102 
108 
109 
113 static void
115 {
116  struct GNUNET_BIO_WriteHandle *wh;
117  char *fn;
118  struct IndexInfo *pos;
119 
120  if (GNUNET_OK !=
122  "INDEXDB",
123  &fn))
124  {
126  "fs",
127  "INDEXDB");
128  return;
129  }
130  wh = GNUNET_BIO_write_open (fn);
131  if (NULL == wh)
132  {
134  _("Could not open `%s'.\n"),
135  fn);
136  GNUNET_free (fn);
137  return;
138  }
139  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
140  if ((GNUNET_OK !=
141  GNUNET_BIO_write (wh,
142  &pos->file_id,
143  sizeof (struct GNUNET_HashCode))) ||
144  (GNUNET_OK !=
146  pos->filename)))
147  break;
148  if (GNUNET_OK != GNUNET_BIO_write_close (wh))
149  {
151  _("Error writing `%s'.\n"),
152  fn);
153  GNUNET_free (fn);
154  return;
155  }
156  GNUNET_free (fn);
157 }
158 
159 
163 static void
165 {
166  struct GNUNET_BIO_ReadHandle *rh;
167  char *fn;
168  struct IndexInfo *pos;
169  char *fname;
170  struct GNUNET_HashCode hc;
171  size_t slen;
172  char *emsg;
173 
174  if (GNUNET_OK !=
176  "FS",
177  "INDEXDB",
178  &fn))
179  {
181  "fs",
182  "INDEXDB");
183  return;
184  }
185  if (GNUNET_NO == GNUNET_DISK_file_test (fn))
186  {
187  /* no index info yet */
188  GNUNET_free (fn);
189  return;
190  }
191  rh = GNUNET_BIO_read_open (fn);
192  if (NULL == rh)
193  {
195  _("Could not open `%s'.\n"),
196  fn);
197  GNUNET_free (fn);
198  return;
199  }
200  while ( (GNUNET_OK ==
201  GNUNET_BIO_read (rh,
202  "Hash of indexed file",
203  &hc,
204  sizeof (struct GNUNET_HashCode))) &&
205  (GNUNET_OK ==
207  "Name of indexed file",
208  &fname,
209  1024 * 16)) &&
210  (fname != NULL) )
211  {
212  slen = strlen (fname) + 1;
213  pos = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
214  pos->file_id = hc;
215  pos->filename = (const char *) &pos[1];
216  GNUNET_memcpy (&pos[1], fname, slen);
217  if (GNUNET_SYSERR ==
220  {
221  GNUNET_free (pos);
222  }
223  else
224  {
225  GNUNET_CONTAINER_DLL_insert (indexed_files_head,
226  indexed_files_tail,
227  pos);
228  }
229  GNUNET_free (fname);
230  }
231  if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
232  GNUNET_free (emsg);
233  GNUNET_free (fn);
234 }
235 
236 
246 static void
247 remove_cont (void *cls, int success,
249  const char *msg)
250 {
251  if (GNUNET_OK != success)
253  _("Failed to delete bogus block: %s\n"), msg);
254 }
255 
256 
277 int
279  uint32_t size,
280  const void *data,
281  enum GNUNET_BLOCK_Type type,
282  uint32_t priority,
283  uint32_t anonymity,
284  uint32_t replication,
286  uint64_t uid,
288  void *cont_cls)
289 {
290  const struct OnDemandBlock *odb;
291  struct GNUNET_HashCode nkey;
294  struct GNUNET_HashCode query;
295  ssize_t nsize;
296  char ndata[DBLOCK_SIZE];
297  char edata[DBLOCK_SIZE];
298  const char *fn;
299  struct GNUNET_DISK_FileHandle *fh;
300  uint64_t off;
301  struct IndexInfo *ii;
302 
303  if (size != sizeof (struct OnDemandBlock))
304  {
305  GNUNET_break (0);
307  key,
308  size,
309  data,
310  -1,
311  -1,
312  &remove_cont, NULL);
313  return GNUNET_SYSERR;
314  }
315  odb = (const struct OnDemandBlock *) data;
316  off = GNUNET_ntohll (odb->offset);
318  &odb->file_id);
319  if (NULL == ii)
320  {
321  GNUNET_break (0);
323  "Failed to find index %s\n",
324  GNUNET_h2s (&odb->file_id));
325  return GNUNET_SYSERR;
326  }
327  fn = ii->filename;
328  if ((NULL == fn) || (0 != ACCESS (fn, R_OK)))
329  {
331  gettext_noop ("# index blocks removed: original file inaccessible"),
332  1,
333  GNUNET_YES);
335  key,
336  size,
337  data,
338  -1,
339  -1,
340  &remove_cont,
341  NULL);
342  return GNUNET_SYSERR;
343  }
344  if ( (NULL ==
345  (fh =
349  (off != GNUNET_DISK_file_seek (fh,
350  off,
352  (-1 == (nsize = GNUNET_DISK_file_read (fh,
353  ndata,
354  sizeof (ndata)))) )
355  {
357  _("Could not access indexed file `%s' (%s) at offset %llu: %s\n"),
358  GNUNET_h2s (&odb->file_id),
359  fn,
360  (unsigned long long) off,
361  (fn == NULL) ? _("not indexed") : STRERROR (errno));
362  if (fh != NULL)
365  key,
366  size,
367  data,
368  -1,
369  -1,
370  &remove_cont,
371  NULL);
372  return GNUNET_SYSERR;
373  }
375  GNUNET_CRYPTO_hash (ndata,
376  nsize,
377  &nkey);
379  &skey,
380  &iv);
382  nsize,
383  &skey,
384  &iv,
385  edata);
386  GNUNET_CRYPTO_hash (edata,
387  nsize,
388  &query);
389  if (0 != memcmp (&query,
390  key,
391  sizeof (struct GNUNET_HashCode)))
392  {
394  _("Indexed file `%s' changed at offset %llu\n"),
395  fn,
396  (unsigned long long) off);
398  key,
399  size,
400  data,
401  -1,
402  -1,
403  &remove_cont,
404  NULL);
405  return GNUNET_SYSERR;
406  }
408  "On-demand encoded block for query `%s'\n",
409  GNUNET_h2s (key));
410  cont (cont_cls,
411  key,
412  nsize,
413  edata,
415  priority,
416  anonymity,
417  replication,
418  expiration,
419  uid);
420  return GNUNET_OK;
421 }
422 
423 
429 void
431 {
432  struct GNUNET_MQ_Envelope *env;
433  struct IndexInfoMessage *iim;
434  struct GNUNET_MessageHeader *iem;
435  size_t slen;
436  const char *fn;
437  struct IndexInfo *pos;
438 
439  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
440  {
441  fn = pos->filename;
442  slen = strlen (fn) + 1;
443  if (slen + sizeof (struct IndexInfoMessage) >=
445  {
446  GNUNET_break (0);
447  break;
448  }
449  env = GNUNET_MQ_msg_extra (iim,
450  slen,
452  iim->reserved = 0;
453  iim->file_id = pos->file_id;
454  GNUNET_memcpy (&iim[1],
455  fn,
456  slen);
457  GNUNET_MQ_send (mq,
458  env);
459  }
460  env = GNUNET_MQ_msg (iem,
462  GNUNET_MQ_send (mq,
463  env);
464 }
465 
466 
473 int
475 {
476  struct IndexInfo *pos;
477 
478  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
479  {
480  if (0 == memcmp (&pos->file_id,
481  fid,
482  sizeof (struct GNUNET_HashCode)))
483  {
484  GNUNET_CONTAINER_DLL_remove (indexed_files_head,
485  indexed_files_tail,
486  pos);
489  &pos->file_id,
490  pos));
491  GNUNET_free (pos);
492  write_index_list ();
493  return GNUNET_YES;
494  }
495  }
496  return GNUNET_NO;
497 }
498 
499 
506 void
508  const struct GNUNET_HashCode *file_id)
509 {
510  struct IndexInfo *ii;
511  size_t slen;
512 
514  file_id);
515  if (NULL != ii)
516  {
518  _("Index request received for file `%s' is already indexed as `%s'. Permitting anyway.\n"),
519  filename,
520  ii->filename);
521  return;
522  }
524  "Adding file %s to index as %s\n",
525  filename,
526  GNUNET_h2s (file_id));
527  slen = strlen (filename) + 1;
528  ii = GNUNET_malloc (sizeof (struct IndexInfo) + slen);
529  ii->file_id = *file_id;
530  ii->filename = (const char *) &ii[1];
531  GNUNET_memcpy (&ii[1],
532  filename,
533  slen);
534  GNUNET_CONTAINER_DLL_insert (indexed_files_head,
535  indexed_files_tail,
536  ii);
539  &ii->file_id,
540  ii,
542  write_index_list ();
543 }
544 
545 
549 void
551 {
552  struct IndexInfo *pos;
553 
554  while (NULL != (pos = indexed_files_head))
555  {
556  GNUNET_CONTAINER_DLL_remove (indexed_files_head,
557  indexed_files_tail,
558  pos);
559  if (pos->fhc != NULL)
563  &pos->file_id,
564  pos));
565  GNUNET_free (pos);
566  }
568  ifm = NULL;
569  cfg = NULL;
570 }
571 
572 
579 int
581  struct GNUNET_DATASTORE_Handle *d)
582 {
583  cfg = c;
584  dsh = d;
586  GNUNET_YES);
587  read_index_list ();
588  return GNUNET_OK;
589 }
590 
591 /* end of gnunet-service-fs_indexing.c */
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
int GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory)...
Definition: disk.c:669
static struct GNUNET_TIME_Absolute min_expiration
Minimum time that content should have to not be discarded instantly (time stamp of any content that w...
Open the file for reading.
int GNUNET_FS_indexing_do_unindex(const struct GNUNET_HashCode *fid)
Remove a file from the index.
struct IndexInfo * next
This is a doubly linked list.
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
static char * expiration
Credential TTL.
struct GNUNET_HashCode file_id
Hash of the contents of the file.
int GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1817
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
struct GNUNET_SERVER_TransmitContext * tc
Context for transmitting confirmation to client, NULL if we&#39;ve done this already. ...
static void remove_cont(void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called from datastore&#39;s remove function.
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:881
void GNUNET_FS_indexing_done()
Shutdown the module.
static struct GNUNET_CONTAINER_MultiHashMap * ifm
Maps hash over content of indexed files to the respective &#39;struct IndexInfo&#39;.
int GNUNET_BIO_write_string(struct GNUNET_BIO_WriteHandle *h, const char *s)
Write a string to a file.
Definition: bio.c:547
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define STRERROR(i)
Definition: plibc.h:676
const char * filename
Name of the indexed file.
static unsigned int replication
definitions for the entire fs module
Nobody is allowed to do anything to the file.
#define GNUNET_MQ_msg(mvar, type)
Allocate a GNUNET_MQ_Envelope.
Definition: gnunet_mq_lib.h:67
struct GNUNET_STATISTICS_Handle * GSF_stats
Handle for reporting statistics.
#define GNUNET_NO
Definition: gnunet_common.h:81
shared data structures of gnunet-service-fs.c
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:78
Message send by FS service in response to a request asking for a list of all indexed files...
Definition: fs.h:199
static struct IndexInfo * indexed_files_head
Head of linked list of indexed files.
off_t GNUNET_DISK_file_seek(const struct GNUNET_DISK_FileHandle *h, off_t offset, enum GNUNET_DISK_Seek whence)
Move the read/write pointer in a file.
Definition: disk.c:241
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
struct GNUNET_HashCode file_id
Hash of the indexed file.
Definition: fs.h:215
Internal representation of the hash map.
int GNUNET_BIO_read_close(struct GNUNET_BIO_ReadHandle *h, char **emsg)
Close an open file.
Definition: bio.c:119
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
int GNUNET_FS_indexing_init(const struct GNUNET_CONFIGURATION_Handle *c, struct GNUNET_DATASTORE_Handle *d)
Initialize the indexing submodule.
void GNUNET_FS_indexing_send_list(struct GNUNET_MQ_Handle *mq)
Transmit information about indexed files to mq.
void * GNUNET_CONTAINER_multihashmap_get(const struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key)
Given a key find a value in the map matching the key.
#define _(String)
GNU gettext support macro.
Definition: platform.h:208
struct GNUNET_BIO_ReadHandle * GNUNET_BIO_read_open(const char *fn)
Open a file for reading.
Definition: bio.c:94
#define GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY
Reply to client with an indexed file name.
int GNUNET_BIO_read(struct GNUNET_BIO_ReadHandle *h, const char *what, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: bio.c:145
#define GNUNET_MQ_msg_extra(mvar, esize, type)
Allocate an envelope, with extra space allocated after the space needed by the message struct...
Definition: gnunet_mq_lib.h:52
#define GNUNET_memcpy(dst, src, n)
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
static char * fn
Filename of the unique file.
static void write_index_list()
Write the current index information list to disk.
void GNUNET_CRYPTO_hash(const void *block, size_t size, struct GNUNET_HashCode *ret)
Compute hash of a given block.
Definition: crypto_hash.c:44
ssize_t GNUNET_CRYPTO_symmetric_encrypt(const void *block, size_t size, const struct GNUNET_CRYPTO_SymmetricSessionKey *sessionkey, const struct GNUNET_CRYPTO_SymmetricInitializationVector *iv, void *result)
Encrypt a block using a symmetric sessionkey.
void GNUNET_CONTAINER_multihashmap_destroy(struct GNUNET_CONTAINER_MultiHashMap *map)
Destroy a hash map.
void(* GNUNET_DATASTORE_DatumProcessor)(void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Process a datum that was stored in the datastore.
int GNUNET_CONTAINER_multihashmap_remove(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, const void *value)
Remove the given key-value pair from the map.
static int fh
Handle to the unique file.
Handle for buffered writing.
Definition: bio.c:399
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_remove(struct GNUNET_DATASTORE_Handle *h, const struct GNUNET_HashCode *key, size_t size, const void *data, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Explicitly remove some content from the database.
void GNUNET_CRYPTO_hash_file_cancel(struct GNUNET_CRYPTO_FileHashContext *fhc)
Cancel a file hashing operation.
A 512-bit hashcode.
In-memory information about indexed files (also available on-disk).
int GNUNET_BIO_read_string(struct GNUNET_BIO_ReadHandle *h, const char *what, char **result, size_t max_length)
Read 0-terminated string from a file.
Definition: bio.c:237
void GNUNET_CRYPTO_hash_to_aes_key(const struct GNUNET_HashCode *hc, struct GNUNET_CRYPTO_SymmetricSessionKey *skey, struct GNUNET_CRYPTO_SymmetricInitializationVector *iv)
Convert a hashcode into a key.
Definition: crypto_hash.c:212
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:40
There must only be one value per key; storing a value should fail if a value under the same key alrea...
struct GNUNET_HashCode key
The key used in the DHT.
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
static unsigned int size
Size of the "table".
Definition: peer.c:67
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
struct GNUNET_HashCode file_id
Hash code of the entire content of the file that was indexed (used to uniquely identify the plaintext...
Definition: block_fs.h:91
indexing for the file-sharing service
static struct GNUNET_TESTBED_BarrierWaitHandle * wh
Our barrier wait handle.
int GNUNET_CONTAINER_multihashmap_put(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, void *value, enum GNUNET_CONTAINER_MultiHashMapOption opt)
Store a key-value pair in the map.
Context used when hashing a file.
int GNUNET_BIO_write_close(struct GNUNET_BIO_WriteHandle *h)
Close an open file for writing.
Definition: bio.c:457
#define GNUNET_MAX_MESSAGE_SIZE
Largest supported message (to be precise, one byte more than the largest possible message...
Handle to a message queue.
Definition: mq.c:85
#define GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END
Reply to client indicating end of list.
static void read_index_list()
Read index information from disk.
Seek an absolute position (from the start of the file).
configuration data
Definition: configuration.c:85
struct GNUNET_BIO_WriteHandle * GNUNET_BIO_write_open(const char *fn)
Open a file for writing.
Definition: bio.c:430
Handle to the datastore service.
static struct GNUNET_DATASTORE_Handle * dsh
Datastore handle.
struct GNUNET_CRYPTO_FileHashContext * fhc
Context for hashing of the file.
struct GNUNET_MQ_Handle * mq
Definition: 003.c:5
#define GNUNET_log(kind,...)
uint64_t offset
At which offset should we be able to find this on-demand encoded block? (in NBO)
Definition: block_fs.h:97
int GNUNET_CONFIGURATION_get_value_filename(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be the name of a file or directory.
Data block (leaf) in the CHK tree.
struct GNUNET_CONTAINER_MultiHashMap * GNUNET_CONTAINER_multihashmap_create(unsigned int len, int do_not_copy_keys)
Create a multi hash map.
struct IndexInfo * prev
This is a doubly linked list.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
Header for all communications.
Time for absolute times used by GNUnet, in microseconds.
#define GNUNET_YES
Definition: gnunet_common.h:80
static unsigned int anonymity
void GNUNET_MQ_send(struct GNUNET_MQ_Handle *mq, struct GNUNET_MQ_Envelope *ev)
Send a message with the given message queue.
Definition: mq.c:353
int GNUNET_BIO_write(struct GNUNET_BIO_WriteHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: bio.c:505
static const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
int GNUNET_FS_handle_on_demand_block(const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid, GNUNET_DATASTORE_DatumProcessor cont, void *cont_cls)
We&#39;ve received an on-demand encoded block from the datastore.
uint32_t reserved
Always zero.
Definition: fs.h:210
uint32_t data
The data value.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1673
#define ACCESS(p, m)
Definition: plibc.h:656
Handle used to access files (and pipes).
#define GNUNET_malloc(size)
Wrapper around malloc.
Handle for buffered reading.
Definition: bio.c:53
void GNUNET_FS_add_to_index(const char *filename, const struct GNUNET_HashCode *file_id)
Add the given file to the list of indexed files.
uint64_t GNUNET_ntohll(uint64_t n)
Convert unsigned 64-bit integer to host byte order.
Definition: common_endian.c:48
index block (indexing a DBlock that can be obtained directly from reading the plaintext file) ...
Definition: block_fs.h:84
#define GNUNET_free(ptr)
Wrapper around free.
#define gettext_noop(String)
Definition: gettext.h:69
static struct IndexInfo * indexed_files_tail
Tail of linked list of indexed files.