GNUnet  0.17.5
gnunet-service-fs_indexing.h File Reference

indexing for the file-sharing service More...

#include "gnunet_block_lib.h"
#include "gnunet_core_service.h"
#include "gnunet_datastore_service.h"
#include "gnunet_peer_lib.h"
#include "gnunet_protocols.h"
#include "gnunet_signatures.h"
#include "gnunet_util_lib.h"
Include dependency graph for gnunet-service-fs_indexing.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.


int GNUNET_FS_handle_on_demand_block (const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid, GNUNET_DATASTORE_DatumProcessor cont, void *cont_cls)
 We've received an on-demand encoded block from the datastore. More...
void GNUNET_FS_indexing_send_list (struct GNUNET_MQ_Handle *mq)
 Transmit information about indexed files to mq. More...
int GNUNET_FS_indexing_do_unindex (const struct GNUNET_HashCode *fid)
 Remove a file from the index. More...
void GNUNET_FS_add_to_index (const char *filename, const struct GNUNET_HashCode *file_id)
 Add the given file to the list of indexed files. More...
int GNUNET_FS_indexing_init (const struct GNUNET_CONFIGURATION_Handle *c, struct GNUNET_DATASTORE_Handle *d)
 Initialize the indexing submodule. More...
void GNUNET_FS_indexing_done (void)
 Shutdown the module. More...

Detailed Description

indexing for the file-sharing service

Christian Grothoff

Definition in file gnunet-service-fs_indexing.h.

Function Documentation

◆ GNUNET_FS_handle_on_demand_block()

int GNUNET_FS_handle_on_demand_block ( const struct GNUNET_HashCode key,
uint32_t  size,
const void *  data,
enum GNUNET_BLOCK_Type  type,
uint32_t  priority,
uint32_t  anonymity,
uint32_t  replication,
struct GNUNET_TIME_Absolute  expiration,
uint64_t  uid,
GNUNET_DATASTORE_DatumProcessor  cont,
void *  cont_cls 

We've received an on-demand encoded block from the datastore.

Attempt to do on-demand encoding and (if successful), call the continuation with the resulting block. On error, clean up and ask the datastore for more results.

keykey for the content
sizenumber of bytes in data
datacontent stored
typetype of the content
prioritypriority of the content
anonymityanonymity-level for the content
replicationreplication-level for the content
expirationexpiration time for the content
uidunique identifier for the datum; maybe 0 if no unique identifier is available
contfunction to call with the actual block (at most once, on success)
cont_clsclosure for cont
GNUNET_OK on success

Definition at line 250 of file gnunet-service-fs_indexing.c.

261 {
262  const struct OnDemandBlock *odb;
263  struct GNUNET_HashCode nkey;
266  struct GNUNET_HashCode query;
267  ssize_t nsize;
268  char ndata[DBLOCK_SIZE];
269  char edata[DBLOCK_SIZE];
270  const char *fn;
271  struct GNUNET_DISK_FileHandle *fh;
272  uint64_t off;
273  struct IndexInfo *ii;
275  if (size != sizeof(struct OnDemandBlock))
276  {
277  GNUNET_break (0);
278  GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
279  return GNUNET_SYSERR;
280  }
281  odb = (const struct OnDemandBlock *) data;
282  off = GNUNET_ntohll (odb->offset);
284  if (NULL == ii)
285  {
286  GNUNET_break (0);
288  "Failed to find index %s\n",
289  GNUNET_h2s (&odb->file_id));
290  return GNUNET_SYSERR;
291  }
292  fn = ii->filename;
293  if ((NULL == fn) || (0 != access (fn, R_OK)))
294  {
296  GSF_stats,
297  gettext_noop ("# index blocks removed: original file inaccessible"),
298  1,
300  GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
301  return GNUNET_SYSERR;
302  }
303  if ((NULL == (fh = GNUNET_DISK_file_open (fn,
306  (off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)) ||
307  (-1 == (nsize = GNUNET_DISK_file_read (fh, ndata, sizeof(ndata)))))
308  {
310  _ (
311  "Could not access indexed file `%s' (%s) at offset %llu: %s\n"),
312  GNUNET_h2s (&odb->file_id),
313  fn,
314  (unsigned long long) off,
315  (fn == NULL) ? _ ("not indexed") : strerror (errno));
316  if (fh != NULL)
318  GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
319  return GNUNET_SYSERR;
320  }
322  GNUNET_CRYPTO_hash (ndata, nsize, &nkey);
323  GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
324  GNUNET_CRYPTO_symmetric_encrypt (ndata, nsize, &skey, &iv, edata);
325  GNUNET_CRYPTO_hash (edata, nsize, &query);
326  if (0 != memcmp (&query, key, sizeof(struct GNUNET_HashCode)))
327  {
329  _ ("Indexed file `%s' changed at offset %llu\n"),
330  fn,
331  (unsigned long long) off);
332  GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
333  return GNUNET_SYSERR;
334  }
336  "On-demand encoded block for query `%s'\n",
337  GNUNET_h2s (key));
338  cont (cont_cls,
339  key,
340  nsize,
341  edata,
343  priority,
344  anonymity,
345  replication,
346  expiration,
347  uid);
348  return GNUNET_OK;
349 }
Size of the individual blocks used for file-sharing.
Definition: fs.h:40
#define gettext_noop(String)
Definition: gettext.h:69
static char * expiration
Credential TTL.
Definition: gnunet-abd.c:96
static unsigned int replication
struct GNUNET_HashCode key
The key used in the DHT.
static unsigned int anonymity
uint32_t data
The data value.
struct GNUNET_STATISTICS_Handle * GSF_stats
Handle for reporting statistics.
static struct GNUNET_CONTAINER_MultiHashMap * ifm
Maps hash over content of indexed files to the respective 'struct IndexInfo'.
static void remove_cont(void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called from datastore's remove function.
static struct GNUNET_DATASTORE_Handle * dsh
Datastore handle.
static struct GNUNET_DISK_FileHandle * fh
File handle to STDIN, for reading restart/quit commands.
Data block (leaf) in the CHK tree.
ssize_t GNUNET_CRYPTO_symmetric_encrypt(const void *block, size_t size, const struct GNUNET_CRYPTO_SymmetricSessionKey *sessionkey, const struct GNUNET_CRYPTO_SymmetricInitializationVector *iv, void *result)
Encrypt a block using a symmetric sessionkey.
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_remove(struct GNUNET_DATASTORE_Handle *h, const struct GNUNET_HashCode *key, size_t size, const void *data, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Explicitly remove some content from the database.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1235
off_t GNUNET_DISK_file_seek(const struct GNUNET_DISK_FileHandle *h, off_t offset, enum GNUNET_DISK_Seek whence)
Move the read/write pointer in a file.
Definition: disk.c:206
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1306
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:622
Open the file for reading.
Nobody is allowed to do anything to the file.
Seek an absolute position (from the start of the file).
void GNUNET_CRYPTO_hash(const void *block, size_t size, struct GNUNET_HashCode *ret)
Compute hash of a given block.
Definition: crypto_hash.c:41
void GNUNET_CRYPTO_hash_to_aes_key(const struct GNUNET_HashCode *hc, struct GNUNET_CRYPTO_SymmetricSessionKey *skey, struct GNUNET_CRYPTO_SymmetricInitializationVector *iv)
Convert a hashcode into a key.
Definition: crypto_hash.c:151
void * GNUNET_CONTAINER_multihashmap_get(const struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key)
Given a key find a value in the map matching the key.
#define GNUNET_log(kind,...)
uint64_t GNUNET_ntohll(uint64_t n)
Convert unsigned 64-bit integer to host byte order.
Definition: common_endian.c:53
Definition: gnunet_common.h:99
Definition: gnunet_common.h:97
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
Handle used to access files (and pipes).
A 512-bit hashcode.
In-memory information about indexed files (also available on-disk).
const char * filename
Name of the indexed file.
index block (indexing a DBlock that can be obtained directly from reading the plaintext file)
Definition: block_fs.h:82
uint64_t offset
At which offset should we be able to find this on-demand encoded block? (in NBO)
Definition: block_fs.h:94
struct GNUNET_HashCode file_id
Hash code of the entire content of the file that was indexed (used to uniquely identify the plaintext...
Definition: block_fs.h:88

References _, anonymity, data, DBLOCK_SIZE, dsh, expiration, fh, OnDemandBlock::file_id, IndexInfo::filename, gettext_noop, GNUNET_BLOCK_TYPE_FS_DBLOCK, GNUNET_break, GNUNET_CONTAINER_multihashmap_get(), GNUNET_CRYPTO_hash(), GNUNET_CRYPTO_hash_to_aes_key(), GNUNET_CRYPTO_symmetric_encrypt(), GNUNET_DATASTORE_remove(), GNUNET_DISK_file_close(), GNUNET_DISK_file_open(), GNUNET_DISK_file_read(), GNUNET_DISK_file_seek(), GNUNET_DISK_OPEN_READ, GNUNET_DISK_PERM_NONE, GNUNET_DISK_SEEK_SET, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_WARNING, GNUNET_h2s(), GNUNET_log, GNUNET_ntohll(), GNUNET_OK, GNUNET_STATISTICS_update(), GNUNET_SYSERR, GNUNET_YES, GSF_stats, ifm, key, OnDemandBlock::offset, remove_cont(), replication, and size.

Referenced by handle_datastore_reply(), process_local_reply(), and process_migration_content().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GNUNET_FS_indexing_send_list()

void GNUNET_FS_indexing_send_list ( struct GNUNET_MQ_Handle mq)

Transmit information about indexed files to mq.

mqmessage queue to send information to

Definition at line 358 of file gnunet-service-fs_indexing.c.

359 {
360  struct GNUNET_MQ_Envelope *env;
361  struct IndexInfoMessage *iim;
362  struct GNUNET_MessageHeader *iem;
363  size_t slen;
364  const char *fn;
365  struct IndexInfo *pos;
367  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
368  {
369  fn = pos->filename;
370  slen = strlen (fn) + 1;
371  if (slen + sizeof(struct IndexInfoMessage) >= GNUNET_MAX_MESSAGE_SIZE)
372  {
373  GNUNET_break (0);
374  break;
375  }
376  env =
378  iim->reserved = 0;
379  iim->file_id = pos->file_id;
380  GNUNET_memcpy (&iim[1], fn, slen);
381  GNUNET_MQ_send (mq, env);
382  }
384  GNUNET_MQ_send (mq, env);
385 }
struct GNUNET_MQ_Handle * mq
Definition: 003.c:5
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
static struct IndexInfo * indexed_files_head
Head of linked list of indexed files.
Largest supported message (to be precise, one byte more than the largest possible message,...
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
void GNUNET_MQ_send(struct GNUNET_MQ_Handle *mq, struct GNUNET_MQ_Envelope *ev)
Send a message with the given message queue.
Definition: mq.c:302
#define GNUNET_MQ_msg_extra(mvar, esize, type)
Allocate an envelope, with extra space allocated after the space needed by the message struct.
Definition: gnunet_mq_lib.h:56
#define GNUNET_MQ_msg(mvar, type)
Allocate a GNUNET_MQ_Envelope.
Definition: gnunet_mq_lib.h:71
Reply to client with an indexed file name.
Reply to client indicating end of list.
Header for all communications.
Message send by FS service in response to a request asking for a list of all indexed files.
Definition: fs.h:196
struct GNUNET_HashCode file_id
Hash of the indexed file.
Definition: fs.h:211
uint32_t reserved
Always zero.
Definition: fs.h:206
struct IndexInfo * next
This is a doubly linked list.
struct GNUNET_HashCode file_id
Hash of the contents of the file.

References env, IndexInfoMessage::file_id, IndexInfo::file_id, IndexInfo::filename, GNUNET_break, GNUNET_MAX_MESSAGE_SIZE, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END, GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY, GNUNET_MQ_msg, GNUNET_MQ_msg_extra, GNUNET_MQ_send(), indexed_files_head, mq, IndexInfo::next, and IndexInfoMessage::reserved.

Referenced by handle_client_index_list_get().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GNUNET_FS_indexing_do_unindex()

int GNUNET_FS_indexing_do_unindex ( const struct GNUNET_HashCode fid)

Remove a file from the index.

fididentifier of the file to remove
GNUNET_YES if the fid was found

Definition at line 395 of file gnunet-service-fs_indexing.c.

396 {
397  struct IndexInfo *pos;
399  for (pos = indexed_files_head; NULL != pos; pos = pos->next)
400  {
401  if (0 == memcmp (&pos->file_id, fid, sizeof(struct GNUNET_HashCode)))
402  {
404  GNUNET_break (
405  GNUNET_OK ==
407  GNUNET_free (pos);
408  write_index_list ();
409  return GNUNET_YES;
410  }
411  }
412  return GNUNET_NO;
413 }
static struct IndexInfo * indexed_files_tail
Tail of linked list of indexed files.
static void write_index_list()
Write the current index information list to disk.
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
enum GNUNET_GenericReturnValue GNUNET_CONTAINER_multihashmap_remove(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, const void *value)
Remove the given key-value pair from the map.
Definition: gnunet_common.h:98
#define GNUNET_free(ptr)
Wrapper around free.

References IndexInfo::file_id, GNUNET_break, GNUNET_CONTAINER_DLL_remove, GNUNET_CONTAINER_multihashmap_remove(), GNUNET_free, GNUNET_NO, GNUNET_OK, GNUNET_YES, ifm, indexed_files_head, indexed_files_tail, IndexInfo::next, and write_index_list().

Referenced by handle_client_unindex().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GNUNET_FS_add_to_index()

void GNUNET_FS_add_to_index ( const char *  filename,
const struct GNUNET_HashCode file_id 

Add the given file to the list of indexed files.

filenamename of the file
file_idhash identifier for filename

Definition at line 423 of file gnunet-service-fs_indexing.c.

425 {
426  struct IndexInfo *ii;
427  size_t slen;
430  if (NULL != ii)
431  {
432  GNUNET_log (
434  _ (
435  "Index request received for file `%s' is already indexed as `%s'. Permitting anyway.\n"),
436  filename,
437  ii->filename);
438  return;
439  }
441  "Adding file %s to index as %s\n",
442  filename,
443  GNUNET_h2s (file_id));
444  slen = strlen (filename) + 1;
445  ii = GNUNET_malloc (sizeof(struct IndexInfo) + slen);
446  ii->file_id = *file_id;
447  ii->filename = (const char *) &ii[1];
448  GNUNET_memcpy (&ii[1], filename, slen);
452  ifm,
453  &ii->file_id,
454  ii,
456  write_index_list ();
457 }
static char * filename
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
enum GNUNET_GenericReturnValue GNUNET_CONTAINER_multihashmap_put(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, void *value, enum GNUNET_CONTAINER_MultiHashMapOption opt)
Store a key-value pair in the map.
There must only be one value per key; storing a value should fail if a value under the same key alrea...
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_malloc(size)
Wrapper around malloc.

References _, IndexInfo::file_id, filename, IndexInfo::filename, GNUNET_assert, GNUNET_CONTAINER_DLL_insert, GNUNET_CONTAINER_multihashmap_get(), GNUNET_CONTAINER_multihashmap_put(), GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_INFO, GNUNET_h2s(), GNUNET_log, GNUNET_malloc, GNUNET_memcpy, GNUNET_OK, ifm, indexed_files_head, indexed_files_tail, and write_index_list().

Referenced by signal_index_ok().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GNUNET_FS_indexing_init()

int GNUNET_FS_indexing_init ( const struct GNUNET_CONFIGURATION_Handle c,
struct GNUNET_DATASTORE_Handle d 

Initialize the indexing submodule.

cconfiguration to use
ddatastore to use
GNUNET_OK on success

Definition at line 485 of file gnunet-service-fs_indexing.c.

487 {
488  cfg = c;
489  dsh = d;
491  read_index_list ();
492  return GNUNET_OK;
493 }
static const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
static void read_index_list()
Read index information from disk.
struct GNUNET_CONTAINER_MultiHashMap * GNUNET_CONTAINER_multihashmap_create(unsigned int len, int do_not_copy_keys)
Create a multi hash map.

References cfg, dsh, GNUNET_CONTAINER_multihashmap_create(), GNUNET_OK, GNUNET_YES, ifm, and read_index_list().

Referenced by run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ GNUNET_FS_indexing_done()

void GNUNET_FS_indexing_done ( void  )

Shutdown the module.

Definition at line 464 of file gnunet-service-fs_indexing.c.

465 {
466  struct IndexInfo *pos;
468  while (NULL != (pos = indexed_files_head))
469  {
471  if (pos->fhc != NULL)
473  GNUNET_break (
474  GNUNET_OK ==
476  GNUNET_free (pos);
477  }
479  ifm = NULL;
480  cfg = NULL;
481 }
void GNUNET_CONTAINER_multihashmap_destroy(struct GNUNET_CONTAINER_MultiHashMap *map)
Destroy a hash map.
void GNUNET_CRYPTO_hash_file_cancel(struct GNUNET_CRYPTO_FileHashContext *fhc)
Cancel a file hashing operation.
struct GNUNET_CRYPTO_FileHashContext * fhc
Context for hashing of the file.

References cfg, IndexInfo::fhc, IndexInfo::file_id, GNUNET_break, GNUNET_CONTAINER_DLL_remove, GNUNET_CONTAINER_multihashmap_destroy(), GNUNET_CONTAINER_multihashmap_remove(), GNUNET_CRYPTO_hash_file_cancel(), GNUNET_free, GNUNET_OK, ifm, indexed_files_head, and indexed_files_tail.

Referenced by shutdown_task().

Here is the call graph for this function:
Here is the caller graph for this function: