GNUnet 0.21.0
gnunet-service-fs_indexing.c
Go to the documentation of this file.
1/*
2 This file is part of GNUnet.
3 Copyright (C) 2009, 2010 GNUnet e.V.
4
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
9
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
14
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
26#include "platform.h"
27#include <float.h>
28#include "gnunet_core_service.h"
30#include "gnunet_protocols.h"
31#include "gnunet_signatures.h"
32#include "gnunet_util_lib.h"
33#include "gnunet-service-fs.h"
35#include "fs.h"
36
42{
46 struct IndexInfo *next;
47
51 struct IndexInfo *prev;
52
57 const char *filename;
58
63 struct GNUNET_SERVER_TransmitContext *tc;
64
69
74};
75
76
82
87
94
98static const struct GNUNET_CONFIGURATION_Handle *cfg;
99
105
106
110static void
112{
113 struct GNUNET_BIO_WriteHandle *wh;
114 char *fn;
115 struct IndexInfo *pos;
116
117 if (GNUNET_OK !=
118 GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn))
119 {
121 "fs",
122 "INDEXDB");
123 return;
124 }
126 if (NULL == wh)
127 {
129 _ ("Could not open `%s'.\n"),
130 fn);
131 GNUNET_free (fn);
132 return;
133 }
134 for (pos = indexed_files_head; NULL != pos; pos = pos->next)
135 if ((GNUNET_OK != GNUNET_BIO_write (wh,
136 "fs-indexing-file-id",
137 &pos->file_id,
138 sizeof(struct GNUNET_HashCode))) ||
140 "fs-indexing-filename",
141 pos->filename)))
142 break;
143 if (GNUNET_OK != GNUNET_BIO_write_close (wh, NULL))
144 {
146 _ ("Error writing `%s'.\n"),
147 fn);
148 GNUNET_free (fn);
149 return;
150 }
151 GNUNET_free (fn);
152}
153
154
158static void
160{
161 struct GNUNET_BIO_ReadHandle *rh;
162 char *fn;
163 struct IndexInfo *pos;
164 char *fname;
165 struct GNUNET_HashCode hc;
166 size_t slen;
167 char *emsg;
168
169 if (GNUNET_OK !=
170 GNUNET_CONFIGURATION_get_value_filename (cfg, "FS", "INDEXDB", &fn))
171 {
173 "fs",
174 "INDEXDB");
175 return;
176 }
178 {
179 /* no index info yet */
180 GNUNET_free (fn);
181 return;
182 }
184 if (NULL == rh)
185 {
187 _ ("Could not open `%s'.\n"),
188 fn);
189 GNUNET_free (fn);
190 return;
191 }
192 while (
194 "Hash of indexed file",
195 &hc,
196 sizeof(struct GNUNET_HashCode))) &&
197 (GNUNET_OK ==
198 GNUNET_BIO_read_string (rh, "Name of indexed file", &fname, 1024 * 16)) &&
199 (fname != NULL))
200 {
201 slen = strlen (fname) + 1;
202 pos = GNUNET_malloc (sizeof(struct IndexInfo) + slen);
203 pos->file_id = hc;
204 pos->filename = (const char *) &pos[1];
205 GNUNET_memcpy (&pos[1], fname, slen);
207 ifm,
208 &pos->file_id,
209 pos,
211 {
212 GNUNET_free (pos);
213 }
214 else
215 {
217 }
218 GNUNET_free (fname);
219 }
220 if (GNUNET_OK != GNUNET_BIO_read_close (rh, &emsg))
221 GNUNET_free (emsg);
222 GNUNET_free (fn);
223}
224
225
235static void
236remove_cont (void *cls,
237 int success,
239 const char *msg)
240{
241 if (GNUNET_OK != success)
243 _ ("Failed to delete bogus block: %s\n"),
244 msg);
245}
246
247
248int
250 uint32_t size,
251 const void *data,
253 uint32_t priority,
254 uint32_t anonymity,
255 uint32_t replication,
257 uint64_t uid,
259 void *cont_cls)
260{
261 const struct OnDemandBlock *odb;
262 struct GNUNET_HashCode nkey;
265 struct GNUNET_HashCode query;
266 ssize_t nsize;
267 char ndata[DBLOCK_SIZE];
268 char edata[DBLOCK_SIZE];
269 const char *fn;
271 uint64_t off;
272 struct IndexInfo *ii;
273
274 if (size != sizeof(struct OnDemandBlock))
275 {
276 GNUNET_break (0);
277 GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
278 return GNUNET_SYSERR;
279 }
280 odb = (const struct OnDemandBlock *) data;
281 off = GNUNET_ntohll (odb->offset);
283 if (NULL == ii)
284 {
285 GNUNET_break (0);
287 "Failed to find index %s\n",
288 GNUNET_h2s (&odb->file_id));
289 return GNUNET_SYSERR;
290 }
291 fn = ii->filename;
292 if ((NULL == fn) || (0 != access (fn, R_OK)))
293 {
295 GSF_stats,
296 gettext_noop ("# index blocks removed: original file inaccessible"),
297 1,
298 GNUNET_YES);
299 GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
300 return GNUNET_SYSERR;
301 }
302 if ((NULL == (fh = GNUNET_DISK_file_open (fn,
306 (-1 == (nsize = GNUNET_DISK_file_read (fh, ndata, sizeof(ndata)))))
307 {
309 _ (
310 "Could not access indexed file `%s' (%s) at offset %llu: %s\n"),
311 GNUNET_h2s (&odb->file_id),
312 fn,
313 (unsigned long long) off,
314 (fn == NULL) ? _ ("not indexed") : strerror (errno));
315 if (fh != NULL)
317 GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
318 return GNUNET_SYSERR;
319 }
321 GNUNET_CRYPTO_hash (ndata, nsize, &nkey);
322 GNUNET_CRYPTO_hash_to_aes_key (&nkey, &skey, &iv);
323 GNUNET_CRYPTO_symmetric_encrypt (ndata, nsize, &skey, &iv, edata);
324 GNUNET_CRYPTO_hash (edata, nsize, &query);
325 if (0 != memcmp (&query, key, sizeof(struct GNUNET_HashCode)))
326 {
328 _ ("Indexed file `%s' changed at offset %llu\n"),
329 fn,
330 (unsigned long long) off);
331 GNUNET_DATASTORE_remove (dsh, key, size, data, -1, -1, &remove_cont, NULL);
332 return GNUNET_SYSERR;
333 }
335 "On-demand encoded block for query `%s'\n",
336 GNUNET_h2s (key));
337 cont (cont_cls,
338 key,
339 nsize,
340 edata,
342 priority,
343 anonymity,
346 uid);
347 return GNUNET_OK;
348}
349
350
356void
358{
359 struct GNUNET_MQ_Envelope *env;
360 struct IndexInfoMessage *iim;
361 struct GNUNET_MessageHeader *iem;
362 size_t slen;
363 const char *fn;
364 struct IndexInfo *pos;
365
366 for (pos = indexed_files_head; NULL != pos; pos = pos->next)
367 {
368 fn = pos->filename;
369 slen = strlen (fn) + 1;
370 if (slen + sizeof(struct IndexInfoMessage) >= GNUNET_MAX_MESSAGE_SIZE)
371 {
372 GNUNET_break (0);
373 break;
374 }
375 env =
377 iim->reserved = 0;
378 iim->file_id = pos->file_id;
379 GNUNET_memcpy (&iim[1], fn, slen);
381 }
384}
385
386
393int
395{
396 struct IndexInfo *pos;
397
398 for (pos = indexed_files_head; NULL != pos; pos = pos->next)
399 {
400 if (0 == memcmp (&pos->file_id, fid, sizeof(struct GNUNET_HashCode)))
401 {
404 GNUNET_OK ==
406 GNUNET_free (pos);
408 return GNUNET_YES;
409 }
410 }
411 return GNUNET_NO;
412}
413
414
421void
423 const struct GNUNET_HashCode *file_id)
424{
425 struct IndexInfo *ii;
426 size_t slen;
427
429 if (NULL != ii)
430 {
431 GNUNET_log (
433 _ (
434 "Index request received for file `%s' is already indexed as `%s'. Permitting anyway.\n"),
435 filename,
436 ii->filename);
437 return;
438 }
440 "Adding file %s to index as %s\n",
441 filename,
443 slen = strlen (filename) + 1;
444 ii = GNUNET_malloc (sizeof(struct IndexInfo) + slen);
445 ii->file_id = *file_id;
446 ii->filename = (const char *) &ii[1];
447 GNUNET_memcpy (&ii[1], filename, slen);
451 ifm,
452 &ii->file_id,
453 ii,
456}
457
458
462void
464{
465 struct IndexInfo *pos;
466
467 while (NULL != (pos = indexed_files_head))
468 {
470 if (pos->fhc != NULL)
473 GNUNET_OK ==
475 GNUNET_free (pos);
476 }
478 ifm = NULL;
479 cfg = NULL;
480}
481
482
483int
486{
487 cfg = c;
488 dsh = d;
491 return GNUNET_OK;
492}
493
494
495/* end of gnunet-service-fs_indexing.c */
struct GNUNET_MQ_Handle * mq
Definition: 003.c:5
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
static mp_limb_t d[(((256)+GMP_NUMB_BITS - 1)/GMP_NUMB_BITS)]
definitions for the entire fs module
#define DBLOCK_SIZE
Size of the individual blocks used for file-sharing.
Definition: fs.h:41
#define gettext_noop(String)
Definition: gettext.h:70
static unsigned int replication
Desired replication level.
static char * data
The data to insert into the dht.
struct GNUNET_HashCode key
The key used in the DHT.
static struct GNUNET_TIME_Relative expiration
User supplied expiration value.
static unsigned int anonymity
static char * filename
static uint32_t type
Type string converted to DNS type value.
static struct GNUNET_TIME_Absolute min_expiration
Minimum time that content should have to not be discarded instantly (time stamp of any content that w...
struct GNUNET_STATISTICS_Handle * GSF_stats
Handle for reporting statistics.
shared data structures of gnunet-service-fs.c
static struct GNUNET_CONTAINER_MultiHashMap * ifm
Maps hash over content of indexed files to the respective 'struct IndexInfo'.
int GNUNET_FS_indexing_do_unindex(const struct GNUNET_HashCode *fid)
Remove a file from the index.
void GNUNET_FS_add_to_index(const char *filename, const struct GNUNET_HashCode *file_id)
Add the given file to the list of indexed files.
static void remove_cont(void *cls, int success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called from datastore's remove function.
static struct IndexInfo * indexed_files_tail
Tail of linked list of indexed files.
int GNUNET_FS_indexing_init(const struct GNUNET_CONFIGURATION_Handle *c, struct GNUNET_DATASTORE_Handle *d)
Initialize the indexing submodule.
static const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
static struct IndexInfo * indexed_files_head
Head of linked list of indexed files.
void GNUNET_FS_indexing_send_list(struct GNUNET_MQ_Handle *mq)
Transmit information about indexed files to mq.
static void read_index_list()
Read index information from disk.
void GNUNET_FS_indexing_done()
Shutdown the module.
static void write_index_list()
Write the current index information list to disk.
int GNUNET_FS_handle_on_demand_block(const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid, GNUNET_DATASTORE_DatumProcessor cont, void *cont_cls)
We've received an on-demand encoded block from the datastore.
static struct GNUNET_DATASTORE_Handle * dsh
Datastore handle.
indexing for the file-sharing service
static struct GNUNET_DISK_FileHandle * fh
File handle to STDIN, for reading restart/quit commands.
Core service; the main API for encrypted P2P communications.
Constants for network protocols.
#define GNUNET_MAX_MESSAGE_SIZE
Largest supported message (to be precise, one byte more than the largest possible message,...
enum GNUNET_GenericReturnValue GNUNET_BIO_read(struct GNUNET_BIO_ReadHandle *h, const char *what, void *result, size_t len)
Read some contents into a buffer.
Definition: bio.c:291
enum GNUNET_GenericReturnValue GNUNET_BIO_write_close(struct GNUNET_BIO_WriteHandle *h, char **emsg)
Close an IO handle.
Definition: bio.c:556
struct GNUNET_BIO_WriteHandle * GNUNET_BIO_write_open_file(const char *fn)
Open a file for writing.
Definition: bio.c:508
enum GNUNET_GenericReturnValue GNUNET_BIO_write(struct GNUNET_BIO_WriteHandle *h, const char *what, const void *buffer, size_t n)
Write a buffer to a handle.
Definition: bio.c:752
enum GNUNET_GenericReturnValue GNUNET_BIO_read_string(struct GNUNET_BIO_ReadHandle *h, const char *what, char **result, size_t max_length)
Read 0-terminated string.
Definition: bio.c:330
enum GNUNET_GenericReturnValue GNUNET_BIO_write_string(struct GNUNET_BIO_WriteHandle *h, const char *what, const char *s)
Write a 0-terminated string.
Definition: bio.c:789
enum GNUNET_GenericReturnValue GNUNET_BIO_read_close(struct GNUNET_BIO_ReadHandle *h, char **emsg)
Close an open handle.
Definition: bio.c:162
struct GNUNET_BIO_ReadHandle * GNUNET_BIO_read_open_file(const char *fn)
Open a file for reading.
Definition: bio.c:114
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_filename(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be the name of a file or directory.
ssize_t GNUNET_CRYPTO_symmetric_encrypt(const void *block, size_t size, const struct GNUNET_CRYPTO_SymmetricSessionKey *sessionkey, const struct GNUNET_CRYPTO_SymmetricInitializationVector *iv, void *result)
Encrypt a block using a symmetric sessionkey.
void(* GNUNET_DATASTORE_DatumProcessor)(void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Process a datum that was stored in the datastore.
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_remove(struct GNUNET_DATASTORE_Handle *h, const struct GNUNET_HashCode *key, size_t size, const void *data, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Explicitly remove some content from the database.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1237
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition: disk.c:482
off_t GNUNET_DISK_file_seek(const struct GNUNET_DISK_FileHandle *h, off_t offset, enum GNUNET_DISK_Seek whence)
Move the read/write pointer in a file.
Definition: disk.c:205
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1308
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:622
@ GNUNET_DISK_OPEN_READ
Open the file for reading.
@ GNUNET_DISK_PERM_NONE
Nobody is allowed to do anything to the file.
@ GNUNET_DISK_SEEK_SET
Seek an absolute position (from the start of the file).
#define GNUNET_CONTAINER_DLL_remove(head, tail, element)
Remove an element from a DLL.
#define GNUNET_CONTAINER_DLL_insert(head, tail, element)
Insert an element at the head of a DLL.
void GNUNET_CRYPTO_hash(const void *block, size_t size, struct GNUNET_HashCode *ret)
Compute hash of a given block.
Definition: crypto_hash.c:41
void GNUNET_CRYPTO_hash_to_aes_key(const struct GNUNET_HashCode *hc, struct GNUNET_CRYPTO_SymmetricSessionKey *skey, struct GNUNET_CRYPTO_SymmetricInitializationVector *iv)
Convert a hashcode into a key.
Definition: crypto_hash.c:152
void * GNUNET_CONTAINER_multihashmap_get(const struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key)
Given a key find a value in the map matching the key.
enum GNUNET_GenericReturnValue GNUNET_CONTAINER_multihashmap_remove(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, const void *value)
Remove the given key-value pair from the map.
enum GNUNET_GenericReturnValue GNUNET_CONTAINER_multihashmap_put(struct GNUNET_CONTAINER_MultiHashMap *map, const struct GNUNET_HashCode *key, void *value, enum GNUNET_CONTAINER_MultiHashMapOption opt)
Store a key-value pair in the map.
void GNUNET_CONTAINER_multihashmap_destroy(struct GNUNET_CONTAINER_MultiHashMap *map)
Destroy a hash map.
struct GNUNET_CONTAINER_MultiHashMap * GNUNET_CONTAINER_multihashmap_create(unsigned int len, int do_not_copy_keys)
Create a multi hash map.
@ GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY
There must only be one value per key; storing a value should fail if a value under the same key alrea...
#define GNUNET_log(kind,...)
uint64_t GNUNET_ntohll(uint64_t n)
Convert unsigned 64-bit integer to host byte order.
Definition: common_endian.c:54
void GNUNET_CRYPTO_hash_file_cancel(struct GNUNET_CRYPTO_FileHashContext *fhc)
Cancel a file hashing operation.
#define GNUNET_memcpy(dst, src, n)
Call memcpy() but check for n being 0 first.
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_BULK
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
#define GNUNET_malloc(size)
Wrapper around malloc.
#define GNUNET_free(ptr)
Wrapper around free.
void GNUNET_MQ_send(struct GNUNET_MQ_Handle *mq, struct GNUNET_MQ_Envelope *ev)
Send a message with the given message queue.
Definition: mq.c:304
#define GNUNET_MQ_msg_extra(mvar, esize, type)
Allocate an envelope, with extra space allocated after the space needed by the message struct.
Definition: gnunet_mq_lib.h:63
#define GNUNET_MQ_msg(mvar, type)
Allocate a GNUNET_MQ_Envelope.
Definition: gnunet_mq_lib.h:78
#define GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_ENTRY
Reply to client with an indexed file name.
#define GNUNET_MESSAGE_TYPE_FS_INDEX_LIST_END
Reply to client indicating end of list.
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
static unsigned int size
Size of the "table".
Definition: peer.c:68
#define _(String)
GNU gettext support macro.
Definition: platform.h:178
GNUNET_BLOCK_Type
WARNING: This header is generated! In order to add DHT block types, you must register them in GANA,...
@ GNUNET_BLOCK_TYPE_FS_DBLOCK
Data block (leaf) in the CHK tree.
Handle for buffered reading.
Definition: bio.c:69
Handle for buffered writing.
Definition: bio.c:466
Internal representation of the hash map.
Context used when hashing a file.
Handle to the datastore service.
Handle used to access files (and pipes).
A 512-bit hashcode.
Handle to a message queue.
Definition: mq.c:87
Header for all communications.
Time for absolute times used by GNUnet, in microseconds.
Message send by FS service in response to a request asking for a list of all indexed files.
Definition: fs.h:197
struct GNUNET_HashCode file_id
Hash of the indexed file.
Definition: fs.h:212
uint32_t reserved
Always zero.
Definition: fs.h:207
In-memory information about indexed files (also available on-disk).
const char * filename
Name of the indexed file.
struct IndexInfo * next
This is a doubly linked list.
struct IndexInfo * prev
This is a doubly linked list.
struct GNUNET_SERVER_TransmitContext * tc
Context for transmitting confirmation to client, NULL if we've done this already.
struct GNUNET_HashCode file_id
Hash of the contents of the file.
struct GNUNET_CRYPTO_FileHashContext * fhc
Context for hashing of the file.
index block (indexing a DBlock that can be obtained directly from reading the plaintext file)
Definition: block_fs.h:86
uint64_t offset
At which offset should we be able to find this on-demand encoded block? (in NBO)
Definition: block_fs.h:98
struct GNUNET_HashCode file_id
Hash code of the entire content of the file that was indexed (used to uniquely identify the plaintext...
Definition: block_fs.h:92