GNUnet  0.11.x
datacache.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2004, 2005, 2006, 2007, 2009, 2010, 2015 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
25 #include "platform.h"
26 #include "gnunet_util_lib.h"
27 #include "gnunet_datacache_lib.h"
30 
31 
32 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache", __VA_ARGS__)
33 
34 #define LOG_STRERROR_FILE(kind, op, fn) \
35  GNUNET_log_from_strerror_file (kind, "datacache", op, fn)
36 
41 {
46 
51 
56 
60  char *section;
61 
67 
71  char *short_name;
72 
76  char *lib_name;
77 
81  char *bloom_name;
82 
87 
91  unsigned long long utilization;
92 };
93 
94 
103 static void
104 env_delete_notify (void *cls, const struct GNUNET_HashCode *key, size_t size)
105 {
106  struct GNUNET_DATACACHE_Handle *h = cls;
107 
109  "Content under key `%s' discarded\n",
110  GNUNET_h2s (key));
111  GNUNET_assert (h->utilization >= size);
112  h->utilization -= size;
114  GNUNET_STATISTICS_update (h->stats,
115  gettext_noop ("# bytes stored"),
116  -(long long) size,
117  GNUNET_NO);
118  GNUNET_STATISTICS_update (h->stats,
119  gettext_noop ("# items stored"),
120  -1,
121  GNUNET_NO);
122 }
123 
124 
134  const char *section)
135 {
136  unsigned int bf_size;
137  unsigned long long quota;
139  char *libname;
140  char *name;
141  const struct GNUNET_OS_ProjectData *pd;
142 
143  if (GNUNET_OK !=
144  GNUNET_CONFIGURATION_get_value_size (cfg, section, "QUOTA", &quota))
145  {
147  return NULL;
148  }
149  if (GNUNET_OK !=
150  GNUNET_CONFIGURATION_get_value_string (cfg, section, "DATABASE", &name))
151  {
152  GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, section, "DATABASE");
153  return NULL;
154  }
155  bf_size = quota / 32; /* 8 bit per entry, 1 bit per 32 kb in DB */
156 
158 
159  if (GNUNET_YES !=
160  GNUNET_CONFIGURATION_get_value_yesno (cfg, section, "DISABLE_BF"))
161  {
162  if (GNUNET_YES !=
163  GNUNET_CONFIGURATION_get_value_yesno (cfg, section, "DISABLE_BF_RC"))
164  {
165  ret->bloom_name = GNUNET_DISK_mktemp ("gnunet-datacachebloom");
166  }
167  if (NULL != ret->bloom_name)
168  {
170  ret->bloom_name,
171  quota / 1024, /* 8 bit per entry in DB, expect 1k entries */
172  5);
173  }
174  if (NULL == ret->filter)
175  {
176  ret->filter =
178  bf_size,
179  5); /* approx. 3% false positives at max use */
180  }
181  }
182  ret->stats = GNUNET_STATISTICS_create ("datacache", cfg);
183  ret->section = GNUNET_strdup (section);
184  ret->env.cfg = cfg;
185  ret->env.delete_notify = &env_delete_notify;
186  ret->env.section = ret->section;
187  ret->env.cls = ret;
188  ret->env.delete_notify = &env_delete_notify;
189  ret->env.quota = quota;
190  LOG (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datacache plugin\n"), name);
191  GNUNET_asprintf (&libname, "libgnunet_plugin_datacache_%s", name);
192  ret->short_name = name;
193  ret->lib_name = libname;
194  /* Load the plugin within GNUnet's default context */
197  ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
198  GNUNET_OS_init(pd);
199  if (NULL == ret->api)
200  {
201  /* Try to load the plugin within the application's context
202  This normally happens when the application is not GNUnet itself but a
203  third party; inside GNUnet this is effectively a double failure. */
204  ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
205  if (NULL == ret->api)
206  {
208  _ ("Failed to load datacache plugin for `%s'\n"),
209  name);
211  return NULL;
212  }
213  }
214  return ret;
215 }
216 
217 
223 void
225 {
226  if (NULL != h->filter)
228  if (NULL != h->api)
229  GNUNET_break (NULL == GNUNET_PLUGIN_unload (h->lib_name, h->api));
230  GNUNET_free (h->lib_name);
231  GNUNET_free (h->short_name);
232  GNUNET_free (h->section);
233  if (NULL != h->bloom_name)
234  {
235  if (0 != unlink (h->bloom_name))
237  "datacache",
238  "unlink",
239  h->bloom_name);
240  GNUNET_free (h->bloom_name);
241  }
243  GNUNET_free (h);
244 }
245 
246 
261 int
263  const struct GNUNET_HashCode *key,
264  uint32_t xor_distance,
265  size_t data_size,
266  const char *data,
267  enum GNUNET_BLOCK_Type type,
268  struct GNUNET_TIME_Absolute discard_time,
269  unsigned int path_info_len,
270  const struct GNUNET_PeerIdentity *path_info)
271 {
272  ssize_t used;
273 
274  used = h->api->put (h->api->cls,
275  key,
276  xor_distance,
277  data_size,
278  data,
279  type,
280  discard_time,
281  path_info_len,
282  path_info);
283  if (-1 == used)
284  {
285  GNUNET_break (0);
286  return GNUNET_SYSERR;
287  }
288  if (0 == used)
289  {
290  /* duplicate */
291  return GNUNET_NO;
292  }
294  "Stored data under key `%s' in cache\n",
295  GNUNET_h2s (key));
296  if (NULL != h->filter)
298  GNUNET_STATISTICS_update (h->stats,
299  gettext_noop ("# bytes stored"),
300  used,
301  GNUNET_NO);
302  GNUNET_STATISTICS_update (h->stats,
303  gettext_noop ("# items stored"),
304  1,
305  GNUNET_NO);
306  while (h->utilization + used > h->env.quota)
307  GNUNET_assert (GNUNET_OK == h->api->del (h->api->cls));
308  h->utilization += used;
309  return GNUNET_OK;
310 }
311 
312 
324 unsigned int
326  const struct GNUNET_HashCode *key,
327  enum GNUNET_BLOCK_Type type,
329  void *iter_cls)
330 {
331  GNUNET_STATISTICS_update (h->stats,
332  gettext_noop ("# requests received"),
333  1,
334  GNUNET_NO);
336  "Processing request for key `%s'\n",
337  GNUNET_h2s (key));
338  if ((NULL != h->filter) &&
340  {
341  GNUNET_STATISTICS_update (h->stats,
342  gettext_noop (
343  "# requests filtered by bloom filter"),
344  1,
345  GNUNET_NO);
347  "Bloomfilter filters request for key `%s'\n",
348  GNUNET_h2s (key));
349  return 0; /* can not be present */
350  }
351  return h->api->get (h->api->cls, key, type, iter, iter_cls);
352 }
353 
354 
363 unsigned int
366  void *iter_cls)
367 {
368  GNUNET_STATISTICS_update (h->stats,
369  gettext_noop (
370  "# requests for random value received"),
371  1,
372  GNUNET_NO);
373  LOG (GNUNET_ERROR_TYPE_DEBUG, "Processing request for random value\n");
374  return h->api->get_random (h->api->cls, iter, iter_cls);
375 }
376 
377 
391 unsigned int
393  const struct GNUNET_HashCode *key,
394  unsigned int num_results,
396  void *iter_cls)
397 {
398  GNUNET_STATISTICS_update (h->stats,
399  gettext_noop (
400  "# proximity search requests received"),
401  1,
402  GNUNET_NO);
404  "Processing proximity search at `%s'\n",
405  GNUNET_h2s (key));
406  return h->api->get_closest (h->api->cls, key, num_results, iter, iter_cls);
407 }
408 
409 
410 /* end of datacache.c */
static void env_delete_notify(void *cls, const struct GNUNET_HashCode *key, size_t size)
Function called by plugins to notify the datacache about content deletions.
Definition: datacache.c:104
#define LOG(kind,...)
Definition: datacache.c:32
#define gettext_noop(String)
Definition: gettext.h:69
static const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration we are using.
Definition: gnunet-abd.c:36
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
struct GNUNET_HashCode key
The key used in the DHT.
uint32_t data
The data value.
static unsigned long long quota
How much space are we allowed to use?
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_load(const char *filename, size_t size, unsigned int k)
Load a Bloom filter from a file.
void GNUNET_CONTAINER_bloomfilter_add(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Add an element to the filter.
int GNUNET_CONTAINER_bloomfilter_test(const struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Test if an element is in the filter.
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_init(const char *data, size_t size, unsigned int k)
Create a Bloom filter from raw bits.
void GNUNET_CONTAINER_bloomfilter_remove(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Remove an element from the filter.
void GNUNET_CONTAINER_bloomfilter_free(struct GNUNET_CONTAINER_BloomFilter *bf)
Free the space associated with a filter in memory, flush to drive if needed (do not free the space on...
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_size(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *size)
Get a configuration value that should be a size in bytes.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
unsigned int GNUNET_DATACACHE_get_closest(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, unsigned int num_results, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results that are "close" to a particular key in the datacache.
Definition: datacache.c:392
unsigned int GNUNET_DATACACHE_get(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results for a particular key in the datacache.
Definition: datacache.c:325
int GNUNET_DATACACHE_put(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, uint32_t xor_distance, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, struct GNUNET_TIME_Absolute discard_time, unsigned int path_info_len, const struct GNUNET_PeerIdentity *path_info)
Store an item in the datastore.
Definition: datacache.c:262
unsigned int GNUNET_DATACACHE_get_random(struct GNUNET_DATACACHE_Handle *h, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Obtain a random element from the datacache.
Definition: datacache.c:364
struct GNUNET_DATACACHE_Handle * GNUNET_DATACACHE_create(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section)
Create a data cache.
Definition: datacache.c:133
void GNUNET_DATACACHE_destroy(struct GNUNET_DATACACHE_Handle *h)
Destroy a data cache (and free associated resources).
Definition: datacache.c:224
int(* GNUNET_DATACACHE_Iterator)(void *cls, const struct GNUNET_HashCode *key, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, struct GNUNET_TIME_Absolute exp, unsigned int path_info_len, const struct GNUNET_PeerIdentity *path_info)
An iterator over a set of items stored in the datacache.
char * GNUNET_DISK_mktemp(const char *t)
Create an (empty) temporary file on disk.
Definition: disk.c:381
#define GNUNET_log_from_strerror_file(level, component, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_default(void)
Return default project data used by 'libgnunetutil' for GNUnet.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_get(void)
void GNUNET_OS_init(const struct GNUNET_OS_ProjectData *pd)
Setup OS subsystem with project data.
void * GNUNET_PLUGIN_unload(const char *library_name, void *arg)
Unload plugin (runs the "done" callback and returns whatever "done" returned).
Definition: plugin.c:269
void * GNUNET_PLUGIN_load(const char *library_name, void *arg)
Setup plugin (runs the "init" callback and returns whatever "init" returned).
Definition: plugin.c:217
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
const char * name
Internal state of the datacache library.
Definition: datacache.c:41
char * bloom_name
Name for the bloom filter file.
Definition: datacache.c:81
unsigned long long utilization
How much space is in use right now?
Definition: datacache.c:91
struct GNUNET_DATACACHE_PluginEnvironment env
Environment provided to our plugin.
Definition: datacache.c:86
char * section
Configuration section to use.
Definition: datacache.c:60
char * short_name
Short name for the plugin (e.g.
Definition: datacache.c:71
const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition: datacache.c:50
struct GNUNET_STATISTICS_Handle * stats
Opaque handle for the statistics service.
Definition: datacache.c:55
struct GNUNET_CONTAINER_BloomFilter * filter
Bloomfilter to quickly tell if we don't have the content.
Definition: datacache.c:45
char * lib_name
Name of the library (e.g.
Definition: datacache.c:76
struct GNUNET_DATACACHE_PluginFunctions * api
API of the transport as returned by the plugin's initialization function.
Definition: datacache.c:66
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
void * cls
Closure to use for callbacks.
struct returned by the initialization function of the plugin
A 512-bit hashcode.
Project-specific data used to help the OS subsystem find installation paths.
const char * libname
Name of a library that is installed in the "lib/" directory of the project, such as "libgnunetutil".
The identity of the host (wraps the signing key of the peer).
Handle for the service.
Time for absolute times used by GNUnet, in microseconds.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model