GNUnet  0.11.x
datacache.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2004, 2005, 2006, 2007, 2009, 2010, 2015 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
25 #include "platform.h"
26 #include "gnunet_util_lib.h"
27 #include "gnunet_datacache_lib.h"
30 
31 
32 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache", __VA_ARGS__)
33 
34 #define LOG_STRERROR_FILE(kind, op, fn) \
35  GNUNET_log_from_strerror_file (kind, "datacache", op, fn)
36 
41 {
46 
51 
56 
60  char *section;
61 
67 
71  char *short_name;
72 
76  char *lib_name;
77 
81  char *bloom_name;
82 
87 
91  unsigned long long utilization;
92 };
93 
94 
103 static void
105  const struct GNUNET_HashCode *key,
106  size_t size)
107 {
108  struct GNUNET_DATACACHE_Handle *h = cls;
109 
111  "Content under key `%s' discarded\n",
112  GNUNET_h2s (key));
113  GNUNET_assert (h->utilization >= size);
114  h->utilization -= size;
116  GNUNET_STATISTICS_update (h->stats,
117  "# bytes stored",
118  -(long long) size,
119  GNUNET_NO);
120  GNUNET_STATISTICS_update (h->stats,
121  "# items stored",
122  -1,
123  GNUNET_NO);
124 }
125 
126 
129  const char *section)
130 {
131  unsigned int bf_size;
132  unsigned long long quota;
134  char *libname;
135  char *name;
136  const struct GNUNET_OS_ProjectData *pd;
137 
138  if (GNUNET_OK !=
140  section,
141  "QUOTA",
142  &quota))
143  {
145  section,
146  "QUOTA");
147  return NULL;
148  }
149  if (GNUNET_OK !=
151  section,
152  "DATABASE",
153  &name))
154  {
156  section,
157  "DATABASE");
158  return NULL;
159  }
160  bf_size = quota / 32; /* 8 bit per entry, 1 bit per 32 kb in DB */
161 
163 
164  if (GNUNET_YES !=
166  section,
167  "DISABLE_BF"))
168  {
169  if (GNUNET_YES !=
171  section,
172  "DISABLE_BF_RC"))
173  {
174  ret->bloom_name = GNUNET_DISK_mktemp ("gnunet-datacachebloom");
175  }
176  if (NULL != ret->bloom_name)
177  {
179  ret->bloom_name,
180  quota / 1024, /* 8 bit per entry in DB, expect 1k entries */
181  5);
182  }
183  if (NULL == ret->filter)
184  {
185  ret->filter =
187  bf_size,
188  5); /* approx. 3% false positives at max use */
189  }
190  }
191  ret->stats = GNUNET_STATISTICS_create ("datacache",
192  cfg);
193  ret->section = GNUNET_strdup (section);
194  ret->env.cfg = cfg;
195  ret->env.delete_notify = &env_delete_notify;
196  ret->env.section = ret->section;
197  ret->env.cls = ret;
198  ret->env.delete_notify = &env_delete_notify;
199  ret->env.quota = quota;
201  "Loading `%s' datacache plugin\n",
202  name);
204  "libgnunet_plugin_datacache_%s",
205  name);
206  ret->short_name = name;
207  ret->lib_name = libname;
208  /* Load the plugin within GNUnet's default context */
212  &ret->env);
213  GNUNET_OS_init (pd);
214  if (NULL == ret->api)
215  {
216  /* Try to load the plugin within the application's context
217  This normally happens when the application is not GNUnet itself but a
218  third party; inside GNUnet this is effectively a double failure. */
220  &ret->env);
221  if (NULL == ret->api)
222  {
224  "Failed to load datacache plugin for `%s'\n",
225  name);
227  return NULL;
228  }
229  }
230  return ret;
231 }
232 
233 
234 void
236 {
237  if (NULL != h->filter)
239  if (NULL != h->api)
240  GNUNET_break (NULL ==
241  GNUNET_PLUGIN_unload (h->lib_name,
242  h->api));
243  GNUNET_free (h->lib_name);
244  GNUNET_free (h->short_name);
245  GNUNET_free (h->section);
246  if (NULL != h->bloom_name)
247  {
248  if (0 != unlink (h->bloom_name))
250  "datacache",
251  "unlink",
252  h->bloom_name);
253  GNUNET_free (h->bloom_name);
254  }
256  GNUNET_free (h);
257 }
258 
259 
262  const struct GNUNET_HashCode *key,
263  uint32_t xor_distance,
264  size_t data_size,
265  const char *data,
266  enum GNUNET_BLOCK_Type type,
267  struct GNUNET_TIME_Absolute discard_time,
268  unsigned int path_info_len,
269  const struct GNUNET_DHT_PathElement *path_info)
270 {
271  ssize_t used;
272 
273  used = h->api->put (h->api->cls,
274  key,
275  xor_distance,
276  data_size,
277  data,
278  type,
279  discard_time,
280  path_info_len,
281  path_info);
282  if (-1 == used)
283  {
284  GNUNET_break (0);
285  return GNUNET_SYSERR;
286  }
287  if (0 == used)
288  {
289  /* duplicate */
290  return GNUNET_NO;
291  }
293  "Stored data under key `%s' in cache\n",
294  GNUNET_h2s (key));
295  if (NULL != h->filter)
297  key);
298  GNUNET_STATISTICS_update (h->stats,
299  "# bytes stored",
300  used,
301  GNUNET_NO);
302  GNUNET_STATISTICS_update (h->stats,
303  "# items stored",
304  1,
305  GNUNET_NO);
306  while (h->utilization + used > h->env.quota)
308  h->api->del (h->api->cls));
309  h->utilization += used;
310  return GNUNET_OK;
311 }
312 
313 
314 unsigned int
316  const struct GNUNET_HashCode *key,
317  enum GNUNET_BLOCK_Type type,
319  void *iter_cls)
320 {
321  GNUNET_STATISTICS_update (h->stats,
322  "# requests received",
323  1,
324  GNUNET_NO);
326  "Processing request for key `%s'\n",
327  GNUNET_h2s (key));
328  if ((NULL != h->filter) &&
329  (GNUNET_OK !=
331  {
332  GNUNET_STATISTICS_update (h->stats,
333  "# requests filtered by bloom filter",
334  1,
335  GNUNET_NO);
337  "Bloomfilter filters request for key `%s'\n",
338  GNUNET_h2s (key));
339  return 0; /* can not be present */
340  }
341  return h->api->get (h->api->cls,
342  key,
343  type,
344  iter, iter_cls);
345 }
346 
347 
348 unsigned int
350  const struct GNUNET_HashCode *key,
351  enum GNUNET_BLOCK_Type type,
352  unsigned int num_results,
354  void *iter_cls)
355 {
356  GNUNET_STATISTICS_update (h->stats,
357  "# proximity search requests received",
358  1,
359  GNUNET_NO);
361  "Processing proximity search at `%s'\n",
362  GNUNET_h2s (key));
363  return h->api->get_closest (h->api->cls,
364  key,
365  type,
366  num_results,
367  iter, iter_cls);
368 }
369 
370 
371 /* end of datacache.c */
static void env_delete_notify(void *cls, const struct GNUNET_HashCode *key, size_t size)
Function called by plugins to notify the datacache about content deletions.
Definition: datacache.c:104
#define LOG(kind,...)
Definition: datacache.c:32
static const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration we are using.
Definition: gnunet-abd.c:36
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
struct GNUNET_HashCode key
The key used in the DHT.
uint32_t data
The data value.
static unsigned long long quota
How much space are we allowed to use?
GNUNET_GenericReturnValue
Named constants for return values.
Definition: gnunet_common.h:92
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
GNUNET_BLOCK_Type
WARNING: This header is generated! In order to add DHT block types, you must register them in GANA,...
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_load(const char *filename, size_t size, unsigned int k)
Load a Bloom filter from a file.
void GNUNET_CONTAINER_bloomfilter_add(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Add an element to the filter.
int GNUNET_CONTAINER_bloomfilter_test(const struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Test if an element is in the filter.
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_init(const char *data, size_t size, unsigned int k)
Create a Bloom filter from raw bits.
void GNUNET_CONTAINER_bloomfilter_remove(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Remove an element from the filter.
void GNUNET_CONTAINER_bloomfilter_free(struct GNUNET_CONTAINER_BloomFilter *bf)
Free the space associated with a filter in memory, flush to drive if needed (do not free the space on...
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_size(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *size)
Get a configuration value that should be a size in bytes.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
unsigned int GNUNET_DATACACHE_get(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results for a particular key in the datacache.
Definition: datacache.c:315
enum GNUNET_GenericReturnValue(* GNUNET_DATACACHE_Iterator)(void *cls, const struct GNUNET_HashCode *key, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, struct GNUNET_TIME_Absolute exp, unsigned int path_info_len, const struct GNUNET_DHT_PathElement *path_info)
An iterator over a set of items stored in the datacache.
enum GNUNET_GenericReturnValue GNUNET_DATACACHE_put(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, uint32_t xor_distance, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, struct GNUNET_TIME_Absolute discard_time, unsigned int path_info_len, const struct GNUNET_DHT_PathElement *path_info)
Store an item in the datacache.
Definition: datacache.c:261
struct GNUNET_DATACACHE_Handle * GNUNET_DATACACHE_create(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section)
Create a data cache.
Definition: datacache.c:128
void GNUNET_DATACACHE_destroy(struct GNUNET_DATACACHE_Handle *h)
Destroy a data cache (and free associated resources).
Definition: datacache.c:235
unsigned int GNUNET_DATACACHE_get_closest(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int num_results, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results that are "close" to a particular key in the datacache.
Definition: datacache.c:349
char * GNUNET_DISK_mktemp(const char *t)
Create an (empty) temporary file on disk.
Definition: disk.c:381
#define GNUNET_log_from_strerror_file(level, component, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_default(void)
Return default project data used by 'libgnunetutil' for GNUnet.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_get(void)
void GNUNET_OS_init(const struct GNUNET_OS_ProjectData *pd)
Setup OS subsystem with project data.
void * GNUNET_PLUGIN_unload(const char *library_name, void *arg)
Unload plugin (runs the "done" callback and returns whatever "done" returned).
Definition: plugin.c:269
void * GNUNET_PLUGIN_load(const char *library_name, void *arg)
Setup plugin (runs the "init" callback and returns whatever "init" returned).
Definition: plugin.c:217
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
static unsigned int size
Size of the "table".
Definition: peer.c:67
const char * name
Internal state of the datacache library.
Definition: datacache.c:41
char * bloom_name
Name for the bloom filter file.
Definition: datacache.c:81
unsigned long long utilization
How much space is in use right now?
Definition: datacache.c:91
struct GNUNET_DATACACHE_PluginEnvironment env
Environment provided to our plugin.
Definition: datacache.c:86
char * section
Configuration section to use.
Definition: datacache.c:60
char * short_name
Short name for the plugin (e.g.
Definition: datacache.c:71
const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition: datacache.c:50
struct GNUNET_STATISTICS_Handle * stats
Opaque handle for the statistics service.
Definition: datacache.c:55
struct GNUNET_CONTAINER_BloomFilter * filter
Bloomfilter to quickly tell if we don't have the content.
Definition: datacache.c:45
char * lib_name
Name of the library (e.g.
Definition: datacache.c:76
struct GNUNET_DATACACHE_PluginFunctions * api
API of the transport as returned by the plugin's initialization function.
Definition: datacache.c:66
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
void * cls
Closure to use for callbacks.
struct returned by the initialization function of the plugin
A (signed) path tracking a block's flow through the DHT is represented by an array of path elements,...
A 512-bit hashcode.
Project-specific data used to help the OS subsystem find installation paths.
const char * libname
Name of a library that is installed in the "lib/" directory of the project, such as "libgnunetutil".
Handle for the service.
Time for absolute times used by GNUnet, in microseconds.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model