GNUnet  0.17.6
datacache.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2004-2010, 2015, 2022 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
25 #include "platform.h"
26 #include "gnunet_util_lib.h"
27 #include "gnunet_datacache_lib.h"
30 
31 
32 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache", __VA_ARGS__)
33 
34 #define LOG_STRERROR_FILE(kind, op, fn) \
35  GNUNET_log_from_strerror_file (kind, "datacache", op, fn)
36 
41 {
46 
51 
56 
60  char *section;
61 
67 
71  char *short_name;
72 
76  char *lib_name;
77 
81  char *bloom_name;
82 
87 
91  unsigned long long utilization;
92 };
93 
94 
103 static void
105  const struct GNUNET_HashCode *key,
106  size_t size)
107 {
108  struct GNUNET_DATACACHE_Handle *h = cls;
109 
111  "Content under key `%s' discarded\n",
112  GNUNET_h2s (key));
113  GNUNET_assert (h->utilization >= size);
114  h->utilization -= size;
116  GNUNET_STATISTICS_update (h->stats,
117  "# bytes stored",
118  -(long long) size,
119  GNUNET_NO);
120  GNUNET_STATISTICS_update (h->stats,
121  "# items stored",
122  -1,
123  GNUNET_NO);
124 }
125 
126 
129  const char *section)
130 {
131  unsigned int bf_size;
132  unsigned long long quota;
134  char *libname;
135  char *name;
136  const struct GNUNET_OS_ProjectData *pd;
137 
138  if (GNUNET_OK !=
140  section,
141  "QUOTA",
142  &quota))
143  {
145  section,
146  "QUOTA");
147  return NULL;
148  }
149  if (GNUNET_OK !=
151  section,
152  "DATABASE",
153  &name))
154  {
156  section,
157  "DATABASE");
158  return NULL;
159  }
160  bf_size = quota / 32; /* 8 bit per entry, 1 bit per 32 kb in DB */
161 
163 
164  if (GNUNET_YES !=
166  section,
167  "DISABLE_BF"))
168  {
169  if (GNUNET_YES !=
171  section,
172  "DISABLE_BF_RC"))
173  {
174  ret->bloom_name = GNUNET_DISK_mktemp ("gnunet-datacachebloom");
175  }
176  if (NULL != ret->bloom_name)
177  {
179  ret->bloom_name,
180  quota / 1024, /* 8 bit per entry in DB, expect 1k entries */
181  5);
182  }
183  if (NULL == ret->filter)
184  {
185  ret->filter =
187  bf_size,
188  5); /* approx. 3% false positives at max use */
189  }
190  }
191  ret->stats = GNUNET_STATISTICS_create ("datacache",
192  cfg);
193  ret->section = GNUNET_strdup (section);
194  ret->env.cfg = cfg;
195  ret->env.delete_notify = &env_delete_notify;
196  ret->env.section = ret->section;
197  ret->env.cls = ret;
198  ret->env.delete_notify = &env_delete_notify;
199  ret->env.quota = quota;
201  "Loading `%s' datacache plugin\n",
202  name);
204  "libgnunet_plugin_datacache_%s",
205  name);
206  ret->short_name = name;
207  ret->lib_name = libname;
208  /* Load the plugin within GNUnet's default context */
212  &ret->env);
213  GNUNET_OS_init (pd);
214  if (NULL == ret->api)
215  {
216  /* Try to load the plugin within the application's context
217  This normally happens when the application is not GNUnet itself but a
218  third party; inside GNUnet this is effectively a double failure. */
220  &ret->env);
221  if (NULL == ret->api)
222  {
224  "Failed to load datacache plugin for `%s'\n",
225  name);
227  return NULL;
228  }
229  }
230  return ret;
231 }
232 
233 
234 void
236 {
237  if (NULL != h->filter)
239  if (NULL != h->api)
240  GNUNET_break (NULL ==
241  GNUNET_PLUGIN_unload (h->lib_name,
242  h->api));
243  GNUNET_free (h->lib_name);
244  GNUNET_free (h->short_name);
245  GNUNET_free (h->section);
246  if (NULL != h->bloom_name)
247  {
248  if (0 != unlink (h->bloom_name))
250  "datacache",
251  "unlink",
252  h->bloom_name);
253  GNUNET_free (h->bloom_name);
254  }
256  GNUNET_free (h);
257 }
258 
259 
262  uint32_t xor_distance,
263  const struct GNUNET_DATACACHE_Block *block)
264 {
265  ssize_t used;
266 
267  used = h->api->put (h->api->cls,
268  xor_distance,
269  block);
270  if (-1 == used)
271  {
272  GNUNET_break (0);
273  return GNUNET_SYSERR;
274  }
275  if (0 == used)
276  {
277  /* duplicate */
278  return GNUNET_NO;
279  }
281  "Stored data under key `%s' in cache\n",
282  GNUNET_h2s (&block->key));
283  if (NULL != h->filter)
285  &block->key);
286  GNUNET_STATISTICS_update (h->stats,
287  "# bytes stored",
288  used,
289  GNUNET_NO);
290  GNUNET_STATISTICS_update (h->stats,
291  "# items stored",
292  1,
293  GNUNET_NO);
294  while (h->utilization + used > h->env.quota)
296  h->api->del (h->api->cls));
297  h->utilization += used;
298  return GNUNET_OK;
299 }
300 
301 
302 unsigned int
304  const struct GNUNET_HashCode *key,
305  enum GNUNET_BLOCK_Type type,
307  void *iter_cls)
308 {
309  GNUNET_STATISTICS_update (h->stats,
310  "# requests received",
311  1,
312  GNUNET_NO);
314  "Processing request for key `%s'\n",
315  GNUNET_h2s (key));
316  if ( (NULL != h->filter) &&
317  (GNUNET_OK !=
319  key)) )
320  {
321  GNUNET_STATISTICS_update (h->stats,
322  "# requests filtered by bloom filter",
323  1,
324  GNUNET_NO);
326  "Bloomfilter filters request for key `%s'\n",
327  GNUNET_h2s (key));
328  return 0; /* can not be present */
329  }
330  return h->api->get (h->api->cls,
331  key,
332  type,
333  iter, iter_cls);
334 }
335 
336 
337 unsigned int
339  const struct GNUNET_HashCode *key,
340  enum GNUNET_BLOCK_Type type,
341  unsigned int num_results,
343  void *iter_cls)
344 {
345  GNUNET_STATISTICS_update (h->stats,
346  "# proximity search requests received",
347  1,
348  GNUNET_NO);
350  "Processing proximity search at `%s'\n",
351  GNUNET_h2s (key));
352  return h->api->get_closest (h->api->cls,
353  key,
354  type,
355  num_results,
356  iter,
357  iter_cls);
358 }
359 
360 
361 /* end of datacache.c */
static void env_delete_notify(void *cls, const struct GNUNET_HashCode *key, size_t size)
Function called by plugins to notify the datacache about content deletions.
Definition: datacache.c:104
#define LOG(kind,...)
Definition: datacache.c:32
static const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration we are using.
Definition: gnunet-abd.c:36
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
struct GNUNET_HashCode key
The key used in the DHT.
static unsigned long long quota
How much space are we allowed to use?
API for database backends for the datacache.
GNUNET_BLOCK_Type
WARNING: This header is generated! In order to add DHT block types, you must register them in GANA,...
API to create, modify and access statistics.
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_load(const char *filename, size_t size, unsigned int k)
Load a Bloom filter from a file.
void GNUNET_CONTAINER_bloomfilter_add(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Add an element to the filter.
bool GNUNET_CONTAINER_bloomfilter_test(const struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Test if an element is in the filter.
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_init(const char *data, size_t size, unsigned int k)
Create a Bloom filter from raw bits.
void GNUNET_CONTAINER_bloomfilter_remove(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Remove an element from the filter.
void GNUNET_CONTAINER_bloomfilter_free(struct GNUNET_CONTAINER_BloomFilter *bf)
Free the space associated with a filter in memory, flush to drive if needed (do not free the space on...
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_size(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *size)
Get a configuration value that should be a size in bytes.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
unsigned int GNUNET_DATACACHE_get(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results for a particular key in the datacache.
Definition: datacache.c:303
enum GNUNET_GenericReturnValue GNUNET_DATACACHE_put(struct GNUNET_DATACACHE_Handle *h, uint32_t xor_distance, const struct GNUNET_DATACACHE_Block *block)
Store an item in the datacache.
Definition: datacache.c:261
enum GNUNET_GenericReturnValue(* GNUNET_DATACACHE_Iterator)(void *cls, const struct GNUNET_DATACACHE_Block *block)
An iterator over a set of items stored in the datacache.
struct GNUNET_DATACACHE_Handle * GNUNET_DATACACHE_create(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section)
Create a data cache.
Definition: datacache.c:128
void GNUNET_DATACACHE_destroy(struct GNUNET_DATACACHE_Handle *h)
Destroy a data cache (and free associated resources).
Definition: datacache.c:235
unsigned int GNUNET_DATACACHE_get_closest(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int num_results, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results that are "close" to a particular key in the datacache.
Definition: datacache.c:338
char * GNUNET_DISK_mktemp(const char *t)
Create an (empty) temporary file on disk.
Definition: disk.c:381
GNUNET_GenericReturnValue
Named constants for return values.
Definition: gnunet_common.h:96
@ GNUNET_OK
Definition: gnunet_common.h:99
@ GNUNET_YES
@ GNUNET_NO
Definition: gnunet_common.h:98
@ GNUNET_SYSERR
Definition: gnunet_common.h:97
#define GNUNET_log_from_strerror_file(level, component, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_default(void)
Return default project data used by 'libgnunetutil' for GNUnet.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_get(void)
void GNUNET_OS_init(const struct GNUNET_OS_ProjectData *pd)
Setup OS subsystem with project data.
void * GNUNET_PLUGIN_unload(const char *library_name, void *arg)
Unload plugin (runs the "done" callback and returns whatever "done" returned).
Definition: plugin.c:241
void * GNUNET_PLUGIN_load(const char *library_name, void *arg)
Setup plugin (runs the "init" callback and returns whatever "init" returned).
Definition: plugin.c:197
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
static unsigned int size
Size of the "table".
Definition: peer.c:67
const char * name
Information about a block stored in the datacache.
struct GNUNET_HashCode key
Key of the block.
Internal state of the datacache library.
Definition: datacache.c:41
char * bloom_name
Name for the bloom filter file.
Definition: datacache.c:81
unsigned long long utilization
How much space is in use right now?
Definition: datacache.c:91
struct GNUNET_DATACACHE_PluginEnvironment env
Environment provided to our plugin.
Definition: datacache.c:86
char * section
Configuration section to use.
Definition: datacache.c:60
char * short_name
Short name for the plugin (e.g.
Definition: datacache.c:71
const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition: datacache.c:50
struct GNUNET_STATISTICS_Handle * stats
Opaque handle for the statistics service.
Definition: datacache.c:55
struct GNUNET_CONTAINER_BloomFilter * filter
Bloomfilter to quickly tell if we don't have the content.
Definition: datacache.c:45
char * lib_name
Name of the library (e.g.
Definition: datacache.c:76
struct GNUNET_DATACACHE_PluginFunctions * api
API of the transport as returned by the plugin's initialization function.
Definition: datacache.c:66
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
void * cls
Closure to use for callbacks.
struct returned by the initialization function of the plugin
A 512-bit hashcode.
Project-specific data used to help the OS subsystem find installation paths.
const char * libname
Name of a library that is installed in the "lib/" directory of the project, such as "libgnunetutil".
Handle for the service.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model