GNUnet  0.19.2
datacache.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2004-2010, 2015, 2022 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
25 #include "platform.h"
26 #include "gnunet_util_lib.h"
27 #include "gnunet_datacache_lib.h"
30 
31 
32 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache", __VA_ARGS__)
33 
34 #define LOG_STRERROR_FILE(kind, op, fn) \
35  GNUNET_log_from_strerror_file (kind, "datacache", op, fn)
36 
41 {
46 
51 
56 
60  char *section;
61 
67 
71  char *short_name;
72 
76  char *lib_name;
77 
81  char *bloom_name;
82 
87 
91  unsigned long long utilization;
92 };
93 
94 
103 static void
105  const struct GNUNET_HashCode *key,
106  size_t size)
107 {
108  struct GNUNET_DATACACHE_Handle *h = cls;
109 
111  "Content under key `%s' discarded\n",
112  GNUNET_h2s (key));
113  GNUNET_assert (h->utilization >= size);
114  h->utilization -= size;
116  GNUNET_STATISTICS_update (h->stats,
117  "# bytes stored",
118  -(long long) size,
119  GNUNET_NO);
120  GNUNET_STATISTICS_update (h->stats,
121  "# items stored",
122  -1,
123  GNUNET_NO);
124 }
125 
126 
129  const char *section)
130 {
131  unsigned int bf_size;
132  unsigned long long quota;
134  char *libname;
135  char *name;
136  const struct GNUNET_OS_ProjectData *pd;
137 
138  if (GNUNET_OK !=
140  section,
141  "QUOTA",
142  &quota))
143  {
145  section,
146  "QUOTA");
147  return NULL;
148  }
149  if (GNUNET_OK !=
151  section,
152  "DATABASE",
153  &name))
154  {
156  section,
157  "DATABASE");
158  return NULL;
159  }
160  bf_size = quota / 32; /* 8 bit per entry, 1 bit per 32 kb in DB */
161 
163 
164  if (GNUNET_YES !=
166  section,
167  "DISABLE_BF"))
168  {
169  if (GNUNET_YES !=
171  section,
172  "DISABLE_BF_RC"))
173  {
174  ret->bloom_name = GNUNET_DISK_mktemp ("gnunet-datacachebloom");
175  }
176  if (NULL != ret->bloom_name)
177  {
179  ret->bloom_name,
180  quota / 1024, /* 8 bit per entry in DB, expect 1k entries */
181  5);
182  }
183  if (NULL == ret->filter)
184  {
185  ret->filter =
187  bf_size,
188  5); /* approx. 3% false positives at max use */
189  }
190  }
191  ret->stats = GNUNET_STATISTICS_create ("datacache",
192  cfg);
193  ret->section = GNUNET_strdup (section);
194  ret->env.cfg = cfg;
195  ret->env.delete_notify = &env_delete_notify;
196  ret->env.section = ret->section;
197  ret->env.cls = ret;
198  ret->env.delete_notify = &env_delete_notify;
199  ret->env.quota = quota;
201  "Loading `%s' datacache plugin\n",
202  name);
204  "libgnunet_plugin_datacache_%s",
205  name);
206  ret->short_name = name;
207  ret->lib_name = libname;
208  /* Load the plugin within GNUnet's default context */
212  &ret->env);
213  GNUNET_OS_init (pd);
214  if (NULL == ret->api)
215  {
216  /* Try to load the plugin within the application's context
217  This normally happens when the application is not GNUnet itself but a
218  third party; inside GNUnet this is effectively a double failure. */
220  &ret->env);
221  if (NULL == ret->api)
222  {
224  "Failed to load datacache plugin for `%s'\n",
225  name);
227  return NULL;
228  }
229  }
230  return ret;
231 }
232 
233 
234 void
236 {
237  if (NULL != h->filter)
239  if (NULL != h->api)
240  GNUNET_break (NULL ==
241  GNUNET_PLUGIN_unload (h->lib_name,
242  h->api));
243  GNUNET_free (h->lib_name);
244  GNUNET_free (h->short_name);
245  GNUNET_free (h->section);
246  if (NULL != h->bloom_name)
247  {
248  if (0 != unlink (h->bloom_name))
250  "datacache",
251  "unlink",
252  h->bloom_name);
253  GNUNET_free (h->bloom_name);
254  }
256  GNUNET_free (h);
257 }
258 
259 
262  uint32_t xor_distance,
263  const struct GNUNET_DATACACHE_Block *block)
264 {
265  ssize_t used;
266 
267  used = h->api->put (h->api->cls,
268  xor_distance,
269  block);
270  if (-1 == used)
271  {
272  GNUNET_break (0);
273  return GNUNET_SYSERR;
274  }
275  if (0 == used)
276  {
277  /* duplicate */
278  return GNUNET_NO;
279  }
281  "Stored data under key `%s' in cache\n",
282  GNUNET_h2s (&block->key));
283  if (NULL != h->filter)
285  &block->key);
286  GNUNET_STATISTICS_update (h->stats,
287  "# bytes stored",
288  used,
289  GNUNET_NO);
290  GNUNET_STATISTICS_update (h->stats,
291  "# items stored",
292  1,
293  GNUNET_NO);
294  while (h->utilization + used > h->env.quota)
296  h->api->del (h->api->cls));
297  h->utilization += used;
298  return GNUNET_OK;
299 }
300 
301 
302 unsigned int
304  const struct GNUNET_HashCode *key,
305  enum GNUNET_BLOCK_Type type,
307  void *iter_cls)
308 {
309  GNUNET_STATISTICS_update (h->stats,
310  "# requests received",
311  1,
312  GNUNET_NO);
314  "Processing request for key `%s'\n",
315  GNUNET_h2s (key));
316  if ( (NULL != h->filter) &&
317  (GNUNET_OK !=
319  key)) )
320  {
321  GNUNET_STATISTICS_update (h->stats,
322  "# requests filtered by bloom filter",
323  1,
324  GNUNET_NO);
326  "Bloomfilter filters request for key `%s'\n",
327  GNUNET_h2s (key));
328  return 0; /* can not be present */
329  }
330  return h->api->get (h->api->cls,
331  key,
332  type,
333  iter, iter_cls);
334 }
335 
336 
337 unsigned int
339  const struct GNUNET_HashCode *key,
340  enum GNUNET_BLOCK_Type type,
341  unsigned int num_results,
343  void *iter_cls)
344 {
345  GNUNET_STATISTICS_update (h->stats,
346  "# proximity search requests received",
347  1,
348  GNUNET_NO);
350  "Processing proximity search at `%s'\n",
351  GNUNET_h2s (key));
352  return h->api->get_closest (h->api->cls,
353  key,
354  type,
355  num_results,
356  iter,
357  iter_cls);
358 }
359 
360 
361 /* end of datacache.c */
GNUNET_BLOCK_Type
WARNING: This header is generated! In order to add DHT block types, you must register them in GANA,...
static void env_delete_notify(void *cls, const struct GNUNET_HashCode *key, size_t size)
Function called by plugins to notify the datacache about content deletions.
Definition: datacache.c:104
#define LOG(kind,...)
Definition: datacache.c:32
static const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration we are using.
Definition: gnunet-abd.c:36
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static struct GNUNET_ARM_Handle * h
Connection with ARM.
Definition: gnunet-arm.c:99
struct GNUNET_HashCode key
The key used in the DHT.
static unsigned long long quota
How much space are we allowed to use?
API for database backends for the datacache.
API to create, modify and access statistics.
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_load(const char *filename, size_t size, unsigned int k)
Load a Bloom filter from a file.
void GNUNET_CONTAINER_bloomfilter_add(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Add an element to the filter.
bool GNUNET_CONTAINER_bloomfilter_test(const struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Test if an element is in the filter.
struct GNUNET_CONTAINER_BloomFilter * GNUNET_CONTAINER_bloomfilter_init(const char *data, size_t size, unsigned int k)
Create a Bloom filter from raw bits.
void GNUNET_CONTAINER_bloomfilter_remove(struct GNUNET_CONTAINER_BloomFilter *bf, const struct GNUNET_HashCode *e)
Remove an element from the filter.
void GNUNET_CONTAINER_bloomfilter_free(struct GNUNET_CONTAINER_BloomFilter *bf)
Free the space associated with a filter in memory, flush to drive if needed (do not free the space on...
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_size(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, unsigned long long *size)
Get a configuration value that should be a size in bytes.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_yesno(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option)
Get a configuration value that should be in a set of "YES" or "NO".
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_string(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be a string.
unsigned int GNUNET_DATACACHE_get(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results for a particular key in the datacache.
Definition: datacache.c:303
enum GNUNET_GenericReturnValue GNUNET_DATACACHE_put(struct GNUNET_DATACACHE_Handle *h, uint32_t xor_distance, const struct GNUNET_DATACACHE_Block *block)
Store an item in the datacache.
Definition: datacache.c:261
enum GNUNET_GenericReturnValue(* GNUNET_DATACACHE_Iterator)(void *cls, const struct GNUNET_DATACACHE_Block *block)
An iterator over a set of items stored in the datacache.
struct GNUNET_DATACACHE_Handle * GNUNET_DATACACHE_create(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section)
Create a data cache.
Definition: datacache.c:128
void GNUNET_DATACACHE_destroy(struct GNUNET_DATACACHE_Handle *h)
Destroy a data cache (and free associated resources).
Definition: datacache.c:235
unsigned int GNUNET_DATACACHE_get_closest(struct GNUNET_DATACACHE_Handle *h, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int num_results, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results that are "close" to a particular key in the datacache.
Definition: datacache.c:338
char * GNUNET_DISK_mktemp(const char *t)
Create an (empty) temporary file on disk.
Definition: disk.c:380
GNUNET_GenericReturnValue
Named constants for return values.
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_log_from_strerror_file(level, component, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
int int GNUNET_asprintf(char **buf, const char *format,...) __attribute__((format(printf
Like asprintf, just portable.
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_default(void)
Return default project data used by 'libgnunetutil' for GNUnet.
const struct GNUNET_OS_ProjectData * GNUNET_OS_project_data_get(void)
void GNUNET_OS_init(const struct GNUNET_OS_ProjectData *pd)
Setup OS subsystem with project data.
void * GNUNET_PLUGIN_unload(const char *library_name, void *arg)
Unload plugin (runs the "done" callback and returns whatever "done" returned).
Definition: plugin.c:242
void * GNUNET_PLUGIN_load(const char *library_name, void *arg)
Setup plugin (runs the "init" callback and returns whatever "init" returned).
Definition: plugin.c:198
struct GNUNET_STATISTICS_Handle * GNUNET_STATISTICS_create(const char *subsystem, const struct GNUNET_CONFIGURATION_Handle *cfg)
Get handle for the statistics service.
void GNUNET_STATISTICS_update(struct GNUNET_STATISTICS_Handle *handle, const char *name, int64_t delta, int make_persistent)
Set statistic value for the peer.
void GNUNET_STATISTICS_destroy(struct GNUNET_STATISTICS_Handle *h, int sync_first)
Destroy a handle (free all state associated with it).
static unsigned int size
Size of the "table".
Definition: peer.c:68
const char * name
Information about a block stored in the datacache.
struct GNUNET_HashCode key
Key of the block.
Internal state of the datacache library.
Definition: datacache.c:41
char * bloom_name
Name for the bloom filter file.
Definition: datacache.c:81
unsigned long long utilization
How much space is in use right now?
Definition: datacache.c:91
struct GNUNET_DATACACHE_PluginEnvironment env
Environment provided to our plugin.
Definition: datacache.c:86
char * section
Configuration section to use.
Definition: datacache.c:60
char * short_name
Short name for the plugin (e.g.
Definition: datacache.c:71
const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition: datacache.c:50
struct GNUNET_STATISTICS_Handle * stats
Opaque handle for the statistics service.
Definition: datacache.c:55
struct GNUNET_CONTAINER_BloomFilter * filter
Bloomfilter to quickly tell if we don't have the content.
Definition: datacache.c:45
char * lib_name
Name of the library (e.g.
Definition: datacache.c:76
struct GNUNET_DATACACHE_PluginFunctions * api
API of the transport as returned by the plugin's initialization function.
Definition: datacache.c:66
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
void * cls
Closure to use for callbacks.
struct returned by the initialization function of the plugin
A 512-bit hashcode.
Project-specific data used to help the OS subsystem find installation paths.
const char * libname
Name of a library that is installed in the "lib/" directory of the project, such as "libgnunetutil".
Handle for the service.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model