GNUnet  0.11.x
plugin_namecache_sqlite.c
Go to the documentation of this file.
1 /*
2  * This file is part of GNUnet
3  * Copyright (C) 2009-2013 GNUnet e.V.
4  *
5  * GNUnet is free software: you can redistribute it and/or modify it
6  * under the terms of the GNU Affero General Public License as published
7  * by the Free Software Foundation, either version 3 of the License,
8  * or (at your option) any later version.
9  *
10  * GNUnet is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  * Affero General Public License for more details.
14  *
15  * You should have received a copy of the GNU Affero General Public License
16  * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
26 #include "platform.h"
27 #include "gnunet_sq_lib.h"
30 #include "gnunet_gnsrecord_lib.h"
31 #include "namecache.h"
32 #include <sqlite3.h>
33 
44 #define BUSY_TIMEOUT_MS 1000
45 
46 
52 #define LOG_SQLITE(db, level, cmd) do { GNUNET_log_from (level, \
53  "namecache-sqlite", _ ( \
54  "`%s' failed at %s:%d with error: %s\n"), \
55  cmd, \
56  __FILE__, __LINE__, \
57  sqlite3_errmsg ( \
58  db->dbh)); \
59 } while (0)
60 
61 #define LOG(kind, ...) GNUNET_log_from (kind, "namecache-sqlite", __VA_ARGS__)
62 
63 
67 struct Plugin
68 {
69  const struct GNUNET_CONFIGURATION_Handle *cfg;
70 
74  char *fn;
75 
79  sqlite3 *dbh;
80 
84  sqlite3_stmt *cache_block;
85 
89  sqlite3_stmt *delete_block;
90 
94  sqlite3_stmt *lookup_block;
95 
99  sqlite3_stmt *expire_blocks;
100 };
101 
102 
111 static int
113 {
114  struct GNUNET_SQ_ExecuteStatement es[] = {
115  GNUNET_SQ_make_try_execute ("PRAGMA temp_store=MEMORY"),
116  GNUNET_SQ_make_try_execute ("PRAGMA synchronous=NORMAL"),
117  GNUNET_SQ_make_try_execute ("PRAGMA legacy_file_format=OFF"),
118  GNUNET_SQ_make_try_execute ("PRAGMA auto_vacuum=INCREMENTAL"),
119  GNUNET_SQ_make_try_execute ("PRAGMA encoding=\"UTF-8\""),
120  GNUNET_SQ_make_try_execute ("PRAGMA locking_mode=EXCLUSIVE"),
121  GNUNET_SQ_make_try_execute ("PRAGMA page_size=4092"),
122  GNUNET_SQ_make_try_execute ("PRAGMA journal_mode=WAL"),
123  GNUNET_SQ_make_execute ("CREATE TABLE IF NOT EXISTS ns096blocks ("
124  " query BLOB NOT NULL,"
125  " block BLOB NOT NULL,"
126  " expiration_time INT8 NOT NULL"
127  ")"),
128  GNUNET_SQ_make_execute ("CREATE INDEX IF NOT EXISTS ir_query_hash "
129  "ON ns096blocks (query,expiration_time)"),
130  GNUNET_SQ_make_execute ("CREATE INDEX IF NOT EXISTS ir_block_expiration "
131  "ON ns096blocks (expiration_time)"),
133  };
134  struct GNUNET_SQ_PrepareStatement ps[] = {
136  "INSERT INTO ns096blocks (query,block,expiration_time) VALUES (?, ?, ?)",
137  &plugin->cache_block),
138  GNUNET_SQ_make_prepare ("DELETE FROM ns096blocks WHERE expiration_time<?",
139  &plugin->expire_blocks),
141  "DELETE FROM ns096blocks WHERE query=? AND expiration_time<=?",
142  &plugin->delete_block),
143  GNUNET_SQ_make_prepare ("SELECT block FROM ns096blocks WHERE query=? "
144  "ORDER BY expiration_time DESC LIMIT 1",
145  &plugin->lookup_block),
147  };
148  char *afsdir;
149 
150  if (GNUNET_OK !=
152  "namecache-sqlite",
153  "FILENAME",
154  &afsdir))
155  {
157  "namecache-sqlite",
158  "FILENAME");
159  return GNUNET_SYSERR;
160  }
161  if (GNUNET_OK !=
162  GNUNET_DISK_file_test (afsdir))
163  {
164  if (GNUNET_OK !=
166  {
167  GNUNET_break (0);
168  GNUNET_free (afsdir);
169  return GNUNET_SYSERR;
170  }
171  }
172  /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
173  plugin->fn = afsdir;
174 
175  /* Open database and precompile statements */
176  if (SQLITE_OK !=
177  sqlite3_open (plugin->fn, &plugin->dbh))
178  {
180  _ ("Unable to initialize SQLite: %s.\n"),
181  sqlite3_errmsg (plugin->dbh));
182  return GNUNET_SYSERR;
183  }
184  if (GNUNET_OK !=
186  es))
187  {
188  GNUNET_break (0);
190  _ ("Failed to setup database at `%s'\n"),
191  plugin->fn);
192  return GNUNET_SYSERR;
193  }
194  GNUNET_break (SQLITE_OK ==
195  sqlite3_busy_timeout (plugin->dbh,
196  BUSY_TIMEOUT_MS));
197 
198  if (GNUNET_OK !=
199  GNUNET_SQ_prepare (plugin->dbh,
200  ps))
201  {
202  GNUNET_break (0);
204  _ ("Failed to setup database at `%s'\n"),
205  plugin->fn);
206  return GNUNET_SYSERR;
207  }
208 
209  return GNUNET_OK;
210 }
211 
212 
218 static void
220 {
221  int result;
222  sqlite3_stmt *stmt;
223 
224  if (NULL != plugin->cache_block)
225  sqlite3_finalize (plugin->cache_block);
226  if (NULL != plugin->lookup_block)
227  sqlite3_finalize (plugin->lookup_block);
228  if (NULL != plugin->expire_blocks)
229  sqlite3_finalize (plugin->expire_blocks);
230  if (NULL != plugin->delete_block)
231  sqlite3_finalize (plugin->delete_block);
232  result = sqlite3_close (plugin->dbh);
233  if (result == SQLITE_BUSY)
234  {
236  _ (
237  "Tried to close sqlite without finalizing all prepared statements.\n"));
238  stmt = sqlite3_next_stmt (plugin->dbh,
239  NULL);
240  while (stmt != NULL)
241  {
243  "sqlite",
244  "Closing statement %p\n",
245  stmt);
246  result = sqlite3_finalize (stmt);
247  if (result != SQLITE_OK)
249  "sqlite",
250  "Failed to close statement %p: %d\n",
251  stmt,
252  result);
253  stmt = sqlite3_next_stmt (plugin->dbh,
254  NULL);
255  }
256  result = sqlite3_close (plugin->dbh);
257  }
258  if (SQLITE_OK != result)
259  LOG_SQLITE (plugin,
261  "sqlite3_close");
262 
263  GNUNET_free (plugin->fn);
264 }
265 
266 
272 static void
274 {
275  struct GNUNET_TIME_Absolute now;
276  struct GNUNET_SQ_QueryParam params[] = {
279  };
280  int n;
281 
282  now = GNUNET_TIME_absolute_get ();
283  if (GNUNET_OK !=
284  GNUNET_SQ_bind (plugin->expire_blocks,
285  params))
286  {
287  LOG_SQLITE (plugin,
289  "sqlite3_bind_XXXX");
290  GNUNET_SQ_reset (plugin->dbh,
291  plugin->expire_blocks);
292  return;
293  }
294  n = sqlite3_step (plugin->expire_blocks);
295  GNUNET_SQ_reset (plugin->dbh,
296  plugin->expire_blocks);
297  switch (n)
298  {
299  case SQLITE_DONE:
301  "sqlite",
302  "Records expired\n");
303  return;
304 
305  case SQLITE_BUSY:
306  LOG_SQLITE (plugin,
308  "sqlite3_step");
309  return;
310 
311  default:
312  LOG_SQLITE (plugin,
314  "sqlite3_step");
315  return;
316  }
317 }
318 
319 
327 static int
329  const struct GNUNET_GNSRECORD_Block *block)
330 {
331  static struct GNUNET_TIME_Absolute last_expire;
332  struct Plugin *plugin = cls;
333  struct GNUNET_HashCode query;
334  struct GNUNET_TIME_Absolute expiration;
335  size_t block_size = GNUNET_GNSRECORD_block_get_size (block);
336  struct GNUNET_SQ_QueryParam del_params[] = {
340  };
341  struct GNUNET_SQ_QueryParam ins_params[] = {
344  block_size),
347  };
348  int n;
349 
350  /* run expiration of old cache entries once per hour */
351  if (GNUNET_TIME_absolute_get_duration (last_expire).rel_value_us >
352  GNUNET_TIME_UNIT_HOURS.rel_value_us)
353  {
354  last_expire = GNUNET_TIME_absolute_get ();
356  }
358  GNUNET_GNSRECORD_query_from_block (block, &query));
359  expiration = GNUNET_GNSRECORD_block_get_expiration (block);
361  "Caching new version of block %s (expires %s)\n",
362  GNUNET_h2s (&query),
364  if (block_size > 64 * 65536)
365  {
366  GNUNET_break (0);
367  return GNUNET_SYSERR;
368  }
369 
370  /* delete old version of the block */
371  if (GNUNET_OK !=
372  GNUNET_SQ_bind (plugin->delete_block,
373  del_params))
374  {
376  "sqlite3_bind_XXXX");
377  GNUNET_SQ_reset (plugin->dbh,
378  plugin->delete_block);
379  return GNUNET_SYSERR;
380  }
381  n = sqlite3_step (plugin->delete_block);
382  switch (n)
383  {
384  case SQLITE_DONE:
386  "sqlite",
387  "Old block deleted\n");
388  break;
389 
390  case SQLITE_BUSY:
391  LOG_SQLITE (plugin,
393  "sqlite3_step");
394  break;
395 
396  default:
397  LOG_SQLITE (plugin,
399  "sqlite3_step");
400  break;
401  }
402  GNUNET_SQ_reset (plugin->dbh,
403  plugin->delete_block);
404 
405  /* insert new version of the block */
406  if (GNUNET_OK !=
407  GNUNET_SQ_bind (plugin->cache_block,
408  ins_params))
409  {
410  LOG_SQLITE (plugin,
412  "sqlite3_bind_XXXX");
413  GNUNET_SQ_reset (plugin->dbh,
414  plugin->cache_block);
415  return GNUNET_SYSERR;
416  }
418  "Caching block under derived key `%s'\n",
419  GNUNET_h2s_full (&query));
420  n = sqlite3_step (plugin->cache_block);
421  GNUNET_SQ_reset (plugin->dbh,
422  plugin->cache_block);
423  switch (n)
424  {
425  case SQLITE_DONE:
427  "Record stored\n");
428  return GNUNET_OK;
429 
430  case SQLITE_BUSY:
431  LOG_SQLITE (plugin,
433  "sqlite3_step");
434  return GNUNET_NO;
435 
436  default:
437  LOG_SQLITE (plugin,
439  "sqlite3_step");
440  return GNUNET_SYSERR;
441  }
442 }
443 
444 
455 static int
457  const struct GNUNET_HashCode *query,
459  void *iter_cls)
460 {
461  struct Plugin *plugin = cls;
462  int ret;
463  int sret;
464  size_t block_size;
465  const struct GNUNET_GNSRECORD_Block *block;
466  struct GNUNET_SQ_QueryParam params[] = {
469  };
470  struct GNUNET_SQ_ResultSpec rs[] = {
471  GNUNET_SQ_result_spec_variable_size ((void **) &block,
472  &block_size),
474  };
475 
476  if (GNUNET_OK !=
477  GNUNET_SQ_bind (plugin->lookup_block,
478  params))
479  {
480  LOG_SQLITE (plugin,
482  "sqlite3_bind_XXXX");
483  GNUNET_SQ_reset (plugin->dbh,
484  plugin->lookup_block);
485  return GNUNET_SYSERR;
486  }
487  ret = GNUNET_NO;
488  if (SQLITE_ROW ==
489  (sret = sqlite3_step (plugin->lookup_block)))
490  {
491  if (GNUNET_OK !=
493  rs))
494  {
495  GNUNET_break (0);
496  ret = GNUNET_SYSERR;
497  }
498  else if ((block_size < sizeof(struct GNUNET_GNSRECORD_Block)))
499  {
500  GNUNET_break (0);
502  ret = GNUNET_SYSERR;
503  }
504  else
505  {
507  "Found block under derived key `%s'\n",
508  GNUNET_h2s_full (query));
509  iter (iter_cls,
510  block);
512  ret = GNUNET_YES;
513  }
514  }
515  else
516  {
517  if (SQLITE_DONE != sret)
518  {
519  LOG_SQLITE (plugin,
521  "sqlite_step");
522  ret = GNUNET_SYSERR;
523  }
524  else
525  {
527  "No block found under derived key `%s'\n",
528  GNUNET_h2s_full (query));
529  }
530  }
531  GNUNET_SQ_reset (plugin->dbh,
532  plugin->lookup_block);
533  return ret;
534 }
535 
536 
543 void *
545 {
546  static struct Plugin plugin;
547  const struct GNUNET_CONFIGURATION_Handle *cfg = cls;
549 
550  if (NULL != plugin.cfg)
551  return NULL; /* can only initialize once! */
552  memset (&plugin, 0, sizeof(struct Plugin));
553  plugin.cfg = cfg;
554  if (GNUNET_OK != database_setup (&plugin))
555  {
556  database_shutdown (&plugin);
557  return NULL;
558  }
560  api->cls = &plugin;
564  _ ("Sqlite database running\n"));
565  return api;
566 }
567 
568 
575 void *
577 {
579  struct Plugin *plugin = api->cls;
580 
581  database_shutdown (plugin);
582  plugin->cfg = NULL;
583  GNUNET_free (api);
585  "sqlite plugin is finished\n");
586  return NULL;
587 }
588 
589 
590 /* end of plugin_namecache_sqlite.c */
void(* GNUNET_NAMECACHE_BlockCallback)(void *cls, const struct GNUNET_GNSRECORD_Block *block)
Function called for matching blocks.
void * cls
Closure to pass to all plugin functions.
Description of a DB result cell.
#define GNUNET_TIME_UNIT_HOURS
One hour.
#define GNUNET_SQ_EXECUTE_STATEMENT_END
Terminator for executable statement list.
void * libgnunet_plugin_namecache_sqlite_init(void *cls)
Entry point for the plugin.
Information needed to run a list of SQL statements using GNUNET_SQ_exec_statements().
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_SQ_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
struct GNUNET_BLOCK_PluginFunctions * api
Plugin API.
Definition: block.c:47
const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
#define LOG(kind,...)
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
#define GNUNET_SQ_result_spec_end
End of result parameter specification.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
#define GNUNET_new(type)
Allocate a struct or union of the given type.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_filename(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be the name of a file or directory.
sqlite3_stmt * expire_blocks
Precompiled SQL for removing expired blocks.
Information needed to run a list of SQL statements using GNUNET_SQ_exec_statements().
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
static int database_setup(struct Plugin *plugin)
Initialize the database connections and associated data structures (create tables and indices as need...
struct GNUNET_SQ_PrepareStatement GNUNET_SQ_make_prepare(const char *sql, sqlite3_stmt **pstmt)
Create a struct GNUNET_SQ_PrepareStatement
Definition: sq_prepare.c:37
#define _(String)
GNU gettext support macro.
Definition: platform.h:178
int GNUNET_SQ_extract_result(sqlite3_stmt *result, struct GNUNET_SQ_ResultSpec *rs)
Extract results from a query result according to the given specification.
Definition: sq.c:82
int(* cache_block)(void *cls, const struct GNUNET_GNSRECORD_Block *block)
Cache a block in the datastore.
void * cls
Closure for conv and cleaner.
void GNUNET_SQ_cleanup_result(struct GNUNET_SQ_ResultSpec *rs)
Free all memory that was allocated in rs during GNUNET_SQ_extract_result().
Definition: sq.c:117
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
int GNUNET_SQ_exec_statements(sqlite3 *dbh, const struct GNUNET_SQ_ExecuteStatement *es)
Request execution of an array of statements es from Postgres.
Definition: sq_exec.c:76
size_t GNUNET_GNSRECORD_block_get_size(const struct GNUNET_GNSRECORD_Block *block)
Returns the length of this block in bytes.
int(* lookup_block)(void *cls, const struct GNUNET_HashCode *query, GNUNET_NAMECACHE_BlockCallback iter, void *iter_cls)
Get the block for a particular zone and label in the datastore.
sqlite3_stmt * delete_block
Precompiled SQL for deleting an older block.
static void namecache_sqlite_expire_blocks(struct Plugin *plugin)
Removes any expired block.
struct GNUNET_SQ_ExecuteStatement GNUNET_SQ_make_execute(const char *sql)
Create a struct GNUNET_SQ_ExecuteStatement where errors are fatal.
Definition: sq_exec.c:36
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_variable_size(void **dst, size_t *sptr)
Variable-size result expected.
static int result
Global testing status.
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_absolute_time(const struct GNUNET_TIME_Absolute *x)
Generate query parameter for an absolute time value.
A 512-bit hashcode.
int GNUNET_SQ_bind(sqlite3_stmt *stmt, const struct GNUNET_SQ_QueryParam *params)
Execute binding operations for a prepared statement.
Definition: sq.c:37
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:86
static int namecache_sqlite_cache_block(void *cls, const struct GNUNET_GNSRECORD_Block *block)
Cache a block in the datastore.
#define BUSY_TIMEOUT_MS
After how many ms "busy" should a DB operation fail for good? A low value makes sure that we are more...
void * iter_cls
Iterator cls.
const char * GNUNET_h2s_full(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
sqlite3_stmt * cache_block
Precompiled SQL for caching a block.
helper functions for Sqlite3 DB interactions
Description of a DB query parameter.
Definition: gnunet_sq_lib.h:54
struct returned by the initialization function of the plugin
char * fn
Filename used for the DB.
#define GNUNET_SQ_PREPARE_END
Terminator for executable statement list.
void * libgnunet_plugin_namecache_sqlite_done(void *cls)
Exit point from the plugin.
char * n
Definition: gnunet-cmd.c:66
void GNUNET_SQ_reset(sqlite3 *dbh, sqlite3_stmt *stmt)
Reset stmt and log error.
Definition: sq.c:132
configuration data
Definition: configuration.c:84
Handle for a plugin.
Definition: block.c:37
struct GNUNET_PQ_Context * dbh
Native Postgres database handle.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_duration(struct GNUNET_TIME_Absolute whence)
Get the duration of an operation as the difference of the current time and the given start time "henc...
Definition: time.c:263
int GNUNET_SQ_prepare(sqlite3 *dbh, const struct GNUNET_SQ_PrepareStatement *ps)
Prepare all statements given in the (NULL,NULL)-terminated array at ps.
Definition: sq_prepare.c:58
#define GNUNET_log(kind,...)
struct Plugin * plugin
enum GNUNET_GenericReturnValue GNUNET_GNSRECORD_query_from_block(const struct GNUNET_GNSRECORD_Block *block, struct GNUNET_HashCode *query)
Builds the query hash from a block.
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.
Time for absolute times used by GNUnet, in microseconds.
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_create_for_file(const char *filename)
Create the directory structure for storing a file.
Definition: disk.c:562
#define GNUNET_SQ_query_param_end
End of query parameter specification.
Definition: gnunet_sq_lib.h:86
const char * GNUNET_STRINGS_absolute_time_to_string(struct GNUNET_TIME_Absolute t)
Like asctime, except for GNUnet time.
Definition: strings.c:604
#define GNUNET_log_from(kind, comp,...)
sqlite3_stmt * lookup_block
Precompiled SQL for looking up a block.
struct GNUNET_TIME_Absolute GNUNET_GNSRECORD_block_get_expiration(const struct GNUNET_GNSRECORD_Block *block)
Returns the expiration of a block.
GNUNET_PEERSTORE_Processor iter
Iterator.
common internal definitions for namecache service
#define LOG_SQLITE(db, level, cmd)
Log an error message at log-level &#39;level&#39; that indicates a failure of the command &#39;cmd&#39; on file &#39;file...
struct GNUNET_SQ_ExecuteStatement GNUNET_SQ_make_try_execute(const char *sql)
Create a struct GNUNET_SQ_ExecuteStatement where errors should be tolerated.
Definition: sq_exec.c:55
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory)...
Definition: disk.c:437
static int namecache_sqlite_lookup_block(void *cls, const struct GNUNET_HashCode *query, GNUNET_NAMECACHE_BlockCallback iter, void *iter_cls)
Get the block for a particular zone and label in the datastore.
#define GNUNET_free(ptr)
Wrapper around free.
static void database_shutdown(struct Plugin *plugin)
Shutdown database connection and associate data structures.