GNUnet  0.16.x
plugin_datacache_postgres.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2006, 2009, 2010, 2012, 2015, 2017, 2018, 2022 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
26 #include "platform.h"
27 #include "gnunet_util_lib.h"
28 #include "gnunet_pq_lib.h"
30 
31 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache-postgres", __VA_ARGS__)
32 
36 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 24)
37 
41 struct Plugin
42 {
47 
52 
56  unsigned int num_items;
57 };
58 
59 
66 static int
68 {
69  struct GNUNET_PQ_ExecuteStatement es[] = {
71  "CREATE TEMPORARY SEQUENCE IF NOT EXISTS gn011dc_oid_seq"),
72  GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS gn011dc ("
73  " oid OID NOT NULL DEFAULT nextval('gn011dc_oid_seq'),"
74  " type INTEGER NOT NULL,"
75  " prox INTEGER NOT NULL,"
76  " discard_time BIGINT NOT NULL,"
77  " key BYTEA NOT NULL,"
78  " value BYTEA NOT NULL,"
79  " path BYTEA DEFAULT NULL)"),
81  "ALTER SEQUENCE gnu011dc_oid_seq OWNED BY gn011dc.oid"),
83  "CREATE INDEX IF NOT EXISTS idx_oid ON gn011dc (oid)"),
85  "CREATE INDEX IF NOT EXISTS idx_key ON gn011dc (key)"),
87  "CREATE INDEX IF NOT EXISTS idx_dt ON gn011dc (discard_time)"),
89  "ALTER TABLE gn011dc ALTER value SET STORAGE EXTERNAL"),
90  GNUNET_PQ_make_execute ("ALTER TABLE gn011dc ALTER key SET STORAGE PLAIN"),
92  };
93  struct GNUNET_PQ_PreparedStatement ps[] = {
94  GNUNET_PQ_make_prepare ("getkt",
95  "SELECT discard_time,type,value,path FROM gn011dc "
96  "WHERE key=$1 AND type=$2 AND discard_time >= $3",
97  3),
98  GNUNET_PQ_make_prepare ("getk",
99  "SELECT discard_time,type,value,path FROM gn011dc "
100  "WHERE key=$1 AND discard_time >= $2",
101  2),
102  GNUNET_PQ_make_prepare ("getex",
103  "SELECT length(value) AS len,oid,key FROM gn011dc"
104  " WHERE discard_time < $1"
105  " ORDER BY discard_time ASC LIMIT 1",
106  1),
107  GNUNET_PQ_make_prepare ("getm",
108  "SELECT length(value) AS len,oid,key FROM gn011dc"
109  " ORDER BY prox ASC, discard_time ASC LIMIT 1",
110  0),
111  GNUNET_PQ_make_prepare ("get_closest",
112  "(SELECT discard_time,type,value,path,key FROM gn011dc"
113  " WHERE key >= $1"
114  " AND discard_time >= $2"
115  " AND ( (type = $3) OR ( 0 = $3) )"
116  " ORDER BY key ASC"
117  " LIMIT $4)"
118  " UNION "
119  "(SELECT discard_time,type,value,path,key FROM gn011dc"
120  " WHERE key <= $1"
121  " AND discard_time >= $2"
122  " AND ( (type = $3) OR ( 0 = $3) )"
123  " ORDER BY key DESC"
124  " LIMIT $4)",
125  4),
126  GNUNET_PQ_make_prepare ("delrow",
127  "DELETE FROM gn011dc WHERE oid=$1",
128  1),
129  GNUNET_PQ_make_prepare ("put",
130  "INSERT INTO gn011dc (type, prox, discard_time, key, value, path) "
131  "VALUES ($1, $2, $3, $4, $5, $6)",
132  6),
134  };
135 
137  "datacache-postgres",
138  NULL,
139  es,
140  ps);
141  if (NULL == plugin->dbh)
142  return GNUNET_SYSERR;
143  return GNUNET_OK;
144 }
145 
146 
161 static ssize_t
163  const struct GNUNET_HashCode *key,
164  uint32_t prox,
165  size_t data_size,
166  const char *data,
167  enum GNUNET_BLOCK_Type type,
168  struct GNUNET_TIME_Absolute discard_time,
169  unsigned int path_info_len,
170  const struct GNUNET_DHT_PathElement *path_info)
171 {
172  struct Plugin *plugin = cls;
173  uint32_t type32 = (uint32_t) type;
174  struct GNUNET_PQ_QueryParam params[] = {
177  GNUNET_PQ_query_param_absolute_time (&discard_time),
181  path_info_len * sizeof(struct
184  };
186 
188  "put",
189  params);
190  if (0 > ret)
191  return -1;
192  plugin->num_items++;
193  return data_size + OVERHEAD;
194 }
195 
196 
201 {
206 
210  void *iter_cls;
211 
215  const struct GNUNET_HashCode *key;
216 };
217 
218 
228 static void
229 handle_results (void *cls,
230  PGresult *result,
231  unsigned int num_results)
232 {
233  struct HandleResultContext *hrc = cls;
234 
235  for (unsigned int i = 0; i < num_results; i++)
236  {
237  struct GNUNET_TIME_Absolute expiration_time;
238  uint32_t type;
239  void *data;
240  size_t data_size;
241  struct GNUNET_DHT_PathElement *path;
242  size_t path_len;
243  struct GNUNET_PQ_ResultSpec rs[] = {
244  GNUNET_PQ_result_spec_absolute_time ("discard_time",
245  &expiration_time),
247  &type),
249  &data,
250  &data_size),
252  (void **) &path,
253  &path_len),
255  };
256 
257  if (GNUNET_YES !=
259  rs,
260  i))
261  {
262  GNUNET_break (0);
263  return;
264  }
265  if (0 != (path_len % sizeof(struct GNUNET_DHT_PathElement)))
266  {
267  GNUNET_break (0);
268  path_len = 0;
269  }
270  path_len %= sizeof(struct GNUNET_DHT_PathElement);
272  "Found result of size %u bytes and type %u in database\n",
273  (unsigned int) data_size,
274  (unsigned int) type);
275  if ((NULL != hrc->iter) &&
276  (GNUNET_SYSERR ==
277  hrc->iter (hrc->iter_cls,
278  hrc->key,
279  data_size,
280  data,
281  (enum GNUNET_BLOCK_Type) type,
282  expiration_time,
283  path_len,
284  path)))
285  {
287  "Ending iteration (client error)\n");
289  return;
290  }
292  }
293 }
294 
295 
307 static unsigned int
309  const struct GNUNET_HashCode *key,
310  enum GNUNET_BLOCK_Type type,
312  void *iter_cls)
313 {
314  struct Plugin *plugin = cls;
315  uint32_t type32 = (uint32_t) type;
316  struct GNUNET_TIME_Absolute now = { 0 };
317  struct GNUNET_PQ_QueryParam paramk[] = {
321  };
322  struct GNUNET_PQ_QueryParam paramkt[] = {
327  };
329  struct HandleResultContext hr_ctx;
330 
331  now = GNUNET_TIME_absolute_get ();
332  hr_ctx.iter = iter;
333  hr_ctx.iter_cls = iter_cls;
334  hr_ctx.key = key;
336  (0 == type) ? "getk" : "getkt",
337  (0 == type) ? paramk : paramkt,
339  &hr_ctx);
340  if (res < 0)
341  return 0;
342  return res;
343 }
344 
345 
353 static int
355 {
356  struct Plugin *plugin = cls;
357  struct GNUNET_PQ_QueryParam pempty[] = {
359  };
360  uint32_t size;
361  uint32_t oid;
362  struct GNUNET_HashCode key;
363  struct GNUNET_PQ_ResultSpec rs[] = {
365  &size),
367  &oid),
369  &key),
371  };
373  struct GNUNET_PQ_QueryParam dparam[] = {
376  };
377  struct GNUNET_TIME_Absolute now;
378  struct GNUNET_PQ_QueryParam xparam[] = {
381  };
382 
383  now = GNUNET_TIME_absolute_get ();
385  "getex",
386  xparam,
387  rs);
388  if (0 >= res)
390  "getm",
391  pempty,
392  rs);
393  if (0 > res)
394  return GNUNET_SYSERR;
396  {
397  /* no result */
399  "Ending iteration (no more results)\n");
400  return 0;
401  }
403  "delrow",
404  dparam);
405  if (0 > res)
406  {
408  return GNUNET_SYSERR;
409  }
410  plugin->num_items--;
412  &key,
413  size + OVERHEAD);
415  return GNUNET_OK;
416 }
417 
418 
423 {
428 
432  void *iter_cls;
433 };
434 
435 
445 static void
446 extract_result_cb (void *cls,
447  PGresult *result,
448  unsigned int num_results)
449 {
450  struct ExtractResultContext *erc = cls;
451 
452  if (NULL == erc->iter)
453  return;
454  for (unsigned int i = 0; i < num_results; i++)
455  {
456  struct GNUNET_TIME_Absolute expiration_time;
457  uint32_t type;
458  void *data;
459  size_t data_size;
460  struct GNUNET_DHT_PathElement *path;
461  size_t path_len;
462  struct GNUNET_HashCode key;
463  struct GNUNET_PQ_ResultSpec rs[] = {
465  &expiration_time),
467  &type),
469  &data,
470  &data_size),
472  (void **) &path,
473  &path_len),
475  &key),
477  };
478 
479  if (GNUNET_YES !=
481  rs,
482  i))
483  {
484  GNUNET_break (0);
485  return;
486  }
487  if (0 != (path_len % sizeof(struct GNUNET_DHT_PathElement)))
488  {
489  GNUNET_break (0);
490  path_len = 0;
491  }
492  path_len %= sizeof(struct GNUNET_DHT_PathElement);
494  "Found result of size %u bytes and type %u in database\n",
495  (unsigned int) data_size,
496  (unsigned int) type);
497  if (GNUNET_SYSERR ==
498  erc->iter (erc->iter_cls,
499  &key,
500  data_size,
501  data,
502  (enum GNUNET_BLOCK_Type) type,
503  expiration_time,
504  path_len,
505  path))
506  {
508  "Ending iteration (client error)\n");
510  break;
511  }
513  }
514 }
515 
516 
531 static unsigned int
533  const struct GNUNET_HashCode *key,
534  enum GNUNET_BLOCK_Type type,
535  unsigned int num_results,
537  void *iter_cls)
538 {
539  struct Plugin *plugin = cls;
540  uint32_t num_results32 = (uint32_t) num_results;
541  uint32_t type32 = (uint32_t) type;
542  struct GNUNET_TIME_Absolute now;
543  struct GNUNET_PQ_QueryParam params[] = {
547  GNUNET_PQ_query_param_uint32 (&num_results32),
549  };
551  struct ExtractResultContext erc;
552 
553  erc.iter = iter;
554  erc.iter_cls = iter_cls;
555  now = GNUNET_TIME_absolute_get ();
557  "get_closest",
558  params,
560  &erc);
561  if (0 > res)
562  {
564  "Ending iteration (postgres error)\n");
565  return 0;
566  }
568  {
569  /* no result */
571  "Ending iteration (no more results)\n");
572  return 0;
573  }
574  return res;
575 }
576 
577 
584 void *
586 {
589  struct Plugin *plugin;
590 
591  plugin = GNUNET_new (struct Plugin);
592  plugin->env = env;
593 
595  {
597  return NULL;
598  }
599 
601  api->cls = plugin;
602  api->get = &postgres_plugin_get;
603  api->put = &postgres_plugin_put;
604  api->del = &postgres_plugin_del;
605  api->get_closest = &postgres_plugin_get_closest;
607  "Postgres datacache running\n");
608  return api;
609 }
610 
611 
618 void *
620 {
622  struct Plugin *plugin = api->cls;
623 
626  GNUNET_free (api);
627  return NULL;
628 }
629 
630 
631 /* end of plugin_datacache_postgres.c */
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static size_t data_size
Number of bytes in data.
Definition: gnunet-abd.c:187
static int res
struct Plugin * plugin
The process handle to the testbed service.
struct GNUNET_HashCode key
The key used in the DHT.
uint32_t data
The data value.
static int result
Global testing status.
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
GNUNET_DB_QueryStatus
Status code returned from functions running database commands.
Definition: gnunet_db_lib.h:36
@ GNUNET_DB_STATUS_SUCCESS_NO_RESULTS
The transaction succeeded, but yielded zero results.
Definition: gnunet_db_lib.h:54
GNUNET_BLOCK_Type
WARNING: This header is generated! In order to add DHT block types, you must register them in GANA,...
helper functions for Postgres DB interactions
struct GNUNET_PQ_QueryParam GNUNET_PQ_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.
#define GNUNET_PQ_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
enum GNUNET_DB_QueryStatus GNUNET_PQ_eval_prepared_multi_select(struct GNUNET_PQ_Context *db, const char *statement_name, const struct GNUNET_PQ_QueryParam *params, GNUNET_PQ_PostgresResultHandler rh, void *rh_cls)
Execute a named prepared statement that is a SELECT statement which may return multiple results in co...
Definition: pq_eval.c:209
struct GNUNET_PQ_ResultSpec GNUNET_PQ_result_spec_uint32(const char *name, uint32_t *u32)
uint32_t expected.
enum GNUNET_DB_QueryStatus GNUNET_PQ_eval_prepared_singleton_select(struct GNUNET_PQ_Context *db, const char *statement_name, const struct GNUNET_PQ_QueryParam *params, struct GNUNET_PQ_ResultSpec *rs)
Execute a named prepared statement that is a SELECT statement which must return a single result in co...
Definition: pq_eval.c:258
struct GNUNET_PQ_PreparedStatement GNUNET_PQ_make_prepare(const char *name, const char *sql, unsigned int num_args)
Create a struct GNUNET_PQ_PreparedStatement.
Definition: pq_prepare.c:38
struct GNUNET_PQ_ExecuteStatement GNUNET_PQ_make_execute(const char *sql)
Create a struct GNUNET_PQ_ExecuteStatement where errors are fatal.
Definition: pq_exec.c:36
struct GNUNET_PQ_QueryParam GNUNET_PQ_query_param_absolute_time(const struct GNUNET_TIME_Absolute *x)
Generate query parameter for an absolute time value.
#define GNUNET_PQ_result_spec_auto_from_type(name, dst)
We expect a fixed-size result, with size determined by the type of * dst
void GNUNET_PQ_disconnect(struct GNUNET_PQ_Context *db)
Disconnect from the database, destroying the prepared statements and releasing other associated resou...
Definition: pq_connect.c:509
struct GNUNET_PQ_ResultSpec GNUNET_PQ_result_spec_absolute_time(const char *name, struct GNUNET_TIME_Absolute *at)
Absolute time expected.
struct GNUNET_PQ_Context * GNUNET_PQ_connect_with_cfg(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *load_path_suffix, const struct GNUNET_PQ_ExecuteStatement *es, const struct GNUNET_PQ_PreparedStatement *ps)
Connect to a postgres database using the configuration option "CONFIG" in section.
Definition: pq_connect.c:450
struct GNUNET_PQ_ExecuteStatement GNUNET_PQ_make_try_execute(const char *sql)
Create a struct GNUNET_PQ_ExecuteStatement where errors should be tolerated.
Definition: pq_exec.c:55
#define GNUNET_PQ_query_param_end
End of query parameter specification.
Definition: gnunet_pq_lib.h:97
struct GNUNET_PQ_QueryParam GNUNET_PQ_query_param_uint32(const uint32_t *x)
Generate query parameter for an uint32_t in host byte order.
struct GNUNET_PQ_ResultSpec GNUNET_PQ_result_spec_variable_size(const char *name, void **dst, size_t *sptr)
Variable-size result expected.
void GNUNET_PQ_cleanup_result(struct GNUNET_PQ_ResultSpec *rs)
Free all memory that was allocated in rs during GNUNET_PQ_extract_result().
Definition: pq.c:122
#define GNUNET_PQ_PREPARED_STATEMENT_END
Terminator for prepared statement list.
enum GNUNET_GenericReturnValue GNUNET_PQ_extract_result(PGresult *result, struct GNUNET_PQ_ResultSpec *rs, int row)
Extract results from a query result according to the given specification.
Definition: pq.c:132
#define GNUNET_PQ_EXECUTE_STATEMENT_END
Terminator for executable statement list.
enum GNUNET_DB_QueryStatus GNUNET_PQ_eval_prepared_non_select(struct GNUNET_PQ_Context *db, const char *statement_name, const struct GNUNET_PQ_QueryParam *params)
Execute a named prepared statement that is NOT a SELECT statement in connection using the given param...
Definition: pq_eval.c:164
#define GNUNET_PQ_result_spec_end
End of result parameter specification.
enum GNUNET_GenericReturnValue(* GNUNET_DATACACHE_Iterator)(void *cls, const struct GNUNET_HashCode *key, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, struct GNUNET_TIME_Absolute exp, unsigned int path_info_len, const struct GNUNET_DHT_PathElement *path_info)
An iterator over a set of items stored in the datacache.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:110
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define OVERHEAD
Per-entry overhead estimate.
void * libgnunet_plugin_datacache_postgres_done(void *cls)
Exit point from the plugin.
static int init_connection(struct Plugin *plugin)
Get a database handle.
static unsigned int postgres_plugin_get(void *cls, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results for a particular key in the datastore.
static unsigned int postgres_plugin_get_closest(void *cls, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int num_results, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results that are "close" to a particular key in the datacache.
void * libgnunet_plugin_datacache_postgres_init(void *cls)
Entry point for the plugin.
static void extract_result_cb(void *cls, PGresult *result, unsigned int num_results)
Function to be called with the results of a SELECT statement that has returned num_results results.
static ssize_t postgres_plugin_put(void *cls, const struct GNUNET_HashCode *key, uint32_t prox, size_t data_size, const char *data, enum GNUNET_BLOCK_Type type, struct GNUNET_TIME_Absolute discard_time, unsigned int path_info_len, const struct GNUNET_DHT_PathElement *path_info)
Store an item in the datastore.
#define LOG(kind,...)
static int postgres_plugin_del(void *cls)
Delete the entry with the lowest expiration value from the datacache right now.
static void handle_results(void *cls, PGresult *result, unsigned int num_results)
Function to be called with the results of a SELECT statement that has returned num_results results.
Closure for extract_result_cb.
void * iter_cls
Closure for iter.
GNUNET_DATACACHE_Iterator iter
Function to call for each result found.
void * cls
Closure for all of the callbacks.
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration to use.
GNUNET_DATACACHE_DeleteNotifyCallback delete_notify
Function to call whenever the plugin needs to discard content that it was asked to store.
void * cls
Closure to use for callbacks.
struct returned by the initialization function of the plugin
void * cls
Closure to pass to all plugin functions.
A (signed) path tracking a block's flow through the DHT is represented by an array of path elements,...
A 512-bit hashcode.
Handle to Postgres database.
Definition: pq.h:36
Information needed to run a list of SQL statements using GNUNET_PQ_exec_statements().
Information needed to prepare a list of SQL statements using GNUNET_PQ_prepare_statements().
Description of a DB query parameter.
Definition: gnunet_pq_lib.h:65
Description of a DB result cell.
Time for absolute times used by GNUnet, in microseconds.
Closure for handle_results.
void * iter_cls
Closure for iter.
const struct GNUNET_HashCode * key
Key used.
GNUNET_DATACACHE_Iterator iter
Function to call on each result, may be NULL.
Handle for a plugin.
Definition: block.c:38
struct GNUNET_BLOCK_PluginFunctions * api
Plugin API.
Definition: block.c:47
unsigned int num_items
Number of key-value pairs in the database.
struct GNUNET_DATACACHE_PluginEnvironment * env
Our execution environment.
struct GNUNET_PQ_Context * dbh
Native Postgres database handle.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model