GNUnet  0.17.6
plugin_datacache_postgres.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet
3  Copyright (C) 2006, 2009, 2010, 2012, 2015, 2017, 2018, 2022 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
26 #include "platform.h"
27 #include "gnunet_util_lib.h"
28 #include "gnunet_pq_lib.h"
30 
31 #define LOG(kind, ...) GNUNET_log_from (kind, "datacache-postgres", __VA_ARGS__)
32 
36 #define OVERHEAD (sizeof(struct GNUNET_HashCode) + 24)
37 
41 struct Plugin
42 {
47 
52 
56  unsigned int num_items;
57 };
58 
59 
66 static enum GNUNET_GenericReturnValue
68 {
69  struct GNUNET_PQ_ExecuteStatement es[] = {
71  "CREATE TEMPORARY SEQUENCE IF NOT EXISTS gn180dc_oid_seq"),
72  GNUNET_PQ_make_execute ("CREATE TEMPORARY TABLE IF NOT EXISTS gn180dc ("
73  " oid OID NOT NULL DEFAULT nextval('gn180dc_oid_seq'),"
74  " type INT4 NOT NULL,"
75  " ro INT4 NOT NULL,"
76  " prox INT4 NOT NULL,"
77  " expiration_time INT8 NOT NULL,"
78  " key BYTEA NOT NULL CHECK(LENGTH(key)=64),"
79  " trunc BYTEA NOT NULL CHECK(LENGTH(trunc)=32),"
80  " value BYTEA NOT NULL,"
81  " path BYTEA DEFAULT NULL)"),
83  "ALTER SEQUENCE gnu011dc_oid_seq OWNED BY gn180dc.oid"),
85  "CREATE INDEX IF NOT EXISTS idx_oid ON gn180dc (oid)"),
87  "CREATE INDEX IF NOT EXISTS idx_key ON gn180dc (key)"),
89  "CREATE INDEX IF NOT EXISTS idx_dt ON gn180dc (expiration_time)"),
91  "ALTER TABLE gn180dc ALTER value SET STORAGE EXTERNAL"),
92  GNUNET_PQ_make_execute ("ALTER TABLE gn180dc ALTER key SET STORAGE PLAIN"),
94  };
95  struct GNUNET_PQ_PreparedStatement ps[] = {
96  GNUNET_PQ_make_prepare ("getkt",
97  "SELECT expiration_time,type,ro,value,trunc,path FROM gn180dc "
98  "WHERE key=$1 AND type=$2 AND expiration_time >= $3",
99  3),
100  GNUNET_PQ_make_prepare ("getk",
101  "SELECT expiration_time,type,ro,value,trunc,path FROM gn180dc "
102  "WHERE key=$1 AND expiration_time >= $2",
103  2),
104  GNUNET_PQ_make_prepare ("getex",
105  "SELECT LENGTH(value) AS len,oid,key FROM gn180dc"
106  " WHERE expiration_time < $1"
107  " ORDER BY expiration_time ASC LIMIT 1",
108  1),
109  GNUNET_PQ_make_prepare ("getm",
110  "SELECT LENGTH(value) AS len,oid,key FROM gn180dc"
111  " ORDER BY prox ASC, expiration_time ASC LIMIT 1",
112  0),
113  GNUNET_PQ_make_prepare ("get_closest",
114  "(SELECT expiration_time,type,ro,value,trunc,path,key FROM gn180dc"
115  " WHERE key >= $1"
116  " AND expiration_time >= $2"
117  " AND ( (type = $3) OR ( 0 = $3) )"
118  " ORDER BY key ASC"
119  " LIMIT $4)"
120  " UNION "
121  "(SELECT expiration_time,type,ro,value,trunc,path,key FROM gn180dc"
122  " WHERE key <= $1"
123  " AND expiration_time >= $2"
124  " AND ( (type = $3) OR ( 0 = $3) )"
125  " ORDER BY key DESC"
126  " LIMIT $4)",
127  4),
128  GNUNET_PQ_make_prepare ("delrow",
129  "DELETE FROM gn180dc WHERE oid=$1",
130  1),
131  GNUNET_PQ_make_prepare ("put",
132  "INSERT INTO gn180dc"
133  " (type, ro, prox, expiration_time, key, value, trunc, path) "
134  "VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
135  8),
137  };
138 
140  "datacache-postgres",
141  NULL,
142  es,
143  ps);
144  if (NULL == plugin->dbh)
145  return GNUNET_SYSERR;
146  return GNUNET_OK;
147 }
148 
149 
158 static ssize_t
160  uint32_t prox,
161  const struct GNUNET_DATACACHE_Block *block)
162 {
163  struct Plugin *plugin = cls;
164  uint32_t type32 = (uint32_t) block->type;
165  uint32_t ro32 = (uint32_t) block->type;
166  struct GNUNET_PQ_QueryParam params[] = {
173  block->data_size),
176  block->put_path_length
177  * sizeof(struct GNUNET_DHT_PathElement)),
179  };
181 
183  "put",
184  params);
185  if (0 > ret)
186  return -1;
187  plugin->num_items++;
188  return block->data_size + OVERHEAD;
189 }
190 
191 
196 {
201 
205  void *iter_cls;
206 
210  const struct GNUNET_HashCode *key;
211 };
212 
213 
223 static void
224 handle_results (void *cls,
225  PGresult *result,
226  unsigned int num_results)
227 {
228  struct HandleResultContext *hrc = cls;
229 
230  for (unsigned int i = 0; i < num_results; i++)
231  {
232  uint32_t type32;
233  uint32_t bro32;
234  void *data;
235  struct GNUNET_DATACACHE_Block block;
236  void *path;
237  size_t path_size;
238  struct GNUNET_PQ_ResultSpec rs[] = {
239  GNUNET_PQ_result_spec_absolute_time ("expiration_time",
240  &block.expiration_time),
242  &type32),
244  &bro32),
246  &data,
247  &block.data_size),
249  &block.trunc_peer),
251  &path,
252  &path_size),
254  };
255 
256  if (GNUNET_YES !=
258  rs,
259  i))
260  {
261  GNUNET_break (0);
262  return;
263  }
264  if (0 != (path_size % sizeof(struct GNUNET_DHT_PathElement)))
265  {
266  GNUNET_break (0);
267  path_size = 0;
268  path = NULL;
269  }
270  block.data = data;
271  block.put_path = path;
272  block.put_path_length
273  = path_size / sizeof (struct GNUNET_DHT_PathElement);
274  block.type = (enum GNUNET_BLOCK_Type) type32;
275  block.ro = (enum GNUNET_DHT_RouteOption) bro32;
276  block.key = *hrc->key;
278  "Found result of size %u bytes and type %u in database\n",
279  (unsigned int) block.data_size,
280  (unsigned int) block.type);
281  if ( (NULL != hrc->iter) &&
282  (GNUNET_SYSERR ==
283  hrc->iter (hrc->iter_cls,
284  &block)) )
285  {
287  "Ending iteration (client error)\n");
289  return;
290  }
292  }
293 }
294 
295 
307 static unsigned int
309  const struct GNUNET_HashCode *key,
310  enum GNUNET_BLOCK_Type type,
312  void *iter_cls)
313 {
314  struct Plugin *plugin = cls;
315  uint32_t type32 = (uint32_t) type;
316  struct GNUNET_TIME_Absolute now = { 0 };
317  struct GNUNET_PQ_QueryParam paramk[] = {
321  };
322  struct GNUNET_PQ_QueryParam paramkt[] = {
327  };
329  struct HandleResultContext hr_ctx;
330 
331  now = GNUNET_TIME_absolute_get ();
332  hr_ctx.iter = iter;
333  hr_ctx.iter_cls = iter_cls;
334  hr_ctx.key = key;
336  (0 == type) ? "getk" : "getkt",
337  (0 == type) ? paramk : paramkt,
339  &hr_ctx);
340  if (res < 0)
341  return 0;
342  return res;
343 }
344 
345 
353 static enum GNUNET_GenericReturnValue
354 postgres_plugin_del (void *cls)
355 {
356  struct Plugin *plugin = cls;
357  struct GNUNET_PQ_QueryParam pempty[] = {
359  };
360  uint32_t size;
361  uint32_t oid;
362  struct GNUNET_HashCode key;
363  struct GNUNET_PQ_ResultSpec rs[] = {
365  &size),
367  &oid),
369  &key),
371  };
373  struct GNUNET_PQ_QueryParam dparam[] = {
376  };
377  struct GNUNET_TIME_Absolute now;
378  struct GNUNET_PQ_QueryParam xparam[] = {
381  };
382 
383  now = GNUNET_TIME_absolute_get ();
385  "getex",
386  xparam,
387  rs);
388  if (0 >= res)
390  "getm",
391  pempty,
392  rs);
393  if (0 > res)
394  return GNUNET_SYSERR;
396  {
397  /* no result */
399  "Ending iteration (no more results)\n");
400  return 0;
401  }
403  "delrow",
404  dparam);
405  if (0 > res)
406  {
408  return GNUNET_SYSERR;
409  }
410  plugin->num_items--;
412  &key,
413  size + OVERHEAD);
415  return GNUNET_OK;
416 }
417 
418 
423 {
428 
432  void *iter_cls;
433 };
434 
435 
445 static void
446 extract_result_cb (void *cls,
447  PGresult *result,
448  unsigned int num_results)
449 {
450  struct ExtractResultContext *erc = cls;
451 
452  if (NULL == erc->iter)
453  return;
454  for (unsigned int i = 0; i < num_results; i++)
455  {
456  uint32_t type32;
457  uint32_t bro32;
458  struct GNUNET_DATACACHE_Block block;
459  void *data;
460  void *path;
461  size_t path_size;
462  struct GNUNET_PQ_ResultSpec rs[] = {
463  GNUNET_PQ_result_spec_absolute_time ("expiration_time",
464  &block.expiration_time),
466  &type32),
468  &bro32),
470  &data,
471  &block.data_size),
473  &block.trunc_peer),
475  &path,
476  &path_size),
478  &block.key),
480  };
481 
482  if (GNUNET_YES !=
484  rs,
485  i))
486  {
487  GNUNET_break (0);
488  return;
489  }
490  if (0 != (path_size % sizeof(struct GNUNET_DHT_PathElement)))
491  {
492  GNUNET_break (0);
493  path_size = 0;
494  path = NULL;
495  }
496  block.type = (enum GNUNET_BLOCK_Type) type32;
497  block.ro = (enum GNUNET_DHT_RouteOption) bro32;
498  block.data = data;
499  block.put_path = path;
500  block.put_path_length = path_size / sizeof (struct GNUNET_DHT_PathElement);
502  "Found result of size %u bytes and type %u in database\n",
503  (unsigned int) block.data_size,
504  (unsigned int) block.type);
505  if ( (NULL != erc->iter) &&
506  (GNUNET_SYSERR ==
507  erc->iter (erc->iter_cls,
508  &block)) )
509  {
511  "Ending iteration (client error)\n");
513  break;
514  }
516  }
517 }
518 
519 
534 static unsigned int
536  const struct GNUNET_HashCode *key,
537  enum GNUNET_BLOCK_Type type,
538  unsigned int num_results,
540  void *iter_cls)
541 {
542  struct Plugin *plugin = cls;
543  uint32_t num_results32 = (uint32_t) num_results;
544  uint32_t type32 = (uint32_t) type;
545  struct GNUNET_TIME_Absolute now;
546  struct GNUNET_PQ_QueryParam params[] = {
550  GNUNET_PQ_query_param_uint32 (&num_results32),
552  };
554  struct ExtractResultContext erc;
555 
556  erc.iter = iter;
557  erc.iter_cls = iter_cls;
558  now = GNUNET_TIME_absolute_get ();
560  "get_closest",
561  params,
563  &erc);
564  if (0 > res)
565  {
567  "Ending iteration (postgres error)\n");
568  return 0;
569  }
571  {
572  /* no result */
574  "Ending iteration (no more results)\n");
575  return 0;
576  }
577  return res;
578 }
579 
580 
587 void *
589 {
592  struct Plugin *plugin;
593 
594  plugin = GNUNET_new (struct Plugin);
595  plugin->env = env;
596 
598  {
600  return NULL;
601  }
602 
604  api->cls = plugin;
605  api->get = &postgres_plugin_get;
606  api->put = &postgres_plugin_put;
607  api->del = &postgres_plugin_del;
608  api->get_closest = &postgres_plugin_get_closest;
610  "Postgres datacache running\n");
611  return api;
612 }
613 
614 
621 void *
623 {
625  struct Plugin *plugin = api->cls;
626 
629  GNUNET_free (api);
630  return NULL;
631 }
632 
633 
634 /* end of plugin_datacache_postgres.c */
struct GNUNET_MQ_Envelope * env
Definition: 005.c:1
GNUNET_BLOCK_Type
WARNING: This header is generated! In order to add DHT block types, you must register them in GANA,...
static int ret
Return value of the commandline.
Definition: gnunet-abd.c:81
static int res
struct Plugin * plugin
The process handle to the testbed service.
struct GNUNET_HashCode key
The key used in the DHT.
uint32_t data
The data value.
static int result
Global testing status.
API for database backends for the datacache.
GNUNET_DB_QueryStatus
Status code returned from functions running database commands.
Definition: gnunet_db_lib.h:36
@ GNUNET_DB_STATUS_SUCCESS_NO_RESULTS
The transaction succeeded, but yielded zero results.
Definition: gnunet_db_lib.h:54
helper functions for Postgres DB interactions
struct GNUNET_PQ_QueryParam GNUNET_PQ_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.
#define GNUNET_PQ_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
enum GNUNET_DB_QueryStatus GNUNET_PQ_eval_prepared_multi_select(struct GNUNET_PQ_Context *db, const char *statement_name, const struct GNUNET_PQ_QueryParam *params, GNUNET_PQ_PostgresResultHandler rh, void *rh_cls)
Execute a named prepared statement that is a SELECT statement which may return multiple results in co...
Definition: pq_eval.c:209
struct GNUNET_PQ_ResultSpec GNUNET_PQ_result_spec_uint32(const char *name, uint32_t *u32)
uint32_t expected.
enum GNUNET_DB_QueryStatus GNUNET_PQ_eval_prepared_singleton_select(struct GNUNET_PQ_Context *db, const char *statement_name, const struct GNUNET_PQ_QueryParam *params, struct GNUNET_PQ_ResultSpec *rs)
Execute a named prepared statement that is a SELECT statement which must return a single result in co...
Definition: pq_eval.c:258
struct GNUNET_PQ_PreparedStatement GNUNET_PQ_make_prepare(const char *name, const char *sql, unsigned int num_args)
Create a struct GNUNET_PQ_PreparedStatement.
Definition: pq_prepare.c:38
struct GNUNET_PQ_ExecuteStatement GNUNET_PQ_make_execute(const char *sql)
Create a struct GNUNET_PQ_ExecuteStatement where errors are fatal.
Definition: pq_exec.c:36
struct GNUNET_PQ_QueryParam GNUNET_PQ_query_param_absolute_time(const struct GNUNET_TIME_Absolute *x)
Generate query parameter for an absolute time value.
#define GNUNET_PQ_result_spec_auto_from_type(name, dst)
We expect a fixed-size result, with size determined by the type of * dst
void GNUNET_PQ_disconnect(struct GNUNET_PQ_Context *db)
Disconnect from the database, destroying the prepared statements and releasing other associated resou...
Definition: pq_connect.c:526
struct GNUNET_PQ_ResultSpec GNUNET_PQ_result_spec_absolute_time(const char *name, struct GNUNET_TIME_Absolute *at)
Absolute time expected.
struct GNUNET_PQ_Context * GNUNET_PQ_connect_with_cfg(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *load_path_suffix, const struct GNUNET_PQ_ExecuteStatement *es, const struct GNUNET_PQ_PreparedStatement *ps)
Connect to a postgres database using the configuration option "CONFIG" in section.
Definition: pq_connect.c:461
struct GNUNET_PQ_ExecuteStatement GNUNET_PQ_make_try_execute(const char *sql)
Create a struct GNUNET_PQ_ExecuteStatement where errors should be tolerated.
Definition: pq_exec.c:55
#define GNUNET_PQ_query_param_end
End of query parameter specification.
Definition: gnunet_pq_lib.h:97
struct GNUNET_PQ_QueryParam GNUNET_PQ_query_param_uint32(const uint32_t *x)
Generate query parameter for an uint32_t in host byte order.
struct GNUNET_PQ_ResultSpec GNUNET_PQ_result_spec_variable_size(const char *name, void **dst, size_t *sptr)
Variable-size result expected.
void GNUNET_PQ_cleanup_result(struct GNUNET_PQ_ResultSpec *rs)
Free all memory that was allocated in rs during GNUNET_PQ_extract_result().
Definition: pq.c:122
#define GNUNET_PQ_PREPARED_STATEMENT_END
Terminator for prepared statement list.
enum GNUNET_GenericReturnValue GNUNET_PQ_extract_result(PGresult *result, struct GNUNET_PQ_ResultSpec *rs, int row)
Extract results from a query result according to the given specification.
Definition: pq.c:132
#define GNUNET_PQ_EXECUTE_STATEMENT_END
Terminator for executable statement list.
enum GNUNET_DB_QueryStatus GNUNET_PQ_eval_prepared_non_select(struct GNUNET_PQ_Context *db, const char *statement_name, const struct GNUNET_PQ_QueryParam *params)
Execute a named prepared statement that is NOT a SELECT statement in connection using the given param...
Definition: pq_eval.c:164
#define GNUNET_PQ_result_spec_end
End of result parameter specification.
enum GNUNET_GenericReturnValue(* GNUNET_DATACACHE_Iterator)(void *cls, const struct GNUNET_DATACACHE_Block *block)
An iterator over a set of items stored in the datacache.
GNUNET_DHT_RouteOption
Options for routing.
GNUNET_GenericReturnValue
Named constants for return values.
Definition: gnunet_common.h:96
@ GNUNET_OK
Definition: gnunet_common.h:99
@ GNUNET_YES
@ GNUNET_SYSERR
Definition: gnunet_common.h:97
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition: time.c:110
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define OVERHEAD
Per-entry overhead estimate.
static ssize_t postgres_plugin_put(void *cls, uint32_t prox, const struct GNUNET_DATACACHE_Block *block)
Store an item in the datastore.
void * libgnunet_plugin_datacache_postgres_done(void *cls)
Exit point from the plugin.
static enum GNUNET_GenericReturnValue postgres_plugin_del(void *cls)
Delete the entry with the lowest expiration value from the datacache right now.
static enum GNUNET_GenericReturnValue init_connection(struct Plugin *plugin)
Get a database handle.
static unsigned int postgres_plugin_get(void *cls, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results for a particular key in the datastore.
static unsigned int postgres_plugin_get_closest(void *cls, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int num_results, GNUNET_DATACACHE_Iterator iter, void *iter_cls)
Iterate over the results that are "close" to a particular key in the datacache.
void * libgnunet_plugin_datacache_postgres_init(void *cls)
Entry point for the plugin.
static void extract_result_cb(void *cls, PGresult *result, unsigned int num_results)
Function to be called with the results of a SELECT statement that has returned num_results results.
#define LOG(kind,...)
static void handle_results(void *cls, PGresult *result, unsigned int num_results)
Function to be called with the results of a SELECT statement that has returned num_results results.
Closure for extract_result_cb.
void * iter_cls
Closure for iter.
GNUNET_DATACACHE_Iterator iter
Function to call for each result found.
void * cls
Closure for all of the callbacks.
Information about a block stored in the datacache.
const struct GNUNET_DHT_PathElement * put_path
PUT path taken by the block, array of peer identities.
enum GNUNET_BLOCK_Type type
Type of the block.
const void * data
Actual block data.
enum GNUNET_DHT_RouteOption ro
Options for routing for the block.
struct GNUNET_PeerIdentity trunc_peer
If the path was truncated, this is the peer ID at which the path was truncated.
struct GNUNET_HashCode key
Key of the block.
size_t data_size
Number of bytes in data.
unsigned int put_path_length
Length of the put_path array.
struct GNUNET_TIME_Absolute expiration_time
When does the block expire?
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration to use.
GNUNET_DATACACHE_DeleteNotifyCallback delete_notify
Function to call whenever the plugin needs to discard content that it was asked to store.
void * cls
Closure to use for callbacks.
struct returned by the initialization function of the plugin
void * cls
Closure to pass to all plugin functions.
A (signed) path tracking a block's flow through the DHT is represented by an array of path elements,...
A 512-bit hashcode.
Handle to Postgres database.
Definition: pq.h:36
Information needed to run a list of SQL statements using GNUNET_PQ_exec_statements().
Information needed to prepare a list of SQL statements using GNUNET_PQ_prepare_statements().
Description of a DB query parameter.
Definition: gnunet_pq_lib.h:65
Description of a DB result cell.
Time for absolute times used by GNUnet, in microseconds.
Closure for handle_results.
void * iter_cls
Closure for iter.
const struct GNUNET_HashCode * key
Key used.
GNUNET_DATACACHE_Iterator iter
Function to call on each result, may be NULL.
Handle for a plugin.
Definition: block.c:38
struct GNUNET_BLOCK_PluginFunctions * api
Plugin API.
Definition: block.c:47
unsigned int num_items
Number of key-value pairs in the database.
struct GNUNET_DATACACHE_PluginEnvironment * env
Our execution environment.
struct GNUNET_PQ_Context * dbh
Native Postgres database handle.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model