38 #define MAX_ITEM_SIZE 65536 50 #define BUSY_TIMEOUT_MS 250 58 #define LOG_SQLITE(db, level, cmd) \ 61 GNUNET_log_from (level, \ 63 _ ("`%s' failed at %s:%d with error: %s\n"), \ 67 sqlite3_errmsg (db->dbh)); \ 76 #define LOG_SQLITE_MSG(db, msg, level, cmd) \ 79 GNUNET_log_from (level, \ 81 _ ("`%s' failed at %s:%d with error: %s\n"), \ 85 sqlite3_errmsg (db->dbh)); \ 86 GNUNET_asprintf (msg, \ 87 _ ("`%s' failed at %s:%u with error: %s"), \ 91 sqlite3_errmsg (db->dbh)); \ 118 sqlite3_stmt *
remove;
163 sqlite3_stmt *
get[8];
186 result = sqlite3_prepare_v2 (dbh,
190 (
const char **) &dummy);
193 "Prepared `%s' / %p: %d\n",
214 "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)",
221 "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)",
227 "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)",
234 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)",
240 "Failed to create indices: %s\n",
241 sqlite3_errmsg (dbh));
246 #define CHECK(a) GNUNET_break (a) 250 #define ENULL_DEFINED 1 254 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", e); \ 299 if (NULL != plugin->
env->duc)
306 if (SQLITE_OK != sqlite3_open (plugin->
fn, &plugin->
dbh))
310 _ (
"Unable to initialize SQLite: %s.\n"),
311 sqlite3_errmsg (plugin->
dbh));
316 sqlite3_exec (plugin->
dbh,
"PRAGMA temp_store=MEMORY", NULL, NULL,
ENULL));
319 sqlite3_exec (plugin->
dbh,
"PRAGMA synchronous=OFF", NULL, NULL,
ENULL));
320 CHECK (SQLITE_OK == sqlite3_exec (plugin->
dbh,
321 "PRAGMA legacy_file_format=OFF",
325 CHECK (SQLITE_OK == sqlite3_exec (plugin->
dbh,
326 "PRAGMA auto_vacuum=INCREMENTAL",
330 CHECK (SQLITE_OK == sqlite3_exec (plugin->
dbh,
331 "PRAGMA locking_mode=EXCLUSIVE",
337 sqlite3_exec (plugin->
dbh,
"PRAGMA page_size=4096", NULL, NULL,
ENULL));
345 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn091'",
352 if ((SQLITE_DONE == sqlite3_step (stmt)) &&
353 (SQLITE_OK != sqlite3_exec (plugin->
dbh,
354 "CREATE TABLE gn091 (" 355 " repl INT4 NOT NULL DEFAULT 0," 356 " type INT4 NOT NULL DEFAULT 0," 357 " prio INT4 NOT NULL DEFAULT 0," 358 " anonLevel INT4 NOT NULL DEFAULT 0," 359 " expire INT8 NOT NULL DEFAULT 0," 360 " rvalue INT8 NOT NULL," 361 " hash TEXT NOT NULL DEFAULT ''," 362 " vhash TEXT NOT NULL DEFAULT ''," 363 " value BLOB NOT NULL DEFAULT '')",
369 sqlite3_finalize (stmt);
372 sqlite3_finalize (stmt);
375 #define RESULT_COLUMNS \ 376 "repl, type, prio, anonLevel, expire, hash, value, _ROWID_" 380 "SET prio = prio + ?, " 382 "expire = MAX(expire, ?) " 383 "WHERE hash = ? AND vhash = ?",
387 "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
393 " NOT EXISTS (SELECT 1 FROM gn091 " 394 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) " 395 "ORDER BY rvalue ASC LIMIT 1",
398 "SELECT MAX(repl) FROM gn091",
403 "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) " 404 "ORDER BY expire ASC LIMIT 1",
408 "WHERE _ROWID_ >= ? AND " 411 "ORDER BY _ROWID_ ASC LIMIT 1",
415 "INSERT INTO gn091 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) " 416 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
420 "WHERE _ROWID_ >= ?1 " 421 "ORDER BY _ROWID_ ASC LIMIT 1",
425 "WHERE _ROWID_ >= ?1 AND " 427 "ORDER BY _ROWID_ ASC LIMIT 1",
431 "WHERE _ROWID_ >= ?1 AND " 433 "ORDER BY _ROWID_ ASC LIMIT 1",
437 "WHERE _ROWID_ >= ?1 AND " 440 "ORDER BY _ROWID_ ASC LIMIT 1",
444 "WHERE _ROWID_ >= ?1 AND " 446 "ORDER BY _ROWID_ ASC LIMIT 1",
450 "WHERE _ROWID_ >= ?1 AND " 453 "ORDER BY _ROWID_ ASC LIMIT 1",
457 "WHERE _ROWID_ >= ?1 AND " 460 "ORDER BY _ROWID_ ASC LIMIT 1",
464 "WHERE _ROWID_ >= ?1 AND " 468 "ORDER BY _ROWID_ ASC LIMIT 1",
471 "DELETE FROM gn091 WHERE _ROWID_ = ?",
475 "WHERE hash = ? AND " 498 #if SQLITE_VERSION_NUMBER >= 3007000 502 if (NULL != plugin->
remove)
503 sqlite3_finalize (plugin->
remove);
504 if (NULL != plugin->
delRow)
505 sqlite3_finalize (plugin->
delRow);
506 if (NULL != plugin->
update)
507 sqlite3_finalize (plugin->
update);
509 sqlite3_finalize (plugin->
updRepl);
511 sqlite3_finalize (plugin->
selRepl);
513 sqlite3_finalize (plugin->
maxRepl);
515 sqlite3_finalize (plugin->
selExpi);
520 for (
int i = 0; i < 8; ++i)
521 if (NULL != plugin->
get[i])
522 sqlite3_finalize (plugin->
get[i]);
523 result = sqlite3_close (plugin->
dbh);
524 #if SQLITE_VERSION_NUMBER >= 3007000 525 if (result == SQLITE_BUSY)
531 "Tried to close sqlite without finalizing all prepared statements.\n"));
532 stmt = sqlite3_next_stmt (plugin->
dbh, NULL);
537 "Closing statement %p\n",
539 result = sqlite3_finalize (stmt);
540 if (result != SQLITE_OK)
543 "Failed to close statement %p: %d\n",
546 stmt = sqlite3_next_stmt (plugin->
dbh, NULL);
548 result = sqlite3_close (plugin->
dbh);
551 if (SQLITE_OK != result)
572 if (SQLITE_DONE != sqlite3_step (plugin->
delRow))
633 cont (cont_cls, key, size,
GNUNET_SYSERR,
_ (
"sqlite bind failure"));
636 if (SQLITE_DONE != sqlite3_step (plugin->
update))
646 int changes = sqlite3_changes (plugin->
dbh);
650 cont (cont_cls, key, size,
GNUNET_NO, NULL);
656 uint32_t type32 = (uint32_t) type;
680 "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n",
695 n = sqlite3_step (stmt);
699 if (NULL != plugin->
env->duc)
704 "Stored new entry (%u bytes)\n",
731 cont (cont_cls, key, size, ret, msg);
773 n = sqlite3_step (stmt);
784 "Found reply in database with expiration %s\n",
786 ret = proc (proc_cls,
799 (NULL != plugin->
env->duc))
815 if (SQLITE_OK != sqlite3_reset (stmt))
850 uint32_t type32 =
type;
891 int use_rvalue = random;
892 uint32_t type32 = (uint32_t) type;
894 int use_key = NULL !=
key;
895 sqlite3_stmt *stmt = plugin->
get[use_rvalue * 4 + use_key * 2 + use_type];
1043 "Getting random block based on replication order.\n");
1044 if (SQLITE_ROW != sqlite3_step (plugin->
maxRepl))
1051 repl = sqlite3_column_int (plugin->
maxRepl, 0);
1070 if (SQLITE_DONE != sqlite3_step (plugin->
updRepl))
1107 "Getting random block based on expiration and priority order.\n");
1137 if (SQLITE_OK !=
sq_prepare (plugin->
dbh,
"SELECT hash FROM gn091", &stmt))
1142 proc (proc_cls, NULL, 0);
1145 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1148 proc (proc_cls, &key, 1);
1152 if (SQLITE_DONE != ret)
1154 sqlite3_finalize (stmt);
1155 proc (proc_cls, NULL, 0);
1202 if (SQLITE_DONE != sqlite3_step (plugin->
remove))
1208 cont (cont_cls, key, size,
GNUNET_SYSERR,
"sqlite3_step failed");
1211 int changes = sqlite3_changes (plugin->
dbh);
1215 cont (cont_cls, key, size,
GNUNET_NO, NULL);
1218 if (NULL != plugin->
env->duc)
1221 cont (cont_cls, key, size,
GNUNET_OK, NULL);
1244 if (NULL == estimate)
1246 if (SQLITE_VERSION_NUMBER < 3006000)
1251 _ (
"sqlite version to old to determine size, assuming zero\n"));
1255 CHECK (SQLITE_OK == sqlite3_exec (plugin->
dbh,
"VACUUM", NULL, NULL,
ENULL));
1256 CHECK (SQLITE_OK == sqlite3_exec (plugin->
dbh,
1257 "PRAGMA auto_vacuum=INCREMENTAL",
1262 if (SQLITE_ROW == sqlite3_step (stmt))
1263 pages = sqlite3_column_int64 (stmt, 0);
1266 sqlite3_finalize (stmt);
1268 CHECK (SQLITE_ROW == sqlite3_step (stmt));
1269 page_size = sqlite3_column_int64 (stmt, 0);
1270 sqlite3_finalize (stmt);
1274 "Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1275 (
unsigned long long) pages,
1276 (
unsigned long long) page_size);
1277 *estimate = pages * page_size;
1290 static struct Plugin plugin;
1294 if (NULL != plugin.
env)
1296 memset (&plugin, 0,
sizeof(
struct Plugin));
1316 _ (
"Sqlite database running\n"));
1336 "sqlite plugin is done\n");
1345 if (0 != unlink (fn))
PluginGetKeys get_keys
Iterate over all keys in the database.
PluginPut put
Function to store an item in the datastore.
static int database_setup(const struct GNUNET_CONFIGURATION_Handle *cfg, struct Plugin *plugin)
Initialize the database connections and associated data structures (create tables and indices as need...
int(* PluginDatumProcessor)(void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
An processor over a set of items stored in the datastore.
void * proc_cls
Closure for proc.
PluginGetType get_zero_anonymity
Get datum (of the specified type) with anonymity level zero.
static void sqlite_plugin_put(void *cls, const struct GNUNET_HashCode *key, bool absent, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, PluginPutCont cont, void *cont_cls)
Store an item in the datastore.
Description of a DB result cell.
struct GNUNET_MessageHeader * msg
#define BUSY_TIMEOUT_MS
After how many ms "busy" should a DB operation fail for good? A low value makes sure that we are more...
sqlite3_stmt * update
Precompiled SQL for update.
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_uint32(uint32_t *u32)
uint32_t expected.
int have_uid
Yes if UID was set.
Any type of block, used as a wildcard when searching.
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration to use.
uint64_t GNUNET_CRYPTO_random_u64(enum GNUNET_CRYPTO_Quality mode, uint64_t max)
Random on unsigned 64-bit values.
sqlite3_stmt * updRepl
Precompiled SQL for replication decrement.
static int sq_prepare(sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt)
Prepare a SQL statement.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_SQ_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
PluginRemoveKey remove_key
Function to remove an item from the database.
sqlite3_stmt * delRow
Precompiled SQL for deletion.
const struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
static unsigned int replication
PluginEstimateSize estimate_size
Calculate the current on-disk size of the SQ store.
static int ret
Return value of the commandline.
static struct Experiment * e
#define GNUNET_SQ_result_spec_end
End of result parameter specification.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
#define GNUNET_new(type)
Allocate a struct or union of the given type.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_filename(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be the name of a file or directory.
#define GNUNET_SQ_result_spec_auto_from_type(dst)
We expect a fixed-size result, with size determined by the type of * dst
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
void(* PluginPutCont)(void *cls, const struct GNUNET_HashCode *key, uint32_t size, int status, const char *msg)
Put continuation.
static void sqlite_plugin_get_replication(void *cls, PluginDatumProcessor proc, void *proc_cls)
Get a random item for replication.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur...
void * libgnunet_plugin_datastore_sqlite_init(void *cls)
Entry point for the plugin.
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_absolute_time(struct GNUNET_TIME_Absolute *at)
Absolute time expected.
struct GNUNET_BLOCK_PluginFunctions * api
Plugin API.
#define LOG_SQLITE_MSG(db, msg, level, cmd)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' on file 'file...
sqlite3_stmt * insertContent
Precompiled SQL for insertion.
sqlite3_stmt * get[8]
Precompiled SQL for selection.
static void sqlite_plugin_get_keys(void *cls, PluginKeyProcessor proc, void *proc_cls)
Get all of the keys in the datastore.
int GNUNET_SQ_extract_result(sqlite3_stmt *result, struct GNUNET_SQ_ResultSpec *rs)
Extract results from a query result according to the given specification.
PluginGetRandom get_replication
Function to get a random item with high replication score from the database, lowering the item's repl...
static void database_shutdown(struct Plugin *plugin)
Shutdown database connection and associate data structures.
PluginDrop drop
Delete the database.
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_uint32(const uint32_t *x)
Generate query parameter for an uint32_t in host byte order.
void * cls
Closure for conv and cleaner.
void GNUNET_SQ_cleanup_result(struct GNUNET_SQ_ResultSpec *rs)
Free all memory that was allocated in rs during GNUNET_SQ_extract_result().
static int repl_proc(void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Wrapper for the processor for sqlite_plugin_get_replication().
static void sqlite_plugin_drop(void *cls)
Drop database.
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
static char * value
Value of the record to add/remove.
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
void GNUNET_CRYPTO_hash(const void *block, size_t size, struct GNUNET_HashCode *ret)
Compute hash of a given block.
void(* PluginKeyProcessor)(void *cls, const struct GNUNET_HashCode *key, unsigned int count)
An processor over a set of keys stored in the datastore.
int drop_on_shutdown
Should the database be dropped on shutdown?
PluginGetRandom get_expiration
Function to get a random expired item or, if none are expired, either the oldest entry or one with a ...
struct GNUNET_DATACACHE_PluginEnvironment * env
Our execution environment.
Each plugin is required to return a pointer to a struct of this type as the return value from its ent...
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
sqlite3_stmt * selZeroAnon
Precompiled SQL for expiration selection.
#define MAX_ITEM_SIZE
We allocate items on the stack at times.
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_variable_size(void **dst, size_t *sptr)
Variable-size result expected.
static int result
Global testing status.
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_absolute_time(const struct GNUNET_TIME_Absolute *x)
Generate query parameter for an absolute time value.
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
sqlite3_stmt * selRepl
Precompiled SQL for replication selection.
#define GNUNET_DATASTORE_ENTRY_OVERHEAD
How many bytes of overhead will we assume per entry in any DB (for reservations)? ...
static char * expiration
Credential TTL.
int GNUNET_SQ_bind(sqlite3_stmt *stmt, const struct GNUNET_SQ_QueryParam *params)
Execute binding operations for a prepared statement.
static char * plugin
Solver plugin name as string.
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Context for repl_proc() function.
sqlite3_stmt * remove
Precompiled SQL for remove_key.
static void sqlite_plugin_estimate_size(void *cls, unsigned long long *estimate)
Get an estimate of how much space the database is currently using.
helper functions for Sqlite3 DB interactions
void(* PluginRemoveCont)(void *cls, const struct GNUNET_HashCode *key, uint32_t size, int status, const char *msg)
Remove continuation.
#define LOG_SQLITE(db, level, cmd)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' on file 'file...
Description of a DB query parameter.
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_uint64(uint64_t *u64)
uint64_t expected.
static void create_indices(sqlite3 *dbh)
Create our database indices.
void * cls
Closure to use for all of the following callbacks (except "next_request").
char * fn
Filename used for the DB.
static struct in_addr dummy
Target "dummy" address of the packet we pretend to respond to.
void GNUNET_SQ_reset(sqlite3 *dbh, sqlite3_stmt *stmt)
Reset stmt and log error.
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_uint64(const uint64_t *x)
Generate query parameter for an uint16_t in host byte order.
unsigned long long size
Size of all values we're storing.
struct GNUNET_PQ_Context * dbh
Native Postgres database handle.
static void sqlite_plugin_get_zero_anonymity(void *cls, uint64_t next_uid, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls)
Select a subset of the items in the datastore and call the given processor for the item...
PluginDatumProcessor proc
Function to call for the result (or the NULL).
static void sqlite_plugin_get_expiration(void *cls, PluginDatumProcessor proc, void *proc_cls)
Get a random item that has expired or has low priority.
#define GNUNET_log(kind,...)
static int delete_by_rowid(struct Plugin *plugin, uint64_t rid)
Delete the database entry with the given row identifier.
sqlite3_stmt * maxRepl
Get maximum repl value in database.
sqlite3_stmt * selExpi
Precompiled SQL for expiration selection.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_remaining(struct GNUNET_TIME_Absolute future)
Given a timestamp in the future, how much time remains until then?
void * cls
Closure to use for callbacks.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.
Time for absolute times used by GNUnet, in microseconds.
static unsigned int anonymity
static void execute_get(struct Plugin *plugin, sqlite3_stmt *stmt, PluginDatumProcessor proc, void *proc_cls)
Execute statement that gets a row and call the callback with the result.
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_create_for_file(const char *filename)
Create the directory structure for storing a file.
#define GNUNET_SQ_query_param_end
End of query parameter specification.
const char * GNUNET_STRINGS_absolute_time_to_string(struct GNUNET_TIME_Absolute t)
Like asctime, except for GNUnet time.
#define GNUNET_log_from(kind, comp,...)
PluginGetKey get_key
Get a particular datum matching a given hash from the datastore.
uint32_t data
The data value.
static void sqlite_plugin_get_key(void *cls, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls)
Get results for a particular key in the datastore.
void * libgnunet_plugin_datastore_sqlite_done(void *cls)
Exit point from the plugin.
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory)...
static void sqlite_plugin_remove_key(void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, PluginRemoveCont cont, void *cont_cls)
Remove a particular key in the datastore.
const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration to use.
#define GNUNET_free(ptr)
Wrapper around free.