GNUnet 0.25.1-10-g7666a6f5a
 
Loading...
Searching...
No Matches
plugin_datastore_sqlite.c
Go to the documentation of this file.
1/*
2 * This file is part of GNUnet
3 * Copyright (C) 2009, 2011, 2017 GNUnet e.V.
4 *
5 * GNUnet is free software: you can redistribute it and/or modify it
6 * under the terms of the GNU Affero General Public License as published
7 * by the Free Software Foundation, either version 3 of the License,
8 * or (at your option) any later version.
9 *
10 * GNUnet is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * Affero General Public License for more details.
14 *
15 * You should have received a copy of the GNU Affero General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17
18 SPDX-License-Identifier: AGPL3.0-or-later
19 */
20
27#include "platform.h"
29#include "gnunet_sq_lib.h"
30#include <sqlite3.h>
31
32
38#define MAX_ITEM_SIZE 65536
39
50#define BUSY_TIMEOUT_MS 250
51
52
58#define LOG_SQLITE(db, level, cmd) \
59 do \
60 { \
61 GNUNET_log_from (level, \
62 "sqlite", \
63 _ ("`%s' failed at %s:%d with error: %s\n"), \
64 cmd, \
65 __FILE__, \
66 __LINE__, \
67 sqlite3_errmsg (db->dbh)); \
68 } while (0)
69
70
76#define LOG_SQLITE_MSG(db, msg, level, cmd) \
77 do \
78 { \
79 GNUNET_log_from (level, \
80 "sqlite", \
81 _ ("`%s' failed at %s:%d with error: %s\n"), \
82 cmd, \
83 __FILE__, \
84 __LINE__, \
85 sqlite3_errmsg (db->dbh)); \
86 GNUNET_asprintf (msg, \
87 _ ("`%s' failed at %s:%u with error: %s"), \
88 cmd, \
89 __FILE__, \
90 __LINE__, \
91 sqlite3_errmsg (db->dbh)); \
92 } while (0)
93
94
98struct Plugin
99{
104
108 char *fn;
109
113 sqlite3 *dbh;
114
118 sqlite3_stmt *remove;
119
123 sqlite3_stmt *delRow;
124
128 sqlite3_stmt *update;
129
133 sqlite3_stmt *maxRepl;
134
138 sqlite3_stmt *updRepl;
139
143 sqlite3_stmt *selRepl;
144
148 sqlite3_stmt *selExpi;
149
153 sqlite3_stmt *selZeroAnon;
154
158 sqlite3_stmt *insertContent;
159
163 sqlite3_stmt *get[8];
164
169};
170
171
180static int
181sq_prepare (sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt)
182{
183 char *dummy;
184 int result;
185
186 result = sqlite3_prepare_v2 (dbh,
187 zSql,
188 strlen (zSql),
189 ppStmt,
190 (const char **) &dummy);
192 "sqlite",
193 "Prepared `%s' / %p: %d\n",
194 zSql,
195 *ppStmt,
196 result);
197 return result;
198}
199
200
206static void
207create_indices (sqlite3 *dbh)
208{
209 /* create indices */
210 if (
211 0 !=
212 (SQLITE_OK !=
213 sqlite3_exec (dbh,
214 "CREATE INDEX IF NOT EXISTS idx_hash ON gn091 (hash)",
215 NULL,
216 NULL,
217 NULL))
218 + (SQLITE_OK !=
219 sqlite3_exec (
220 dbh,
221 "CREATE INDEX IF NOT EXISTS idx_anon_type ON gn091 (anonLevel ASC,type)",
222 NULL,
223 NULL,
224 NULL))
225 + (SQLITE_OK !=
226 sqlite3_exec (dbh,
227 "CREATE INDEX IF NOT EXISTS idx_expire ON gn091 (expire ASC)",
228 NULL,
229 NULL,
230 NULL))
231 + (SQLITE_OK !=
232 sqlite3_exec (
233 dbh,
234 "CREATE INDEX IF NOT EXISTS idx_repl_rvalue ON gn091 (repl,rvalue)",
235 NULL,
236 NULL,
237 NULL)))
239 "sqlite",
240 "Failed to create indices: %s\n",
241 sqlite3_errmsg (dbh));
242}
243
244
245#if 0
246#define CHECK(a) GNUNET_break (a)
247#define ENULL NULL
248#else
249#define ENULL &e
250#define ENULL_DEFINED 1
251#define CHECK(a) \
252 if (! (a)) \
253 { \
254 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "%s\n", e); \
255 sqlite3_free (e); \
256 e = NULL; \
257 }
258#endif
259
260
270static int
272 struct Plugin *plugin)
273{
274 sqlite3_stmt *stmt;
275 char *afsdir;
276
277#if ENULL_DEFINED
278 char *e;
279#endif
280
282 "datastore-sqlite",
283 "FILENAME",
284 &afsdir))
285 {
287 "datastore-sqlite",
288 "FILENAME");
289 return GNUNET_SYSERR;
290 }
291 if (GNUNET_OK != GNUNET_DISK_file_test (afsdir))
292 {
294 {
295 GNUNET_break (0);
296 GNUNET_free (afsdir);
297 return GNUNET_SYSERR;
298 }
299 /* database is new or got deleted, reset payload to zero! */
300 if (NULL != plugin->env->duc)
301 plugin->env->duc (plugin->env->cls, 0);
302 }
303 /* afsdir should be UTF-8-encoded. If it isn't, it's a bug */
304 plugin->fn = afsdir;
305
306 /* Open database and precompile statements */
307 if (SQLITE_OK != sqlite3_open (plugin->fn, &plugin->dbh))
308 {
310 "sqlite",
311 _ ("Unable to initialize SQLite: %s.\n"),
312 sqlite3_errmsg (plugin->dbh));
313 return GNUNET_SYSERR;
314 }
315 CHECK (
316 SQLITE_OK ==
317 sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, ENULL));
318 CHECK (
319 SQLITE_OK ==
320 sqlite3_exec (plugin->dbh, "PRAGMA synchronous=OFF", NULL, NULL, ENULL));
321 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
322 "PRAGMA legacy_file_format=OFF",
323 NULL,
324 NULL,
325 ENULL));
326 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
327 "PRAGMA auto_vacuum=INCREMENTAL",
328 NULL,
329 NULL,
330 ENULL));
331 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
332 "PRAGMA locking_mode=EXCLUSIVE",
333 NULL,
334 NULL,
335 ENULL));
336 CHECK (
337 SQLITE_OK ==
338 sqlite3_exec (plugin->dbh, "PRAGMA page_size=4096", NULL, NULL, ENULL));
339
340 CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
341
342
343 /* We have to do it here, because otherwise precompiling SQL might fail */
344 CHECK (SQLITE_OK ==
345 sq_prepare (plugin->dbh,
346 "SELECT 1 FROM sqlite_master WHERE tbl_name = 'gn091'",
347 &stmt));
348
349 /* FIXME: SQLite does not have unsigned integers! This is ok for the type column because
350 * we only test equality on it and can cast it to/from uint32_t. For repl, prio, and anonLevel
351 * we do math or inequality tests, so we can't handle the entire range of uint32_t.
352 * This will also cause problems for expiration times after 294247-01-10-04:00:54 UTC.
353 */if ((SQLITE_DONE == sqlite3_step (stmt)) &&
354 (SQLITE_OK != sqlite3_exec (plugin->dbh,
355 "CREATE TABLE gn091 ("
356 " repl INT4 NOT NULL DEFAULT 0,"
357 " type INT4 NOT NULL DEFAULT 0,"
358 " prio INT4 NOT NULL DEFAULT 0,"
359 " anonLevel INT4 NOT NULL DEFAULT 0,"
360 " expire INT8 NOT NULL DEFAULT 0,"
361 " rvalue INT8 NOT NULL,"
362 " hash TEXT NOT NULL DEFAULT '',"
363 " vhash TEXT NOT NULL DEFAULT '',"
364 " value BLOB NOT NULL DEFAULT '')",
365 NULL,
366 NULL,
367 NULL)))
368 {
369 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
370 sqlite3_finalize (stmt);
371 return GNUNET_SYSERR;
372 }
373 sqlite3_finalize (stmt);
374 create_indices (plugin->dbh);
375
376#define RESULT_COLUMNS \
377 "repl, type, prio, anonLevel, expire, hash, value, _ROWID_"
378 if (
379 (SQLITE_OK != sq_prepare (plugin->dbh,
380 "UPDATE gn091 "
381 "SET prio = prio + ?, "
382 "repl = repl + ?, "
383 "expire = MAX(expire, ?) "
384 "WHERE hash = ? AND vhash = ?",
385 &plugin->update)) ||
386 (SQLITE_OK != sq_prepare (plugin->dbh,
387 "UPDATE gn091 "
388 "SET repl = MAX (0, repl - 1) WHERE _ROWID_ = ?",
389 &plugin->updRepl)) ||
390 (SQLITE_OK != sq_prepare (plugin->dbh,
391 "SELECT " RESULT_COLUMNS " FROM gn091 "
392 "WHERE repl=?2 AND "
393 " (rvalue>=?1 OR "
394 " NOT EXISTS (SELECT 1 FROM gn091 "
395 "WHERE repl=?2 AND rvalue>=?1 LIMIT 1) ) "
396 "ORDER BY rvalue ASC LIMIT 1",
397 &plugin->selRepl)) ||
398 (SQLITE_OK != sq_prepare (plugin->dbh,
399 "SELECT MAX(repl) FROM gn091",
400 &plugin->maxRepl)) ||
401 (SQLITE_OK !=
402 sq_prepare (plugin->dbh,
403 "SELECT " RESULT_COLUMNS " FROM gn091 "
404 "WHERE NOT EXISTS (SELECT 1 FROM gn091 WHERE expire < ?1 LIMIT 1) OR (expire < ?1) "
405 "ORDER BY expire ASC LIMIT 1",
406 &plugin->selExpi)) ||
407 (SQLITE_OK != sq_prepare (plugin->dbh,
408 "SELECT " RESULT_COLUMNS " FROM gn091 "
409 "WHERE _ROWID_ >= ? AND "
410 "anonLevel = 0 AND "
411 "type = ? "
412 "ORDER BY _ROWID_ ASC LIMIT 1",
413 &plugin->selZeroAnon)) ||
414 (SQLITE_OK !=
415 sq_prepare (plugin->dbh,
416 "INSERT INTO gn091 (repl, type, prio, anonLevel, expire, rvalue, hash, vhash, value) "
417 "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
418 &plugin->insertContent)) ||
419 (SQLITE_OK != sq_prepare (plugin->dbh,
420 "SELECT " RESULT_COLUMNS " FROM gn091 "
421 "WHERE _ROWID_ >= ?1 "
422 "ORDER BY _ROWID_ ASC LIMIT 1",
423 &plugin->get[0])) ||
424 (SQLITE_OK != sq_prepare (plugin->dbh,
425 "SELECT " RESULT_COLUMNS " FROM gn091 "
426 "WHERE _ROWID_ >= ?1 AND "
427 "type = ?4 "
428 "ORDER BY _ROWID_ ASC LIMIT 1",
429 &plugin->get[1])) ||
430 (SQLITE_OK != sq_prepare (plugin->dbh,
431 "SELECT " RESULT_COLUMNS " FROM gn091 "
432 "WHERE _ROWID_ >= ?1 AND "
433 "hash = ?3 "
434 "ORDER BY _ROWID_ ASC LIMIT 1",
435 &plugin->get[2])) ||
436 (SQLITE_OK != sq_prepare (plugin->dbh,
437 "SELECT " RESULT_COLUMNS " FROM gn091 "
438 "WHERE _ROWID_ >= ?1 AND "
439 "hash = ?3 AND "
440 "type = ?4 "
441 "ORDER BY _ROWID_ ASC LIMIT 1",
442 &plugin->get[3])) ||
443 (SQLITE_OK != sq_prepare (plugin->dbh,
444 "SELECT " RESULT_COLUMNS " FROM gn091 "
445 "WHERE _ROWID_ >= ?1 AND "
446 "rvalue >= ?2 "
447 "ORDER BY _ROWID_ ASC LIMIT 1",
448 &plugin->get[4])) ||
449 (SQLITE_OK != sq_prepare (plugin->dbh,
450 "SELECT " RESULT_COLUMNS " FROM gn091 "
451 "WHERE _ROWID_ >= ?1 AND "
452 "rvalue >= ?2 AND "
453 "type = ?4 "
454 "ORDER BY _ROWID_ ASC LIMIT 1",
455 &plugin->get[5])) ||
456 (SQLITE_OK != sq_prepare (plugin->dbh,
457 "SELECT " RESULT_COLUMNS " FROM gn091 "
458 "WHERE _ROWID_ >= ?1 AND "
459 "rvalue >= ?2 AND "
460 "hash = ?3 "
461 "ORDER BY _ROWID_ ASC LIMIT 1",
462 &plugin->get[6])) ||
463 (SQLITE_OK != sq_prepare (plugin->dbh,
464 "SELECT " RESULT_COLUMNS " FROM gn091 "
465 "WHERE _ROWID_ >= ?1 AND "
466 "rvalue >= ?2 AND "
467 "hash = ?3 AND "
468 "type = ?4 "
469 "ORDER BY _ROWID_ ASC LIMIT 1",
470 &plugin->get[7])) ||
471 (SQLITE_OK != sq_prepare (plugin->dbh,
472 "DELETE FROM gn091 WHERE _ROWID_ = ?",
473 &plugin->delRow)) ||
474 (SQLITE_OK != sq_prepare (plugin->dbh,
475 "DELETE FROM gn091 "
476 "WHERE hash = ? AND "
477 "value = ? ",
478 &plugin->remove)) ||
479 false)
480 {
481 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "precompiling");
482 return GNUNET_SYSERR;
483 }
484 return GNUNET_OK;
485}
486
487
494static void
496{
497 int result;
498
499#if SQLITE_VERSION_NUMBER >= 3007000
500 sqlite3_stmt *stmt;
501#endif
502
503 if (NULL != plugin->remove)
504 sqlite3_finalize (plugin->remove);
505 if (NULL != plugin->delRow)
506 sqlite3_finalize (plugin->delRow);
507 if (NULL != plugin->update)
508 sqlite3_finalize (plugin->update);
509 if (NULL != plugin->updRepl)
510 sqlite3_finalize (plugin->updRepl);
511 if (NULL != plugin->selRepl)
512 sqlite3_finalize (plugin->selRepl);
513 if (NULL != plugin->maxRepl)
514 sqlite3_finalize (plugin->maxRepl);
515 if (NULL != plugin->selExpi)
516 sqlite3_finalize (plugin->selExpi);
517 if (NULL != plugin->selZeroAnon)
518 sqlite3_finalize (plugin->selZeroAnon);
519 if (NULL != plugin->insertContent)
520 sqlite3_finalize (plugin->insertContent);
521 for (int i = 0; i < 8; ++i)
522 if (NULL != plugin->get[i])
523 sqlite3_finalize (plugin->get[i]);
524 result = sqlite3_close (plugin->dbh);
525#if SQLITE_VERSION_NUMBER >= 3007000
526 if (result == SQLITE_BUSY)
527 {
530 "sqlite",
531 _ (
532 "Tried to close sqlite without finalizing all prepared statements.\n"));
533 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
534 while (NULL != stmt)
535 {
537 "sqlite",
538 "Closing statement %p\n",
539 stmt);
540 result = sqlite3_finalize (stmt);
541 if (result != SQLITE_OK)
543 "sqlite",
544 "Failed to close statement %p: %d\n",
545 stmt,
546 result);
547 stmt = sqlite3_next_stmt (plugin->dbh, NULL);
548 }
549 result = sqlite3_close (plugin->dbh);
550 }
551#endif
552 if (SQLITE_OK != result)
553 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_close");
554 GNUNET_free (plugin->fn);
555}
556
557
565static int
566delete_by_rowid (struct Plugin *plugin, uint64_t rid)
567{
568 struct GNUNET_SQ_QueryParam params[] = { GNUNET_SQ_query_param_uint64 (&rid),
570
571 if (GNUNET_OK != GNUNET_SQ_bind (plugin->delRow, params))
572 return GNUNET_SYSERR;
573 if (SQLITE_DONE != sqlite3_step (plugin->delRow))
574 {
577 "sqlite3_step");
578 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
579 return GNUNET_SYSERR;
580 }
581 GNUNET_SQ_reset (plugin->dbh, plugin->delRow);
582 return GNUNET_OK;
583}
584
585
602static void
604 const struct GNUNET_HashCode *key,
605 bool absent,
606 uint32_t size,
607 const void *data,
609 uint32_t priority,
610 uint32_t anonymity,
611 uint32_t replication,
613 PluginPutCont cont,
614 void *cont_cls)
615{
616 struct Plugin *plugin = cls;
617 struct GNUNET_HashCode vhash;
618 char *msg = NULL;
619
620 GNUNET_CRYPTO_hash (data, size, &vhash);
621
622 if (! absent)
623 {
624 struct GNUNET_SQ_QueryParam params[] =
625 { GNUNET_SQ_query_param_uint32 (&priority),
631
632 if (GNUNET_OK != GNUNET_SQ_bind (plugin->update, params))
633 {
634 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("sqlite bind failure"));
635 return;
636 }
637 if (SQLITE_DONE != sqlite3_step (plugin->update))
638 {
640 &msg,
642 "sqlite3_step");
643 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
645 return;
646 }
647 {
648 int changes = sqlite3_changes (plugin->dbh);
649 GNUNET_SQ_reset (plugin->dbh, plugin->update);
650 if (0 != changes)
651 {
652 cont (cont_cls, key, size, GNUNET_NO, NULL);
653 return;
654 }
655 }
656 }
657
658 {
659 // FIXME Refactor?
660 uint64_t rvalue;
661 uint32_t type32 = (uint32_t) type;
662 int n;
663 int ret;
664 sqlite3_stmt *stmt;
665
666 if (size > MAX_ITEM_SIZE)
667 {
668 cont (cont_cls, key, size, GNUNET_SYSERR, _ ("Data too large"));
669 return;
670 }
673 "sqlite",
674 "Storing in database block with type %u/key `%s'/priority %u/expiration in %s (%s).\n",
675 type,
676 GNUNET_h2s (key),
677 priority,
679 (
680 expiration),
681 GNUNET_YES),
683 stmt = plugin->insertContent;
685 {
686 struct GNUNET_SQ_QueryParam params[] =
697 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
698 {
699 cont (cont_cls, key, size, GNUNET_SYSERR, NULL);
700 return;
701 }
702 }
703 n = sqlite3_step (stmt);
704 switch (n)
705 {
706 case SQLITE_DONE:
707 if (NULL != plugin->env->duc)
708 plugin->env->duc (plugin->env->cls,
711 "sqlite",
712 "Stored new entry (%u bytes)\n",
714 ret = GNUNET_OK;
715 break;
716
717 case SQLITE_BUSY:
718 GNUNET_break (0);
720 &msg,
722 "sqlite3_step");
724 break;
725
726 default:
728 &msg,
730 "sqlite3_step");
731 GNUNET_SQ_reset (plugin->dbh, stmt);
733 database_setup (plugin->env->cfg, plugin);
734 cont (cont_cls, key, size, GNUNET_SYSERR, msg);
736 return;
737 }
738 GNUNET_SQ_reset (plugin->dbh, stmt);
739 cont (cont_cls, key, size, ret, msg);
741 }
742}
743
744
754static void
756 sqlite3_stmt *stmt,
758 void *proc_cls)
759{
760 int n;
762 uint32_t replication;
763 uint32_t type;
764 uint32_t priority;
765 uint32_t anonymity;
766 uint64_t rowid;
767 void *value;
768 size_t value_size;
769 struct GNUNET_HashCode key;
770 int ret;
771 struct GNUNET_SQ_ResultSpec rs[] =
781
782 n = sqlite3_step (stmt);
783 switch (n)
784 {
785 case SQLITE_ROW:
786 if (GNUNET_OK != GNUNET_SQ_extract_result (stmt, rs))
787 {
788 GNUNET_break (0);
789 break;
790 }
792 "sqlite",
793 "Found reply in database with expiration %s\n",
795 ret = proc (proc_cls,
796 &key,
797 value_size,
798 value,
799 type,
800 priority,
801 anonymity,
804 rowid);
806 GNUNET_SQ_reset (plugin->dbh, stmt);
807 if ((GNUNET_NO == ret) && (GNUNET_OK == delete_by_rowid (plugin, rowid)) &&
808 (NULL != plugin->env->duc))
809 plugin->env->duc (plugin->env->cls,
810 -(value_size + GNUNET_DATASTORE_ENTRY_OVERHEAD));
811 return;
812
813 case SQLITE_DONE:
814 /* database must be empty */
815 break;
816
817 case SQLITE_BUSY:
818 case SQLITE_ERROR:
819 case SQLITE_MISUSE:
820 default:
823 "sqlite3_step");
824 if (SQLITE_OK != sqlite3_reset (stmt))
827 "sqlite3_reset");
828 GNUNET_break (0);
829 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
831 database_setup (plugin->env->cfg, plugin);
832 return;
833 }
834 GNUNET_SQ_reset (plugin->dbh, stmt);
835 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
836}
837
838
851static void
853 uint64_t next_uid,
856 void *proc_cls)
857{
858 struct Plugin *plugin = cls;
859 uint32_t type32 = type;
861 &next_uid),
863 &type32),
865
867 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selZeroAnon, params))
868 {
869 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
870 return;
871 }
872 execute_get (plugin, plugin->selZeroAnon, proc, proc_cls);
873}
874
875
889static void
891 uint64_t next_uid,
892 bool random,
893 const struct GNUNET_HashCode *key,
896 void *proc_cls)
897{
898 struct Plugin *plugin = cls;
899 uint64_t rvalue;
900 int use_rvalue = random;
901 uint32_t type32 = (uint32_t) type;
902 int use_type = GNUNET_BLOCK_TYPE_ANY != type;
903 int use_key = NULL != key;
904 sqlite3_stmt *stmt = plugin->get[use_rvalue * 4 + use_key * 2 + use_type];
905 if (random)
906 {
908 next_uid = 0;
909 }
910 else
911 rvalue = 0;
912 { struct GNUNET_SQ_QueryParam params[] =
913 { GNUNET_SQ_query_param_uint64 (&next_uid),
918
919 /* SQLite doesn't like it when you try to bind a parameter greater than the
920 * last numbered parameter, but unused parameters in the middle are OK.
921 */
922 if (! use_type)
923 {
925 if (! use_key)
926 {
928 if (! use_rvalue)
930 }
931 }
932
933 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
934 {
935 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
936 return;
937 }
938 }
939 execute_get (plugin, stmt, proc, proc_cls);
940}
941
942
946struct ReplCtx
947{
952
956 void *proc_cls;
957
961 uint64_t uid;
962
967};
968
969
989static int
990repl_proc (void *cls,
991 const struct GNUNET_HashCode *key,
992 uint32_t size,
993 const void *data,
995 uint32_t priority,
996 uint32_t anonymity,
997 uint32_t replication,
999 uint64_t uid)
1000{
1001 struct ReplCtx *rc = cls;
1002 int ret;
1003
1004 if (GNUNET_SYSERR == rc->have_uid)
1005 rc->have_uid = GNUNET_NO;
1006 ret = rc->proc (rc->proc_cls,
1007 key,
1008 size,
1009 data,
1010 type,
1011 priority,
1012 anonymity,
1014 expiration,
1015 uid);
1016 if (NULL != key)
1017 {
1018 rc->uid = uid;
1019 rc->have_uid = GNUNET_YES;
1020 }
1021 return ret;
1022}
1023
1024
1035static void
1038 void *proc_cls)
1039{
1040 struct Plugin *plugin = cls;
1041 struct ReplCtx rc;
1042 uint64_t rvalue = 0;
1043 uint32_t repl;
1044 struct GNUNET_SQ_QueryParam params_upd_repl[] =
1046
1048 "datastore-sqlite",
1049 "Getting random block based on replication order.\n");
1050 if (SQLITE_ROW != sqlite3_step (plugin->maxRepl))
1051 {
1052 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1053 /* DB empty */
1054 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1055 return;
1056 }
1057 repl = sqlite3_column_int (plugin->maxRepl, 0);
1058 {
1059 struct GNUNET_SQ_QueryParam params_sel_repl[] =
1060 { GNUNET_SQ_query_param_uint64 (&rvalue),
1063 GNUNET_SQ_reset (plugin->dbh, plugin->maxRepl);
1065 if (GNUNET_OK != GNUNET_SQ_bind (plugin->selRepl, params_sel_repl))
1066 {
1067 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1068 return;
1069 }
1070 }
1072 rc.proc = proc;
1073 rc.proc_cls = proc_cls;
1074 execute_get (plugin, plugin->selRepl, &repl_proc, &rc);
1075 if (GNUNET_YES == rc.have_uid)
1076 {
1077 if (GNUNET_OK != GNUNET_SQ_bind (plugin->updRepl, params_upd_repl))
1078 {
1079 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1080 return;
1081 }
1082 if (SQLITE_DONE != sqlite3_step (plugin->updRepl))
1085 "sqlite3_step");
1086 GNUNET_SQ_reset (plugin->dbh, plugin->updRepl);
1087 }
1088 if (GNUNET_SYSERR == rc.have_uid)
1089 {
1090 /* proc was not called at all so far, do it now. */
1091 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1092 }
1093}
1094
1095
1104static void
1107 void *proc_cls)
1108{
1109 struct Plugin *plugin = cls;
1110 sqlite3_stmt *stmt;
1111 struct GNUNET_TIME_Absolute now = { 0 };
1113 &now),
1115
1118 "sqlite",
1119 "Getting random block based on expiration and priority order.\n");
1120 now = GNUNET_TIME_absolute_get ();
1121 stmt = plugin->selExpi;
1122 if (GNUNET_OK != GNUNET_SQ_bind (stmt, params))
1123 {
1124 proc (proc_cls, NULL, 0, NULL, 0, 0, 0, 0, GNUNET_TIME_UNIT_ZERO_ABS, 0);
1125 return;
1126 }
1127 execute_get (plugin, stmt, proc, proc_cls);
1128}
1129
1130
1138static void
1139sqlite_plugin_get_keys (void *cls, PluginKeyProcessor proc, void *proc_cls)
1140{
1141 struct Plugin *plugin = cls;
1142 struct GNUNET_HashCode key;
1143 struct GNUNET_SQ_ResultSpec results[] =
1145 sqlite3_stmt *stmt;
1146 int ret;
1147
1148 GNUNET_assert (NULL != proc);
1149 if (SQLITE_OK != sq_prepare (plugin->dbh, "SELECT hash FROM gn091", &stmt))
1150 {
1153 "sqlite_prepare");
1154 proc (proc_cls, NULL, 0);
1155 return;
1156 }
1157 while (SQLITE_ROW == (ret = sqlite3_step (stmt)))
1158 {
1160 proc (proc_cls, &key, 1);
1161 else
1162 GNUNET_break (0);
1163 }
1164 if (SQLITE_DONE != ret)
1165 LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite_step");
1166 sqlite3_finalize (stmt);
1167 proc (proc_cls, NULL, 0);
1168}
1169
1170
1176static void
1178{
1179 struct Plugin *plugin = cls;
1180
1181 plugin->drop_on_shutdown = GNUNET_YES;
1182}
1183
1184
1195static void
1197 const struct GNUNET_HashCode *key,
1198 uint32_t size,
1199 const void *data,
1200 PluginRemoveCont cont,
1201 void *cont_cls)
1202{
1203 struct Plugin *plugin = cls;
1204 struct GNUNET_SQ_QueryParam params[] =
1208
1209 if (GNUNET_OK != GNUNET_SQ_bind (plugin->remove, params))
1210 {
1211 cont (cont_cls, key, size, GNUNET_SYSERR, "bind failed");
1212 return;
1213 }
1214 if (SQLITE_DONE != sqlite3_step (plugin->remove))
1215 {
1218 "sqlite3_step");
1219 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1220 cont (cont_cls, key, size, GNUNET_SYSERR, "sqlite3_step failed");
1221 return;
1222 }
1223 {
1224 int changes = sqlite3_changes (plugin->dbh);
1225 GNUNET_SQ_reset (plugin->dbh, plugin->remove);
1226 if (0 == changes)
1227 {
1228 cont (cont_cls, key, size, GNUNET_NO, NULL);
1229 return;
1230 }
1231 }
1232 if (NULL != plugin->env->duc)
1233 plugin->env->duc (plugin->env->cls,
1235 cont (cont_cls, key, size, GNUNET_OK, NULL);
1236}
1237
1238
1246static void
1247sqlite_plugin_estimate_size (void *cls, unsigned long long *estimate)
1248{
1249 struct Plugin *plugin = cls;
1250 sqlite3_stmt *stmt;
1251 uint64_t pages;
1252 uint64_t page_size;
1253
1254#if ENULL_DEFINED
1255 char *e;
1256#endif
1257
1258 if (NULL == estimate)
1259 return;
1260 if (SQLITE_VERSION_NUMBER < 3006000)
1261 {
1264 "datastore-sqlite",
1265 _ ("sqlite version to old to determine size, assuming zero\n"));
1266 *estimate = 0;
1267 return;
1268 }
1269 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh, "VACUUM", NULL, NULL, ENULL));
1270 CHECK (SQLITE_OK == sqlite3_exec (plugin->dbh,
1271 "PRAGMA auto_vacuum=INCREMENTAL",
1272 NULL,
1273 NULL,
1274 ENULL));
1275 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_count", &stmt))
1276 {
1279 "datastore-sqlite",
1280 _ ("error preparing statement\n"));
1281 return;
1282 }
1283 if (SQLITE_ROW == sqlite3_step (stmt))
1284 pages = sqlite3_column_int64 (stmt, 0);
1285 else
1286 pages = 0;
1287 sqlite3_finalize (stmt);
1288 if (SQLITE_OK != sq_prepare (plugin->dbh, "PRAGMA page_size", &stmt))
1289 {
1292 "datastore-sqlite",
1293 _ ("error preparing statement\n"));
1294 return;
1295 }
1296 if (SQLITE_ROW != sqlite3_step (stmt))
1297 {
1300 "datastore-sqlite",
1301 _ ("error stepping\n"));
1302 return;
1303 }
1304 page_size = sqlite3_column_int64 (stmt, 0);
1305 sqlite3_finalize (stmt);
1306 GNUNET_log (
1308 _ (
1309 "Using sqlite page utilization to estimate payload (%llu pages of size %llu bytes)\n"),
1310 (unsigned long long) pages,
1311 (unsigned long long) page_size);
1312 *estimate = pages * page_size;
1313}
1314
1315
1316void *
1318
1325void *
1327{
1328 static struct Plugin plugin;
1331
1332 if (NULL != plugin.env)
1333 return NULL; /* can only initialize once! */
1334 memset (&plugin, 0, sizeof(struct Plugin));
1335 plugin.env = env;
1336 if (GNUNET_OK != database_setup (env->cfg, &plugin))
1337 {
1339 return NULL;
1340 }
1342 api->cls = &plugin;
1344 api->put = &sqlite_plugin_put;
1350 api->drop = &sqlite_plugin_drop;
1353 "sqlite",
1354 _ ("Sqlite database running\n"));
1355 return api;
1356}
1357
1358
1359void *
1361
1368void *
1370{
1371 char *fn;
1373 struct Plugin *plugin = api->cls;
1374
1376 "sqlite",
1377 "sqlite plugin is done\n");
1378 fn = NULL;
1379 if (plugin->drop_on_shutdown)
1380 fn = GNUNET_strdup (plugin->fn);
1382 plugin->env = NULL;
1383 GNUNET_free (api);
1384 if (NULL != fn)
1385 {
1386 if (0 != unlink (fn))
1388 GNUNET_free (fn);
1389 }
1390 return NULL;
1391}
1392
1393
1394/* end of plugin_datastore_sqlite.c */
struct GNUNET_MessageHeader * msg
Definition 005.c:2
struct GNUNET_MQ_Envelope * env
Definition 005.c:1
static int ret
Final status code.
Definition gnunet-arm.c:93
static struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition gnunet-arm.c:108
static struct GNUNET_TESTING_PluginFunctions * plugin
Plugin to dynamically load a test case.
static unsigned int replication
Desired replication level.
static char * data
The data to insert into the dht.
struct GNUNET_HashCode key
The key used in the DHT.
static struct GNUNET_TIME_Relative expiration
User supplied expiration value.
static unsigned int anonymity
static struct in_addr dummy
Target "dummy" address of the packet we pretend to respond to.
static char * value
Value of the record to add/remove.
static uint32_t type
Type string converted to DNS type value.
static int result
Global testing status.
static unsigned int results
API for the database backend plugins.
GNUNET_BLOCK_Type
WARNING: This header is generated! In order to add DHT block types, you must register them in GANA,...
@ GNUNET_BLOCK_TYPE_ANY
Identifier for any block.
helper functions for Sqlite3 DB interactions
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_fixed_size(const void *ptr, size_t ptr_size)
Generate query parameter for a buffer ptr of ptr_size bytes.
enum GNUNET_GenericReturnValue GNUNET_SQ_extract_result(sqlite3_stmt *result, struct GNUNET_SQ_ResultSpec *rs)
Extract results from a query result according to the given specification.
Definition sq.c:75
void GNUNET_SQ_cleanup_result(struct GNUNET_SQ_ResultSpec *rs)
Free all memory that was allocated in rs during GNUNET_SQ_extract_result().
Definition sq.c:104
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_uint32(const uint32_t *x)
Generate query parameter for an uint32_t in host byte order.
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_uint64(const uint64_t *x)
Generate query parameter for an uint16_t in host byte order.
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_absolute_time(struct GNUNET_TIME_Absolute *at)
Absolute time expected.
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_uint32(uint32_t *u32)
uint32_t expected.
struct GNUNET_SQ_QueryParam GNUNET_SQ_query_param_absolute_time(const struct GNUNET_TIME_Absolute *x)
Generate query parameter for an absolute time value.
#define GNUNET_SQ_result_spec_end
End of result parameter specification.
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_variable_size(void **dst, size_t *sptr)
Variable-size result expected.
#define GNUNET_SQ_query_param_auto_from_type(x)
Generate fixed-size query parameter with size determined by variable type.
enum GNUNET_GenericReturnValue GNUNET_SQ_bind(sqlite3_stmt *stmt, const struct GNUNET_SQ_QueryParam *params)
Execute binding operations for a prepared statement.
Definition sq.c:30
void GNUNET_SQ_reset(sqlite3 *dbh, sqlite3_stmt *stmt)
Reset stmt and log error.
Definition sq.c:119
#define GNUNET_SQ_query_param_end
End of query parameter specification.
#define GNUNET_SQ_result_spec_auto_from_type(dst)
We expect a fixed-size result, with size determined by the type of * dst
struct GNUNET_SQ_ResultSpec GNUNET_SQ_result_spec_uint64(uint64_t *u64)
uint64_t expected.
enum GNUNET_GenericReturnValue GNUNET_CONFIGURATION_get_value_filename(const struct GNUNET_CONFIGURATION_Handle *cfg, const char *section, const char *option, char **value)
Get a configuration value that should be the name of a file or directory.
uint64_t GNUNET_CRYPTO_random_u64(enum GNUNET_CRYPTO_Quality mode, uint64_t max)
Generate a random unsigned 64-bit value.
@ GNUNET_CRYPTO_QUALITY_WEAK
No good quality of the operation is needed (i.e., random numbers can be pseudo-random).
enum GNUNET_GenericReturnValue(* PluginDatumProcessor)(void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
An processor over a set of items stored in the datastore.
void(* PluginPutCont)(void *cls, const struct GNUNET_HashCode *key, uint32_t size, int status, const char *msg)
Put continuation.
#define GNUNET_DATASTORE_ENTRY_OVERHEAD
How many bytes of overhead will we assume per entry in any DB (for reservations)?
void(* PluginKeyProcessor)(void *cls, const struct GNUNET_HashCode *key, unsigned int count)
An processor over a set of keys stored in the datastore.
void(* PluginRemoveCont)(void *cls, const struct GNUNET_HashCode *key, uint32_t size, int status, const char *msg)
Remove continuation.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_test(const char *fil)
Check that fil corresponds to a filename (of a file that exists and that is not a directory).
Definition disk.c:533
enum GNUNET_GenericReturnValue GNUNET_DISK_directory_create_for_file(const char *filename)
Create the directory structure for storing a file.
Definition disk.c:633
void GNUNET_CRYPTO_hash(const void *block, size_t size, struct GNUNET_HashCode *ret)
Compute hash of a given block.
Definition crypto_hash.c:41
#define GNUNET_log(kind,...)
#define GNUNET_log_from(kind, comp,...)
@ GNUNET_OK
@ GNUNET_YES
@ GNUNET_NO
@ GNUNET_SYSERR
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_break(cond)
Use this for internal assertion violations that are not fatal (can be handled) but should not occur.
const char * GNUNET_h2s(const struct GNUNET_HashCode *hc)
Convert a hash value to a string (for printing debug messages).
void GNUNET_log_config_missing(enum GNUNET_ErrorType kind, const char *section, const char *option)
Log error message about missing configuration option.
#define GNUNET_log_strerror_file(level, cmd, filename)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' with the mess...
@ GNUNET_ERROR_TYPE_WARNING
@ GNUNET_ERROR_TYPE_ERROR
@ GNUNET_ERROR_TYPE_BULK
@ GNUNET_ERROR_TYPE_DEBUG
@ GNUNET_ERROR_TYPE_INFO
#define GNUNET_strdup(a)
Wrapper around GNUNET_xstrdup_.
#define GNUNET_new(type)
Allocate a struct or union of the given type.
#define GNUNET_free(ptr)
Wrapper around free.
struct GNUNET_TIME_Relative GNUNET_TIME_absolute_get_remaining(struct GNUNET_TIME_Absolute future)
Given a timestamp in the future, how much time remains until then?
Definition time.c:406
const char * GNUNET_STRINGS_relative_time_to_string(struct GNUNET_TIME_Relative delta, int do_round)
Give relative time in human-readable fancy format.
Definition strings.c:599
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_get(void)
Get the current time.
Definition time.c:111
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.
const char * GNUNET_STRINGS_absolute_time_to_string(struct GNUNET_TIME_Absolute t)
Like asctime, except for GNUnet time.
Definition strings.c:660
static unsigned int size
Size of the "table".
Definition peer.c:68
#define _(String)
GNU gettext support macro.
Definition platform.h:179
static int database_setup(const struct GNUNET_CONFIGURATION_Handle *cfg, struct Plugin *plugin)
Initialize the database connections and associated data structures (create tables and indices as need...
static void sqlite_plugin_estimate_size(void *cls, unsigned long long *estimate)
Get an estimate of how much space the database is currently using.
#define MAX_ITEM_SIZE
We allocate items on the stack at times.
#define LOG_SQLITE_MSG(db, msg, level, cmd)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' on file 'file...
static int repl_proc(void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Wrapper for the processor for sqlite_plugin_get_replication().
#define BUSY_TIMEOUT_MS
After how many ms "busy" should a DB operation fail for good? A low value makes sure that we are more...
#define RESULT_COLUMNS
static void sqlite_plugin_put(void *cls, const struct GNUNET_HashCode *key, bool absent, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, PluginPutCont cont, void *cont_cls)
Store an item in the datastore.
static void sqlite_plugin_remove_key(void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, PluginRemoveCont cont, void *cont_cls)
Remove a particular key in the datastore.
#define LOG_SQLITE(db, level, cmd)
Log an error message at log-level 'level' that indicates a failure of the command 'cmd' on file 'file...
static int sq_prepare(sqlite3 *dbh, const char *zSql, sqlite3_stmt **ppStmt)
Prepare a SQL statement.
static void sqlite_plugin_get_keys(void *cls, PluginKeyProcessor proc, void *proc_cls)
Get all of the keys in the datastore.
static void create_indices(sqlite3 *dbh)
Create our database indices.
#define CHECK(a)
static int delete_by_rowid(struct Plugin *plugin, uint64_t rid)
Delete the database entry with the given row identifier.
static void database_shutdown(struct Plugin *plugin)
Shutdown database connection and associate data structures.
#define ENULL
static void execute_get(struct Plugin *plugin, sqlite3_stmt *stmt, PluginDatumProcessor proc, void *proc_cls)
Execute statement that gets a row and call the callback with the result.
void * libgnunet_plugin_datastore_sqlite_done(void *cls)
Exit point from the plugin.
static void sqlite_plugin_get_key(void *cls, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls)
Get results for a particular key in the datastore.
static void sqlite_plugin_get_replication(void *cls, PluginDatumProcessor proc, void *proc_cls)
Get a random item for replication.
static void sqlite_plugin_get_zero_anonymity(void *cls, uint64_t next_uid, enum GNUNET_BLOCK_Type type, PluginDatumProcessor proc, void *proc_cls)
Select a subset of the items in the datastore and call the given processor for the item.
static void sqlite_plugin_drop(void *cls)
Drop database.
void * libgnunet_plugin_datastore_sqlite_init(void *cls)
Entry point for the plugin.
static void sqlite_plugin_get_expiration(void *cls, PluginDatumProcessor proc, void *proc_cls)
Get a random item that has expired or has low priority.
void * cls
Closure for all of the callbacks.
The datastore service will pass a pointer to a struct of this type as the first and only argument to ...
Each plugin is required to return a pointer to a struct of this type as the return value from its ent...
PluginGetKey get_key
Get a particular datum matching a given hash from the datastore.
PluginGetRandom get_expiration
Function to get a random expired item or, if none are expired, either the oldest entry or one with a ...
PluginEstimateSize estimate_size
Calculate the current on-disk size of the SQ store.
PluginGetKeys get_keys
Iterate over all keys in the database.
PluginGetRandom get_replication
Function to get a random item with high replication score from the database, lowering the item's repl...
PluginPut put
Function to store an item in the datastore.
void * cls
Closure to use for all of the following callbacks (except "next_request").
PluginGetType get_zero_anonymity
Get datum (of the specified type) with anonymity level zero.
PluginDrop drop
Delete the database.
PluginRemoveKey remove_key
Function to remove an item from the database.
A 512-bit hashcode.
Description of a DB query parameter.
Description of a DB result cell.
void * cls
Closure for conv and cleaner.
void * cls
Closure to pass to start_testcase.
Time for absolute times used by GNUnet, in microseconds.
Handle for a plugin.
Definition block.c:38
struct GNUNET_BLOCK_PluginFunctions * api
Plugin API.
Definition block.c:47
sqlite3_stmt * maxRepl
Get maximum repl value in database.
sqlite3_stmt * insertContent
Precompiled SQL for insertion.
sqlite3_stmt * update
Precompiled SQL for update.
sqlite3_stmt * remove
Precompiled SQL for remove_key.
char * fn
Filename used for the DB.
struct GNUNET_DATACACHE_PluginEnvironment * env
Our execution environment.
int drop_on_shutdown
Should the database be dropped on shutdown?
sqlite3_stmt * selRepl
Precompiled SQL for replication selection.
sqlite3_stmt * delRow
Precompiled SQL for deletion.
sqlite3_stmt * get[8]
Precompiled SQL for selection.
struct GNUNET_PQ_Context * dbh
Native Postgres database handle.
sqlite3_stmt * selZeroAnon
Precompiled SQL for expiration selection.
sqlite3_stmt * selExpi
Precompiled SQL for expiration selection.
sqlite3_stmt * updRepl
Precompiled SQL for replication decrement.
Context for #repl_iter() function.
void * proc_cls
Closure for proc.
uint64_t uid
UID to use.
int have_uid
Yes if UID was set.
PluginDatumProcessor proc
Function to call for the result (or the NULL).