GNUnet 0.21.0
gnunet-datastore.c File Reference

tool to manipulate datastores More...

#include "platform.h"
#include <inttypes.h>
#include "gnunet_util_lib.h"
#include "gnunet_datastore_service.h"
Include dependency graph for gnunet-datastore.c:

Go to the source code of this file.

Data Structures

struct  DataRecord
 

Functions

static void do_shutdown (void *cls)
 
static void start_dump (void)
 Begin dumping the database. More...
 
static void start_insert (void)
 Begin inserting into the database. More...
 
static void do_get (const uint64_t next_uid)
 Perform next GET operation. More...
 
static void get_cb (void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
 Process a datum that was stored in the datastore. More...
 
static void put_cb (void *cls, int32_t success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
 Continuation called to notify client about result of the operation. More...
 
static void run (void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
 Main function that will be run by the scheduler. More...
 
int main (int argc, char *const *argv)
 The main function to manipulate datastores. More...
 

Variables

static GNUNET_NETWORK_STRUCT_END const size_t MAGIC_LEN = 16
 Length of our magic header. More...
 
static const uint8_t MAGIC_BYTES [16] = "GNUNETDATASTORE1"
 Magic header bytes. More...
 
static int dump
 Dump the database. More...
 
static int insert
 Insert into the database. More...
 
static char * file_name
 Dump file name. More...
 
static struct GNUNET_DISK_FileHandlefile_handle
 Dump file handle. More...
 
static int ret
 Global return value. More...
 
static struct GNUNET_DATASTORE_Handledatastore
 Handle for datastore. More...
 
static struct GNUNET_DATASTORE_QueueEntryqe
 Current operation. More...
 
static uint64_t record_count
 Record count. More...
 

Detailed Description

tool to manipulate datastores

Author
Christian Grothoff

Definition in file gnunet-datastore.c.

Function Documentation

◆ do_shutdown()

static void do_shutdown ( void *  cls)
static

Definition at line 125 of file gnunet-datastore.c.

126{
127 if (NULL != qe)
129 if (NULL != datastore)
131 if (NULL != file_handle)
133}
static struct GNUNET_DISK_FileHandle * file_handle
Dump file handle.
static struct GNUNET_DATASTORE_QueueEntry * qe
Current operation.
static struct GNUNET_DATASTORE_Handle * datastore
Handle for datastore.
void GNUNET_DATASTORE_disconnect(struct GNUNET_DATASTORE_Handle *h, int drop)
Disconnect from the datastore service (and free associated resources).
void GNUNET_DATASTORE_cancel(struct GNUNET_DATASTORE_QueueEntry *qe)
Cancel a datastore operation.
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1308
@ GNUNET_NO

References datastore, file_handle, GNUNET_DATASTORE_cancel(), GNUNET_DATASTORE_disconnect(), GNUNET_DISK_file_close(), GNUNET_NO, and qe.

Referenced by run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ start_dump()

static void start_dump ( void  )
static

Begin dumping the database.

Definition at line 268 of file gnunet-datastore.c.

269{
270 record_count = 0;
271
272 if (NULL != file_name)
273 {
280 if (NULL == file_handle)
281 {
282 fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
283 ret = 1;
285 return;
286 }
287 }
288 else
289 {
291 }
293 do_get (0);
294}
static void do_get(const uint64_t next_uid)
Perform next GET operation.
static uint64_t record_count
Record count.
static const uint8_t MAGIC_BYTES[16]
Magic header bytes.
static int ret
Global return value.
static char * file_name
Dump file name.
static GNUNET_NETWORK_STRUCT_END const size_t MAGIC_LEN
Length of our magic header.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1237
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:686
struct GNUNET_DISK_FileHandle * GNUNET_DISK_get_handle_from_int_fd(int fno)
Get a handle from a native integer FD.
Definition: disk.c:1330
@ GNUNET_DISK_OPEN_WRITE
Open the file for writing.
@ GNUNET_DISK_OPEN_TRUNCATE
Truncate file if it exists.
@ GNUNET_DISK_OPEN_CREATE
Create file if it doesn't exist.
@ GNUNET_DISK_PERM_USER_READ
Owner can read.
@ GNUNET_DISK_PERM_USER_WRITE
Owner can write.
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:562
#define _(String)
GNU gettext support macro.
Definition: platform.h:178

References _, do_get(), file_handle, file_name, GNUNET_DISK_file_open(), GNUNET_DISK_file_write(), GNUNET_DISK_get_handle_from_int_fd(), GNUNET_DISK_OPEN_CREATE, GNUNET_DISK_OPEN_TRUNCATE, GNUNET_DISK_OPEN_WRITE, GNUNET_DISK_PERM_USER_READ, GNUNET_DISK_PERM_USER_WRITE, GNUNET_SCHEDULER_shutdown(), MAGIC_BYTES, MAGIC_LEN, record_count, and ret.

Referenced by run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ start_insert()

static void start_insert ( void  )
static

Begin inserting into the database.

Definition at line 389 of file gnunet-datastore.c.

390{
391 record_count = 0;
392
393 if (NULL != file_name)
394 {
398 if (NULL == file_handle)
399 {
400 fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
401 ret = 1;
403 return;
404 }
405 }
406 else
407 {
409 }
410
411 uint8_t buf[MAGIC_LEN];
412 ssize_t len;
413
415 if ((len != MAGIC_LEN) || (0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN)))
416 {
417 fprintf (stderr, _ ("Input file is not of a supported format\n"));
418 return;
419 }
421}
static void put_cb(void *cls, int32_t success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called to notify client about result of the operation.
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:622
@ GNUNET_DISK_OPEN_READ
Open the file for reading.
@ GNUNET_DISK_PERM_NONE
Nobody is allowed to do anything to the file.
@ GNUNET_YES
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.

References _, file_handle, file_name, GNUNET_DISK_file_open(), GNUNET_DISK_file_read(), GNUNET_DISK_get_handle_from_int_fd(), GNUNET_DISK_OPEN_READ, GNUNET_DISK_PERM_NONE, GNUNET_SCHEDULER_shutdown(), GNUNET_TIME_UNIT_ZERO_ABS, GNUNET_YES, MAGIC_BYTES, MAGIC_LEN, put_cb(), record_count, and ret.

Referenced by get_cb(), and run().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ do_get()

static void do_get ( const uint64_t  next_uid)
static

Perform next GET operation.

Definition at line 243 of file gnunet-datastore.c.

244{
245 GNUNET_assert (NULL == qe);
247 next_uid,
248 false /* random */,
249 NULL /* key */,
251 0 /* queue_priority */,
252 1 /* max_queue_size */,
253 &get_cb,
254 NULL /* proc_cls */);
255 if (NULL == qe)
256 {
257 fprintf (stderr, _ ("Error queueing datastore GET operation\n"));
258 ret = 1;
260 }
261}
static void get_cb(void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Process a datum that was stored in the datastore.
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_get_key(struct GNUNET_DATASTORE_Handle *h, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_DatumProcessor proc, void *proc_cls)
Get a result for a particular key from the datastore.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
@ GNUNET_BLOCK_TYPE_ANY
Identifier for any block.

References _, datastore, get_cb(), GNUNET_assert, GNUNET_BLOCK_TYPE_ANY, GNUNET_DATASTORE_get_key(), GNUNET_SCHEDULER_shutdown(), qe, and ret.

Referenced by get_cb(), and start_dump().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ get_cb()

static void get_cb ( void *  cls,
const struct GNUNET_HashCode key,
size_t  size,
const void *  data,
enum GNUNET_BLOCK_Type  type,
uint32_t  priority,
uint32_t  anonymity,
uint32_t  replication,
struct GNUNET_TIME_Absolute  expiration,
uint64_t  uid 
)
static

Process a datum that was stored in the datastore.

Parameters
clsclosure
keykey for the content
sizenumber of bytes in data
datacontent stored
typetype of the content
prioritypriority of the content
anonymityanonymity-level for the content
replicationreplication-level for the content
expirationexpiration time for the content
uidunique identifier for the datum; maybe 0 if no unique identifier is available

Definition at line 173 of file gnunet-datastore.c.

183{
184 qe = NULL;
185 if (NULL == key)
186 {
187 fprintf (stderr, _ ("Dumped %" PRIu64 " records\n"), record_count);
189 file_handle = NULL;
190 if (insert)
191 start_insert ();
192 else
193 {
194 ret = 0;
196 }
197 return;
198 }
199
200 struct DataRecord dr;
201 dr.size = htonl ((uint32_t) size);
202 dr.type = htonl (type);
203 dr.priority = htonl (priority);
204 dr.anonymity = htonl (anonymity);
205 dr.replication = htonl (replication);
206 dr.expiration = GNUNET_TIME_absolute_hton (expiration);
207 dr.key = *key;
208
209 ssize_t len;
210 len = GNUNET_DISK_file_write (file_handle, &dr, sizeof(dr));
211 if (sizeof(dr) != len)
212 {
213 fprintf (stderr,
214 _ ("Short write to file: %zd bytes expecting %zd\n"),
215 len,
216 sizeof(dr));
217 ret = 1;
219 return;
220 }
221
223 if (size != len)
224 {
225 fprintf (stderr,
226 _ ("Short write to file: %zd bytes expecting %zd\n"),
227 len,
228 size);
229 ret = 1;
231 return;
232 }
233
234 record_count++;
235 do_get (uid + 1);
236}
static void start_insert(void)
Begin inserting into the database.
static int insert
Insert into the database.
static unsigned int replication
Desired replication level.
static char * data
The data to insert into the dht.
struct GNUNET_HashCode key
The key used in the DHT.
static struct GNUNET_TIME_Relative expiration
User supplied expiration value.
static unsigned int anonymity
static uint32_t type
Type string converted to DNS type value.
struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton(struct GNUNET_TIME_Absolute a)
Convert absolute time to network byte order.
Definition: time.c:638
static unsigned int size
Size of the "table".
Definition: peer.c:68
uint32_t size
Number of bytes in the item (NBO).
uint32_t priority
Priority of the item (NBO).

References _, DataRecord::anonymity, anonymity, data, do_get(), DataRecord::expiration, expiration, file_handle, GNUNET_DISK_file_close(), GNUNET_DISK_file_write(), GNUNET_SCHEDULER_shutdown(), GNUNET_TIME_absolute_hton(), insert, DataRecord::key, key, DataRecord::priority, qe, record_count, DataRecord::replication, replication, ret, DataRecord::size, size, start_insert(), DataRecord::type, and type.

Referenced by do_get(), and GNUNET_DHT_monitor_start().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ put_cb()

static void put_cb ( void *  cls,
int32_t  success,
struct GNUNET_TIME_Absolute  min_expiration,
const char *  msg 
)
static

Continuation called to notify client about result of the operation.

Parameters
clsclosure
successGNUNET_SYSERR on failure (including timeout/queue drop) GNUNET_NO if content was already there GNUNET_YES (or other positive value) on success
min_expirationminimum expiration time required for 0-priority content to be stored by the datacache at this time, zero for unknown, forever if we have no space for 0-priority content
msgNULL on success, otherwise an error message

Definition at line 311 of file gnunet-datastore.c.

315{
316 qe = NULL;
317 if (GNUNET_SYSERR == success)
318 {
319 fprintf (stderr, _ ("Failed to store item: %s, aborting\n"), msg);
320 ret = 1;
322 return;
323 }
324
325 struct DataRecord dr;
326 ssize_t len;
327
328 len = GNUNET_DISK_file_read (file_handle, &dr, sizeof(dr));
329 if (0 == len)
330 {
331 fprintf (stderr, _ ("Inserted %" PRIu64 " records\n"), record_count);
332 ret = 0;
334 return;
335 }
336 else if (sizeof(dr) != len)
337 {
338 fprintf (stderr,
339 _ ("Short read from file: %zd bytes expecting %zd\n"),
340 len,
341 sizeof(dr));
342 ret = 1;
344 return;
345 }
346
347 const size_t size = ntohl (dr.size);
348 uint8_t data[size];
350 if (size != len)
351 {
352 fprintf (stderr,
353 _ ("Short read from file: %zd bytes expecting %zd\n"),
354 len,
355 size);
356 ret = 1;
358 return;
359 }
360
361 record_count++;
363 0,
364 &dr.key,
365 size,
366 data,
367 ntohl (dr.type),
368 ntohl (dr.priority),
369 ntohl (dr.anonymity),
370 ntohl (dr.replication),
371 GNUNET_TIME_absolute_ntoh (dr.expiration),
372 0,
373 1,
374 &put_cb,
375 NULL);
376 if (NULL == qe)
377 {
378 fprintf (stderr, _ ("Error queueing datastore PUT operation\n"));
379 ret = 1;
381 }
382}
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_put(struct GNUNET_DATASTORE_Handle *h, uint32_t rid, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Store an item in the datastore.
@ GNUNET_SYSERR
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh(struct GNUNET_TIME_AbsoluteNBO a)
Convert absolute time from network byte order.
Definition: time.c:737

References _, DataRecord::anonymity, data, datastore, DataRecord::expiration, file_handle, GNUNET_DATASTORE_put(), GNUNET_DISK_file_read(), GNUNET_SCHEDULER_shutdown(), GNUNET_SYSERR, GNUNET_TIME_absolute_ntoh(), DataRecord::key, msg, DataRecord::priority, put_cb(), qe, record_count, DataRecord::replication, ret, DataRecord::size, size, and DataRecord::type.

Referenced by GNUNET_DHT_monitor_start(), put_cb(), and start_insert().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ run()

static void run ( void *  cls,
char *const *  args,
const char *  cfgfile,
const struct GNUNET_CONFIGURATION_Handle cfg 
)
static

Main function that will be run by the scheduler.

Parameters
clsclosure
argsremaining command-line arguments
cfgfilename of the configuration file used
cfgconfiguration

Definition at line 433 of file gnunet-datastore.c.

437{
440 if (NULL == datastore)
441 {
442 fprintf (stderr, _ ("Failed connecting to the datastore.\n"));
443 ret = 1;
445 return;
446 }
447 if (dump)
448 start_dump ();
449 else if (insert)
450 start_insert ();
451 else
452 {
453 fprintf (stderr,
454 _ ("Please choose at least one operation: %s, %s\n"),
455 "dump",
456 "insert");
457 ret = 1;
459 }
460}
static struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition: gnunet-arm.c:109
static int dump
Dump the database.
static void do_shutdown(void *cls)
static void start_dump(void)
Begin dumping the database.
struct GNUNET_DATASTORE_Handle * GNUNET_DATASTORE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the datastore service.
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received,...
Definition: scheduler.c:1334

References _, cfg, datastore, do_shutdown(), dump, GNUNET_DATASTORE_connect(), GNUNET_SCHEDULER_add_shutdown(), GNUNET_SCHEDULER_shutdown(), insert, ret, start_dump(), and start_insert().

Referenced by main().

Here is the call graph for this function:
Here is the caller graph for this function:

◆ main()

int main ( int  argc,
char *const *  argv 
)

The main function to manipulate datastores.

Parameters
argcnumber of arguments from the command line
argvcommand line arguments
Returns
0 ok, 1 on error

Definition at line 471 of file gnunet-datastore.c.

472{
475 "dump",
477 "Dump all records from the datastore"),
478 &dump),
480 "insert",
482 "Insert records into the datastore"),
483 &insert),
485 "file",
486 "FILENAME",
487 gettext_noop ("File to dump or insert"),
488 &file_name),
490
491 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
492 return 2;
493
494 if (GNUNET_OK !=
495 GNUNET_PROGRAM_run (argc,
496 argv,
497 "gnunet-datastore",
498 gettext_noop ("Manipulate GNUnet datastore"),
499 options,
500 &run,
501 NULL))
502 ret = 1;
503 GNUNET_free_nz ((void *) argv);
504 return ret;
505}
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
#define gettext_noop(String)
Definition: gettext.h:70
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
Main function that will be run by the scheduler.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_flag(char shortName, const char *name, const char *description, int *val)
Allow user to specify a flag (which internally means setting an integer to 1/GNUNET_YES/GNUNET_OK.
@ GNUNET_OK
#define GNUNET_free_nz(ptr)
Wrapper around free.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:400
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1230
Definition of a command line option.

References dump, file_name, gettext_noop, GNUNET_free_nz, GNUNET_GETOPT_OPTION_END, GNUNET_GETOPT_option_filename(), GNUNET_GETOPT_option_flag(), GNUNET_OK, GNUNET_PROGRAM_run(), GNUNET_STRINGS_get_utf8_args(), insert, options, ret, and run().

Here is the call graph for this function:

Variable Documentation

◆ MAGIC_LEN

GNUNET_NETWORK_STRUCT_END const size_t MAGIC_LEN = 16
static

Length of our magic header.

Definition at line 76 of file gnunet-datastore.c.

Referenced by start_dump(), and start_insert().

◆ MAGIC_BYTES

const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1"
static

Magic header bytes.

Definition at line 81 of file gnunet-datastore.c.

Referenced by start_dump(), and start_insert().

◆ dump

int dump
static

Dump the database.

Definition at line 86 of file gnunet-datastore.c.

Referenced by GNUNET_CRYPTO_hkdf_v(), main(), and run().

◆ insert

int insert
static

Insert into the database.

Definition at line 91 of file gnunet-datastore.c.

Referenced by get_cb(), main(), and run().

◆ file_name

char* file_name
static

Dump file name.

Definition at line 96 of file gnunet-datastore.c.

Referenced by main(), start_dump(), start_insert(), store_prefix_file_name(), to_file_raw(), and to_file_raw_unaligned().

◆ file_handle

struct GNUNET_DISK_FileHandle* file_handle
static

Dump file handle.

Definition at line 101 of file gnunet-datastore.c.

Referenced by do_shutdown(), get_cb(), put_cb(), start_dump(), and start_insert().

◆ ret

int ret
static

Global return value.

Definition at line 106 of file gnunet-datastore.c.

Referenced by do_get(), get_cb(), main(), put_cb(), run(), start_dump(), and start_insert().

◆ datastore

struct GNUNET_DATASTORE_Handle* datastore
static

Handle for datastore.

Definition at line 111 of file gnunet-datastore.c.

Referenced by do_get(), do_shutdown(), put_cb(), and run().

◆ qe

◆ record_count

uint64_t record_count
static