GNUnet  0.11.x
gnunet-datastore.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2010, 2013 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
26 #include <inttypes.h>
27 #include "platform.h"
28 #include "gnunet_util_lib.h"
30 
32 
33 struct DataRecord
34 {
38  uint32_t size GNUNET_PACKED;
39 
43  uint32_t type GNUNET_PACKED;
44 
49 
54 
59 
64 
68  struct GNUNET_HashCode key;
69 };
71 
72 
76 static const size_t MAGIC_LEN = 16;
77 
81 static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
82 
86 static int dump;
87 
91 static int insert;
92 
96 static char *file_name;
97 
102 
106 static int ret;
107 
112 
117 
121 static uint64_t record_count;
122 
123 
124 static void
125 do_shutdown (void *cls)
126 {
127  if (NULL != qe)
129  if (NULL != datastore)
131  if (NULL != file_handle)
133 }
134 
135 
139 static void
140 start_dump (void);
141 
142 
146 static void
147 start_insert (void);
148 
149 
153 static void
154 do_get (const uint64_t next_uid);
155 
156 
172 static void
173 get_cb (void *cls,
174  const struct GNUNET_HashCode *key,
175  size_t size,
176  const void *data,
177  enum GNUNET_BLOCK_Type type,
178  uint32_t priority,
179  uint32_t anonymity,
180  uint32_t replication,
182  uint64_t uid)
183 {
184  qe = NULL;
185  if (NULL == key)
186  {
187  fprintf (stderr, _ ("Dumped %" PRIu64 " records\n"), record_count);
189  file_handle = NULL;
190  if (insert)
191  start_insert ();
192  else
193  {
194  ret = 0;
196  }
197  return;
198  }
199 
200  struct DataRecord dr;
201  dr.size = htonl ((uint32_t) size);
202  dr.type = htonl (type);
203  dr.priority = htonl (priority);
204  dr.anonymity = htonl (anonymity);
205  dr.replication = htonl (replication);
207  dr.key = *key;
208 
209  ssize_t len;
210  len = GNUNET_DISK_file_write (file_handle, &dr, sizeof(dr));
211  if (sizeof(dr) != len)
212  {
213  fprintf (stderr,
214  _ ("Short write to file: %zd bytes expecting %zd\n"),
215  len,
216  sizeof(dr));
217  ret = 1;
219  return;
220  }
221 
223  if (size != len)
224  {
225  fprintf (stderr,
226  _ ("Short write to file: %zd bytes expecting %zd\n"),
227  len,
228  size);
229  ret = 1;
231  return;
232  }
233 
234  record_count++;
235  do_get (uid + 1);
236 }
237 
238 
242 static void
243 do_get (const uint64_t next_uid)
244 {
245  GNUNET_assert (NULL == qe);
247  next_uid,
248  false /* random */,
249  NULL /* key */,
251  0 /* queue_priority */,
252  1 /* max_queue_size */,
253  &get_cb,
254  NULL /* proc_cls */);
255  if (NULL == qe)
256  {
257  fprintf (stderr, _ ("Error queueing datastore GET operation\n"));
258  ret = 1;
260  }
261 }
262 
263 
267 static void
269 {
270  record_count = 0;
271 
272  if (NULL != file_name)
273  {
280  if (NULL == file_handle)
281  {
282  fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
283  ret = 1;
285  return;
286  }
287  }
288  else
289  {
291  }
293  do_get (0);
294 }
295 
296 
310 static void
311 put_cb (void *cls,
312  int32_t success,
314  const char *msg)
315 {
316  qe = NULL;
317  if (GNUNET_SYSERR == success)
318  {
319  fprintf (stderr, _ ("Failed to store item: %s, aborting\n"), msg);
320  ret = 1;
322  return;
323  }
324 
325  struct DataRecord dr;
326  ssize_t len;
327 
328  len = GNUNET_DISK_file_read (file_handle, &dr, sizeof(dr));
329  if (0 == len)
330  {
331  fprintf (stderr, _ ("Inserted %" PRIu64 " records\n"), record_count);
332  ret = 0;
334  return;
335  }
336  else if (sizeof(dr) != len)
337  {
338  fprintf (stderr,
339  _ ("Short read from file: %zd bytes expecting %zd\n"),
340  len,
341  sizeof(dr));
342  ret = 1;
344  return;
345  }
346 
347  const size_t size = ntohl (dr.size);
348  uint8_t data[size];
350  if (size != len)
351  {
352  fprintf (stderr,
353  _ ("Short read from file: %zd bytes expecting %zd\n"),
354  len,
355  size);
356  ret = 1;
358  return;
359  }
360 
361  record_count++;
363  0,
364  &dr.key,
365  size,
366  data,
367  ntohl (dr.type),
368  ntohl (dr.priority),
369  ntohl (dr.anonymity),
370  ntohl (dr.replication),
372  0,
373  1,
374  &put_cb,
375  NULL);
376  if (NULL == qe)
377  {
378  fprintf (stderr, _ ("Error queueing datastore PUT operation\n"));
379  ret = 1;
381  }
382 }
383 
384 
388 static void
390 {
391  record_count = 0;
392 
393  if (NULL != file_name)
394  {
398  if (NULL == file_handle)
399  {
400  fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
401  ret = 1;
403  return;
404  }
405  }
406  else
407  {
409  }
410 
411  uint8_t buf[MAGIC_LEN];
412  ssize_t len;
413 
415  if ((len != MAGIC_LEN) || (0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN)))
416  {
417  fprintf (stderr, _ ("Input file is not of a supported format\n"));
418  return;
419  }
421 }
422 
423 
432 static void
433 run (void *cls,
434  char *const *args,
435  const char *cfgfile,
436  const struct GNUNET_CONFIGURATION_Handle *cfg)
437 {
440  if (NULL == datastore)
441  {
442  fprintf (stderr, _ ("Failed connecting to the datastore.\n"));
443  ret = 1;
445  return;
446  }
447  if (dump)
448  start_dump ();
449  else if (insert)
450  start_insert ();
451  else
452  {
453  fprintf (stderr,
454  _ ("Please choose at least one operation: %s, %s\n"),
455  "dump",
456  "insert");
457  ret = 1;
459  }
460 }
461 
462 
470 int
471 main (int argc, char *const *argv)
472 {
475  "dump",
476  gettext_noop (
477  "Dump all records from the datastore"),
478  &dump),
480  "insert",
481  gettext_noop (
482  "Insert records into the datastore"),
483  &insert),
485  "file",
486  "FILENAME",
487  gettext_noop ("File to dump or insert"),
488  &file_name),
490 
491  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
492  return 2;
493 
494  if (GNUNET_OK !=
495  GNUNET_PROGRAM_run (argc,
496  argv,
497  "gnunet-datastore",
498  gettext_noop ("Manipulate GNUnet datastore"),
499  options,
500  &run,
501  NULL))
502  ret = 1;
503  GNUNET_free_nz ((void *) argv);
504  return ret;
505 }
506 
507 
508 /* end of gnunet-datastore.c */
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
struct GNUNET_GETOPT_CommandLineOption options[]
Definition: 002.c:5
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
#define gettext_noop(String)
Definition: gettext.h:69
static char * expiration
Credential TTL.
Definition: gnunet-abd.c:96
static const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration we are using.
Definition: gnunet-abd.c:36
static unsigned int replication
static void do_get(const uint64_t next_uid)
Perform next GET operation.
static struct GNUNET_DISK_FileHandle * file_handle
Dump file handle.
static uint64_t record_count
Record count.
static const uint8_t MAGIC_BYTES[16]
Magic header bytes.
static int dump
Dump the database.
static struct GNUNET_DATASTORE_QueueEntry * qe
Current operation.
static void get_cb(void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Process a datum that was stored in the datastore.
static int ret
Global return value.
static void do_shutdown(void *cls)
static void put_cb(void *cls, int32_t success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called to notify client about result of the operation.
static void start_insert(void)
Begin inserting into the database.
static char * file_name
Dump file name.
static struct GNUNET_DATASTORE_Handle * datastore
Handle for datastore.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
Main function that will be run by the scheduler.
static int insert
Insert into the database.
static GNUNET_NETWORK_STRUCT_END const size_t MAGIC_LEN
Length of our magic header.
int main(int argc, char *const *argv)
The main function to manipulate datastores.
static void start_dump(void)
Begin dumping the database.
struct GNUNET_HashCode key
The key used in the DHT.
static unsigned int anonymity
uint32_t data
The data value.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
static struct GNUNET_TIME_Absolute min_expiration
Minimum time that content should have to not be discarded instantly (time stamp of any content that w...
static char buf[2048]
#define GNUNET_NETWORK_STRUCT_BEGIN
Define as empty, GNUNET_PACKED should suffice, but this won't work on W32.
#define GNUNET_NETWORK_STRUCT_END
Define as empty, GNUNET_PACKED should suffice, but this won't work on W32;.
@ GNUNET_OK
Definition: gnunet_common.h:95
@ GNUNET_YES
Definition: gnunet_common.h:97
@ GNUNET_NO
Definition: gnunet_common.h:94
@ GNUNET_SYSERR
Definition: gnunet_common.h:93
#define GNUNET_PACKED
gcc-ism to get packed structs.
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
@ GNUNET_BLOCK_TYPE_ANY
Any type of block, used as a wildcard when searching.
struct GNUNET_DATASTORE_Handle * GNUNET_DATASTORE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the datastore service.
void GNUNET_DATASTORE_disconnect(struct GNUNET_DATASTORE_Handle *h, int drop)
Disconnect from the datastore service (and free associated resources).
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_get_key(struct GNUNET_DATASTORE_Handle *h, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_DatumProcessor proc, void *proc_cls)
Get a result for a particular key from the datastore.
void GNUNET_DATASTORE_cancel(struct GNUNET_DATASTORE_QueueEntry *qe)
Cancel a datastore operation.
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_put(struct GNUNET_DATASTORE_Handle *h, uint32_t rid, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Store an item in the datastore.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1235
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:686
enum GNUNET_GenericReturnValue GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1306
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:622
struct GNUNET_DISK_FileHandle * GNUNET_DISK_get_handle_from_int_fd(int fno)
Get a handle from a native integer FD.
Definition: disk.c:1328
@ GNUNET_DISK_OPEN_READ
Open the file for reading.
@ GNUNET_DISK_OPEN_WRITE
Open the file for writing.
@ GNUNET_DISK_OPEN_TRUNCATE
Truncate file if it exists.
@ GNUNET_DISK_OPEN_CREATE
Create file if it doesn't exist.
@ GNUNET_DISK_PERM_USER_READ
Owner can read.
@ GNUNET_DISK_PERM_NONE
Nobody is allowed to do anything to the file.
@ GNUNET_DISK_PERM_USER_WRITE
Owner can write.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_flag(char shortName, const char *name, const char *description, int *val)
Allow user to specify a flag (which internally means setting an integer to 1/GNUNET_YES/GNUNET_OK.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
#define GNUNET_free_nz(ptr)
Wrapper around free.
enum GNUNET_GenericReturnValue GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration,...
Definition: program.c:364
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:531
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received,...
Definition: scheduler.c:1331
enum GNUNET_GenericReturnValue GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1209
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh(struct GNUNET_TIME_AbsoluteNBO a)
Convert absolute time from network byte order.
Definition: time.c:542
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.
struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton(struct GNUNET_TIME_Absolute a)
Convert absolute time to network byte order.
Definition: time.c:464
static unsigned int size
Size of the "table".
Definition: peer.c:67
#define _(String)
GNU gettext support macro.
Definition: platform.h:177
uint32_t replication
Desired replication level (NBO).
struct GNUNET_TIME_AbsoluteNBO expiration
Expiration time (NBO).
uint32_t type
Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
uint32_t anonymity
Desired anonymity level (NBO).
uint32_t size
Number of bytes in the item (NBO).
uint32_t priority
Priority of the item (NBO).
struct GNUNET_HashCode key
Key under which the item can be found.
Handle to the datastore service.
Entry in our priority queue.
unsigned int priority
Priority in the queue.
Handle used to access files (and pipes).
Definition of a command line option.
A 512-bit hashcode.
Time for absolute time used by GNUnet, in microseconds and in network byte order.
Time for absolute times used by GNUnet, in microseconds.
enum GNUNET_TESTBED_UnderlayLinkModelType type
the type of this model