GNUnet  0.10.x
gnunet-datastore.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2010, 2013 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19 */
20 
26 #include <inttypes.h>
27 #include "platform.h"
28 #include "gnunet_util_lib.h"
30 
32 
33 struct DataRecord
34 {
38  uint32_t size GNUNET_PACKED;
39 
43  uint32_t type GNUNET_PACKED;
44 
49 
54 
59 
64 
69 
70 };
72 
73 
77 static const size_t MAGIC_LEN = 16;
78 
82 static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
83 
87 static int dump;
88 
92 static int insert;
93 
97 static char *file_name;
98 
103 
107 static int ret;
108 
113 
118 
122 static uint64_t record_count;
123 
124 
125 static void
126 do_shutdown (void *cls)
127 {
128  if (NULL != qe)
130  if (NULL != datastore)
132  if (NULL != file_handle)
133  GNUNET_DISK_file_close (file_handle);
134 }
135 
136 
140 static void
141 start_dump (void);
142 
143 
147 static void
148 start_insert (void);
149 
150 
154 static void
155 do_get (const uint64_t next_uid);
156 
157 
173 static void
174 get_cb (void *cls,
175  const struct GNUNET_HashCode *key,
176  size_t size,
177  const void *data,
178  enum GNUNET_BLOCK_Type type,
179  uint32_t priority,
180  uint32_t anonymity,
181  uint32_t replication,
183  uint64_t uid)
184 {
185  qe = NULL;
186  if (NULL == key)
187  {
188  FPRINTF (stderr,
189  _("Dumped %" PRIu64 " records\n"),
190  record_count);
191  GNUNET_DISK_file_close (file_handle);
192  file_handle = NULL;
193  if (insert)
194  start_insert();
195  else
196  {
197  ret = 0;
199  }
200  return;
201  }
202 
203  struct DataRecord dr;
204  dr.size = htonl ((uint32_t) size);
205  dr.type = htonl (type);
206  dr.priority = htonl (priority);
207  dr.anonymity = htonl (anonymity);
208  dr.replication = htonl (replication);
209  dr.expiration = GNUNET_TIME_absolute_hton (expiration);
210  dr.key = *key;
211 
212  ssize_t len;
213  len = GNUNET_DISK_file_write (file_handle, &dr, sizeof (dr));
214  if (sizeof (dr) != len)
215  {
216  FPRINTF (stderr,
217  _("Short write to file: %zd bytes expecting %zd\n"),
218  len,
219  sizeof (dr));
220  ret = 1;
222  return;
223  }
224 
225  len = GNUNET_DISK_file_write (file_handle, data, size);
226  if (size != len)
227  {
228  FPRINTF (stderr,
229  _("Short write to file: %zd bytes expecting %zd\n"),
230  len,
231  size);
232  ret = 1;
234  return;
235  }
236 
237  record_count++;
238  do_get(uid + 1);
239 }
240 
241 
245 static void
246 do_get (const uint64_t next_uid)
247 {
248  GNUNET_assert (NULL == qe);
249  qe = GNUNET_DATASTORE_get_key (datastore,
250  next_uid,
251  false /* random */,
252  NULL /* key */,
254  0 /* queue_priority */,
255  1 /* max_queue_size */,
256  &get_cb,
257  NULL /* proc_cls */);
258  if (NULL == qe)
259  {
260  FPRINTF (stderr,
261  _("Error queueing datastore GET operation\n"));
262  ret = 1;
264  }
265 }
266 
267 
271 static void
273 {
274  record_count = 0;
275 
276  if (NULL != file_name)
277  {
278  file_handle = GNUNET_DISK_file_open (file_name,
284  if (NULL == file_handle)
285  {
286  FPRINTF (stderr,
287  _("Unable to open dump file: %s\n"),
288  file_name);
289  ret = 1;
291  return;
292  }
293  }
294  else
295  {
296  file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
297  }
299  do_get(0);
300 }
301 
302 
316 static void
317 put_cb (void *cls,
318  int32_t success,
320  const char *msg)
321 {
322  qe = NULL;
323  if (GNUNET_SYSERR == success)
324  {
325  FPRINTF (stderr,
326  _("Failed to store item: %s, aborting\n"),
327  msg);
328  ret = 1;
330  return;
331  }
332 
333  struct DataRecord dr;
334  ssize_t len;
335 
336  len = GNUNET_DISK_file_read (file_handle, &dr, sizeof (dr));
337  if (0 == len)
338  {
339  FPRINTF (stderr,
340  _("Inserted %" PRIu64 " records\n"),
341  record_count);
342  ret = 0;
344  return;
345  }
346  else if (sizeof (dr) != len)
347  {
348  FPRINTF (stderr,
349  _("Short read from file: %zd bytes expecting %zd\n"),
350  len,
351  sizeof (dr));
352  ret = 1;
354  return;
355  }
356 
357  const size_t size = ntohl (dr.size);
358  uint8_t data[size];
359  len = GNUNET_DISK_file_read (file_handle, data, size);
360  if (size != len)
361  {
362  FPRINTF (stderr,
363  _("Short read from file: %zd bytes expecting %zd\n"),
364  len,
365  size);
366  ret = 1;
368  return;
369  }
370 
371  record_count++;
372  qe = GNUNET_DATASTORE_put (datastore,
373  0,
374  &dr.key,
375  size,
376  data,
377  ntohl (dr.type),
378  ntohl (dr.priority),
379  ntohl (dr.anonymity),
380  ntohl (dr.replication),
382  0,
383  1,
384  &put_cb,
385  NULL);
386  if (NULL == qe)
387  {
388  FPRINTF (stderr,
389  _("Error queueing datastore PUT operation\n"));
390  ret = 1;
392  }
393 }
394 
395 
399 static void
401 {
402  record_count = 0;
403 
404  if (NULL != file_name)
405  {
406  file_handle = GNUNET_DISK_file_open (file_name,
409  if (NULL == file_handle)
410  {
411  FPRINTF (stderr,
412  _("Unable to open dump file: %s\n"),
413  file_name);
414  ret = 1;
416  return;
417  }
418  }
419  else
420  {
421  file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
422  }
423 
424  uint8_t buf[MAGIC_LEN];
425  ssize_t len;
426 
427  len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
428  if (len != MAGIC_LEN ||
429  0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN))
430  {
431  FPRINTF (stderr,
432  _("Input file is not of a supported format\n"));
433  return;
434  }
436 }
437 
438 
447 static void
448 run (void *cls,
449  char *const *args,
450  const char *cfgfile,
451  const struct GNUNET_CONFIGURATION_Handle *cfg)
452 {
454  datastore = GNUNET_DATASTORE_connect (cfg);
455  if (NULL == datastore)
456  {
457  FPRINTF (stderr,
458  _("Failed connecting to the datastore.\n"));
459  ret = 1;
461  return;
462  }
463  if (dump)
464  start_dump();
465  else if (insert)
466  start_insert();
467  else
468  {
469  FPRINTF (stderr,
470  _("Please choose at least one operation: %s, %s\n"),
471  "dump",
472  "insert");
473  ret = 1;
475  }
476 }
477 
478 
486 int
487 main (int argc,
488  char *const *argv)
489 {
490  struct GNUNET_GETOPT_CommandLineOption options[] = {
492  "dump",
493  gettext_noop ("Dump all records from the datastore"),
494  &dump),
496  "insert",
497  gettext_noop ("Insert records into the datastore"),
498  &insert),
500  "file",
501  "FILENAME",
502  gettext_noop ("File to dump or insert"),
503  &file_name),
505  };
506  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
507  return 2;
508 
509  if (GNUNET_OK !=
510  GNUNET_PROGRAM_run (argc, argv, "gnunet-datastore",
511  gettext_noop ("Manipulate GNUnet datastore"),
512  options, &run, NULL))
513  ret = 1;
514  GNUNET_free ((void*) argv);
515  return ret;
516 }
517 
518 /* end of gnunet-datastore.c */
static int ret
Global return value.
static struct GNUNET_TIME_Absolute min_expiration
Minimum time that content should have to not be discarded instantly (time stamp of any content that w...
Open the file for reading.
static struct GNUNET_DATASTORE_QueueEntry * qe
Current operation.
Create file if it doesn&#39;t exist.
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh(struct GNUNET_TIME_AbsoluteNBO a)
Convert absolute time from network byte order.
Definition: time.c:670
int GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1817
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.
Any type of block, used as a wildcard when searching.
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
#define FPRINTF
Definition: plibc.h:683
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:881
static struct GNUNET_DISK_FileHandle * file_handle
Dump file handle.
uint32_t replication
Desired replication level (NBO).
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received, or when GNUNET_SCHEDULER_shutdown() is being invoked.
Definition: scheduler.c:1293
int GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1521
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_put(struct GNUNET_DATASTORE_Handle *h, uint32_t rid, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Store an item in the datastore.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static char * file_name
Dump file name.
static struct GNUNET_DATASTORE_Handle * datastore
Handle for datastore.
Nobody is allowed to do anything to the file.
uint32_t type
Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
#define GNUNET_NO
Definition: gnunet_common.h:81
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:78
Definition of a command line option.
struct GNUNET_HashCode key
Key under which the item can be found.
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:524
Time for absolute time used by GNUnet, in microseconds and in network byte order. ...
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
#define GNUNET_NETWORK_STRUCT_BEGIN
Define as empty, GNUNET_PACKED should suffice, but this won&#39;t work on W32.
static void do_shutdown(void *cls)
#define _(String)
GNU gettext support macro.
Definition: platform.h:208
static uint64_t record_count
Record count.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
Entry in our priority queue.
Truncate file if it exists.
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:1049
static void start_insert(void)
Begin inserting into the database.
static char buf[2048]
static GNUNET_NETWORK_STRUCT_END const size_t MAGIC_LEN
Length of our magic header.
static void start_dump(void)
Begin dumping the database.
A 512-bit hashcode.
uint32_t anonymity
Desired anonymity level (NBO).
static void get_cb(void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Process a datum that was stored in the datastore.
uint32_t priority
Priority of the item (NBO).
static struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition: gnunet-arm.c:104
#define GNUNET_SYSERR
Definition: gnunet_common.h:79
Open the file for writing.
int main(int argc, char *const *argv)
The main function to manipulate datastores.
uint32_t size
Number of bytes in the item (NBO).
#define GNUNET_NETWORK_STRUCT_END
Define as empty, GNUNET_PACKED should suffice, but this won&#39;t work on W32;.
void GNUNET_DATASTORE_cancel(struct GNUNET_DATASTORE_QueueEntry *qe)
Cancel a datastore operation.
struct GNUNET_TIME_AbsoluteNBO expiration
Expiration time (NBO).
#define GNUNET_PACKED
gcc-ism to get packed structs.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_get_handle_from_int_fd(int fno)
Get a handle from a native integer FD.
Definition: disk.c:1937
configuration data
Definition: configuration.c:85
void GNUNET_DATASTORE_disconnect(struct GNUNET_DATASTORE_Handle *h, int drop)
Disconnect from the datastore service (and free associated resources).
Handle to the datastore service.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
Main function that will be run by the scheduler.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_flag(char shortName, const char *name, const char *description, int *val)
Allow user to specify a flag (which internally means setting an integer to 1/GNUNET_YES/GNUNET_OK.
int GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration, parse options).
Definition: program.c:361
Time for absolute times used by GNUnet, in microseconds.
#define GNUNET_YES
Definition: gnunet_common.h:80
static void do_get(const uint64_t next_uid)
Perform next GET operation.
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_get_key(struct GNUNET_DATASTORE_Handle *h, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_DatumProcessor proc, void *proc_cls)
Get a result for a particular key from the datastore.
static int dump
Dump the database.
uint32_t data
The data value.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1673
struct GNUNET_DATASTORE_Handle * GNUNET_DATASTORE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the datastore service.
struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton(struct GNUNET_TIME_Absolute a)
Convert absolute time to network byte order.
Definition: time.c:654
Handle used to access files (and pipes).
static int insert
Insert into the database.
static const uint8_t MAGIC_BYTES[16]
Magic header bytes.
static void put_cb(void *cls, int32_t success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called to notify client about result of the operation.
#define GNUNET_free(ptr)
Wrapper around free.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
#define gettext_noop(String)
Definition: gettext.h:69