GNUnet  0.11.x
gnunet-datastore.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2010, 2013 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
26 #include <inttypes.h>
27 #include "platform.h"
28 #include "gnunet_util_lib.h"
30 
32 
33 struct DataRecord
34 {
38  uint32_t size GNUNET_PACKED;
39 
43  uint32_t type GNUNET_PACKED;
44 
49 
54 
59 
64 
69 };
71 
72 
76 static const size_t MAGIC_LEN = 16;
77 
81 static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
82 
86 static int dump;
87 
91 static int insert;
92 
96 static char *file_name;
97 
102 
106 static int ret;
107 
112 
117 
121 static uint64_t record_count;
122 
123 
124 static void
125 do_shutdown (void *cls)
126 {
127  if (NULL != qe)
129  if (NULL != datastore)
131  if (NULL != file_handle)
132  GNUNET_DISK_file_close (file_handle);
133 }
134 
135 
139 static void
140 start_dump (void);
141 
142 
146 static void
147 start_insert (void);
148 
149 
153 static void
154 do_get (const uint64_t next_uid);
155 
156 
172 static void
173 get_cb (void *cls,
174  const struct GNUNET_HashCode *key,
175  size_t size,
176  const void *data,
177  enum GNUNET_BLOCK_Type type,
178  uint32_t priority,
179  uint32_t anonymity,
180  uint32_t replication,
182  uint64_t uid)
183 {
184  qe = NULL;
185  if (NULL == key)
186  {
187  fprintf (stderr, _ ("Dumped %" PRIu64 " records\n"), record_count);
188  GNUNET_DISK_file_close (file_handle);
189  file_handle = NULL;
190  if (insert)
191  start_insert ();
192  else
193  {
194  ret = 0;
196  }
197  return;
198  }
199 
200  struct DataRecord dr;
201  dr.size = htonl ((uint32_t) size);
202  dr.type = htonl (type);
203  dr.priority = htonl (priority);
204  dr.anonymity = htonl (anonymity);
205  dr.replication = htonl (replication);
206  dr.expiration = GNUNET_TIME_absolute_hton (expiration);
207  dr.key = *key;
208 
209  ssize_t len;
210  len = GNUNET_DISK_file_write (file_handle, &dr, sizeof(dr));
211  if (sizeof(dr) != len)
212  {
213  fprintf (stderr,
214  _ ("Short write to file: %zd bytes expecting %zd\n"),
215  len,
216  sizeof(dr));
217  ret = 1;
219  return;
220  }
221 
222  len = GNUNET_DISK_file_write (file_handle, data, size);
223  if (size != len)
224  {
225  fprintf (stderr,
226  _ ("Short write to file: %zd bytes expecting %zd\n"),
227  len,
228  size);
229  ret = 1;
231  return;
232  }
233 
234  record_count++;
235  do_get (uid + 1);
236 }
237 
238 
242 static void
243 do_get (const uint64_t next_uid)
244 {
245  GNUNET_assert (NULL == qe);
246  qe = GNUNET_DATASTORE_get_key (datastore,
247  next_uid,
248  false /* random */,
249  NULL /* key */,
251  0 /* queue_priority */,
252  1 /* max_queue_size */,
253  &get_cb,
254  NULL /* proc_cls */);
255  if (NULL == qe)
256  {
257  fprintf (stderr, _ ("Error queueing datastore GET operation\n"));
258  ret = 1;
260  }
261 }
262 
263 
267 static void
269 {
270  record_count = 0;
271 
272  if (NULL != file_name)
273  {
274  file_handle = GNUNET_DISK_file_open (file_name,
280  if (NULL == file_handle)
281  {
282  fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
283  ret = 1;
285  return;
286  }
287  }
288  else
289  {
290  file_handle = GNUNET_DISK_get_handle_from_int_fd (STDOUT_FILENO);
291  }
293  do_get (0);
294 }
295 
296 
310 static void
311 put_cb (void *cls,
312  int32_t success,
314  const char *msg)
315 {
316  qe = NULL;
317  if (GNUNET_SYSERR == success)
318  {
319  fprintf (stderr, _ ("Failed to store item: %s, aborting\n"), msg);
320  ret = 1;
322  return;
323  }
324 
325  struct DataRecord dr;
326  ssize_t len;
327 
328  len = GNUNET_DISK_file_read (file_handle, &dr, sizeof(dr));
329  if (0 == len)
330  {
331  fprintf (stderr, _ ("Inserted %" PRIu64 " records\n"), record_count);
332  ret = 0;
334  return;
335  }
336  else if (sizeof(dr) != len)
337  {
338  fprintf (stderr,
339  _ ("Short read from file: %zd bytes expecting %zd\n"),
340  len,
341  sizeof(dr));
342  ret = 1;
344  return;
345  }
346 
347  const size_t size = ntohl (dr.size);
348  uint8_t data[size];
349  len = GNUNET_DISK_file_read (file_handle, data, size);
350  if (size != len)
351  {
352  fprintf (stderr,
353  _ ("Short read from file: %zd bytes expecting %zd\n"),
354  len,
355  size);
356  ret = 1;
358  return;
359  }
360 
361  record_count++;
362  qe = GNUNET_DATASTORE_put (datastore,
363  0,
364  &dr.key,
365  size,
366  data,
367  ntohl (dr.type),
368  ntohl (dr.priority),
369  ntohl (dr.anonymity),
370  ntohl (dr.replication),
372  0,
373  1,
374  &put_cb,
375  NULL);
376  if (NULL == qe)
377  {
378  fprintf (stderr, _ ("Error queueing datastore PUT operation\n"));
379  ret = 1;
381  }
382 }
383 
384 
388 static void
390 {
391  record_count = 0;
392 
393  if (NULL != file_name)
394  {
395  file_handle = GNUNET_DISK_file_open (file_name,
398  if (NULL == file_handle)
399  {
400  fprintf (stderr, _ ("Unable to open dump file: %s\n"), file_name);
401  ret = 1;
403  return;
404  }
405  }
406  else
407  {
408  file_handle = GNUNET_DISK_get_handle_from_int_fd (STDIN_FILENO);
409  }
410 
411  uint8_t buf[MAGIC_LEN];
412  ssize_t len;
413 
414  len = GNUNET_DISK_file_read (file_handle, buf, MAGIC_LEN);
415  if ((len != MAGIC_LEN) || (0 != memcmp (buf, MAGIC_BYTES, MAGIC_LEN)))
416  {
417  fprintf (stderr, _ ("Input file is not of a supported format\n"));
418  return;
419  }
421 }
422 
423 
432 static void
433 run (void *cls,
434  char *const *args,
435  const char *cfgfile,
436  const struct GNUNET_CONFIGURATION_Handle *cfg)
437 {
439  datastore = GNUNET_DATASTORE_connect (cfg);
440  if (NULL == datastore)
441  {
442  fprintf (stderr, _ ("Failed connecting to the datastore.\n"));
443  ret = 1;
445  return;
446  }
447  if (dump)
448  start_dump ();
449  else if (insert)
450  start_insert ();
451  else
452  {
453  fprintf (stderr,
454  _ ("Please choose at least one operation: %s, %s\n"),
455  "dump",
456  "insert");
457  ret = 1;
459  }
460 }
461 
462 
470 int
471 main (int argc, char *const *argv)
472 {
473  struct GNUNET_GETOPT_CommandLineOption options[] =
475  "dump",
476  gettext_noop (
477  "Dump all records from the datastore"),
478  &dump),
480  "insert",
481  gettext_noop (
482  "Insert records into the datastore"),
483  &insert),
485  "file",
486  "FILENAME",
487  gettext_noop ("File to dump or insert"),
488  &file_name),
490 
491  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
492  return 2;
493 
494  if (GNUNET_OK !=
495  GNUNET_PROGRAM_run (argc,
496  argv,
497  "gnunet-datastore",
498  gettext_noop ("Manipulate GNUnet datastore"),
499  options,
500  &run,
501  NULL))
502  ret = 1;
503  GNUNET_free ((void *) argv);
504  return ret;
505 }
506 
507 
508 /* end of gnunet-datastore.c */
static int ret
Global return value.
static struct GNUNET_TIME_Absolute min_expiration
Minimum time that content should have to not be discarded instantly (time stamp of any content that w...
Open the file for reading.
static struct GNUNET_DATASTORE_QueueEntry * qe
Current operation.
Create file if it doesn&#39;t exist.
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh(struct GNUNET_TIME_AbsoluteNBO a)
Convert absolute time from network byte order.
Definition: time.c:673
int GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1345
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.
static const struct GNUNET_CONFIGURATION_Handle * cfg
Configuration we are using.
Definition: gnunet-abd.c:36
Any type of block, used as a wildcard when searching.
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:732
static struct GNUNET_DISK_FileHandle * file_handle
Dump file handle.
uint32_t replication
Desired replication level (NBO).
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received, or when GNUNET_SCHEDULER_shutdown() is being invoked.
Definition: scheduler.c:1300
int GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1438
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_put(struct GNUNET_DATASTORE_Handle *h, uint32_t rid, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Store an item in the datastore.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static char * file_name
Dump file name.
static struct GNUNET_DATASTORE_Handle * datastore
Handle for datastore.
Nobody is allowed to do anything to the file.
uint32_t type
Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
Definition of a command line option.
struct GNUNET_HashCode key
Key under which the item can be found.
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:526
Time for absolute time used by GNUnet, in microseconds and in network byte order. ...
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
#define GNUNET_NETWORK_STRUCT_BEGIN
Define as empty, GNUNET_PACKED should suffice, but this won&#39;t work on W32.
static void do_shutdown(void *cls)
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
static uint64_t record_count
Record count.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
Entry in our priority queue.
Definition: datastore_api.c:99
Truncate file if it exists.
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:820
static void start_insert(void)
Begin inserting into the database.
static char buf[2048]
static GNUNET_NETWORK_STRUCT_END const size_t MAGIC_LEN
Length of our magic header.
static void start_dump(void)
Begin dumping the database.
A 512-bit hashcode.
uint32_t anonymity
Desired anonymity level (NBO).
static void get_cb(void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Process a datum that was stored in the datastore.
uint32_t priority
Priority of the item (NBO).
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
Open the file for writing.
int main(int argc, char *const *argv)
The main function to manipulate datastores.
uint32_t size
Number of bytes in the item (NBO).
#define GNUNET_NETWORK_STRUCT_END
Define as empty, GNUNET_PACKED should suffice, but this won&#39;t work on W32;.
void GNUNET_DATASTORE_cancel(struct GNUNET_DATASTORE_QueueEntry *qe)
Cancel a datastore operation.
struct GNUNET_TIME_AbsoluteNBO expiration
Expiration time (NBO).
#define GNUNET_PACKED
gcc-ism to get packed structs.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_get_handle_from_int_fd(int fno)
Get a handle from a native integer FD.
Definition: disk.c:1375
configuration data
Definition: configuration.c:85
void GNUNET_DATASTORE_disconnect(struct GNUNET_DATASTORE_Handle *h, int drop)
Disconnect from the datastore service (and free associated resources).
Handle to the datastore service.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
Main function that will be run by the scheduler.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_flag(char shortName, const char *name, const char *description, int *val)
Allow user to specify a flag (which internally means setting an integer to 1/GNUNET_YES/GNUNET_OK.
int GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration, parse options).
Definition: program.c:367
Time for absolute times used by GNUnet, in microseconds.
#define GNUNET_YES
Definition: gnunet_common.h:77
static void do_get(const uint64_t next_uid)
Perform next GET operation.
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_get_key(struct GNUNET_DATASTORE_Handle *h, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_DatumProcessor proc, void *proc_cls)
Get a result for a particular key from the datastore.
static int dump
Dump the database.
uint32_t data
The data value.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1268
struct GNUNET_DATASTORE_Handle * GNUNET_DATASTORE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the datastore service.
struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton(struct GNUNET_TIME_Absolute a)
Convert absolute time to network byte order.
Definition: time.c:657
Handle used to access files (and pipes).
static int insert
Insert into the database.
static const uint8_t MAGIC_BYTES[16]
Magic header bytes.
static void put_cb(void *cls, int32_t success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called to notify client about result of the operation.
#define GNUNET_free(ptr)
Wrapper around free.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
#define gettext_noop(String)
Definition: gettext.h:69