GNUnet  0.10.x
gnunet-datastore.c
Go to the documentation of this file.
1 /*
2  This file is part of GNUnet.
3  Copyright (C) 2010, 2013 GNUnet e.V.
4 
5  GNUnet is free software: you can redistribute it and/or modify it
6  under the terms of the GNU Affero General Public License as published
7  by the Free Software Foundation, either version 3 of the License,
8  or (at your option) any later version.
9 
10  GNUnet is distributed in the hope that it will be useful, but
11  WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13  Affero General Public License for more details.
14 
15  You should have received a copy of the GNU Affero General Public License
16  along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18  SPDX-License-Identifier: AGPL3.0-or-later
19  */
20 
26 #include <inttypes.h>
27 #include "platform.h"
28 #include "gnunet_util_lib.h"
30 
32 
33 struct DataRecord {
37  uint32_t size GNUNET_PACKED;
38 
42  uint32_t type GNUNET_PACKED;
43 
48 
53 
58 
63 
68 };
70 
71 
75 static const size_t MAGIC_LEN = 16;
76 
80 static const uint8_t MAGIC_BYTES[16] = "GNUNETDATASTORE1";
81 
85 static int dump;
86 
90 static int insert;
91 
95 static char *file_name;
96 
101 
105 static int ret;
106 
111 
116 
120 static uint64_t record_count;
121 
122 
123 static void
124 do_shutdown(void *cls)
125 {
126  if (NULL != qe)
128  if (NULL != datastore)
130  if (NULL != file_handle)
131  GNUNET_DISK_file_close(file_handle);
132 }
133 
134 
138 static void
139 start_dump(void);
140 
141 
145 static void
146 start_insert(void);
147 
148 
152 static void
153 do_get(const uint64_t next_uid);
154 
155 
171 static void
172 get_cb(void *cls,
173  const struct GNUNET_HashCode *key,
174  size_t size,
175  const void *data,
176  enum GNUNET_BLOCK_Type type,
177  uint32_t priority,
178  uint32_t anonymity,
179  uint32_t replication,
181  uint64_t uid)
182 {
183  qe = NULL;
184  if (NULL == key)
185  {
186  fprintf(stderr, _("Dumped %" PRIu64 " records\n"), record_count);
187  GNUNET_DISK_file_close(file_handle);
188  file_handle = NULL;
189  if (insert)
190  start_insert();
191  else
192  {
193  ret = 0;
195  }
196  return;
197  }
198 
199  struct DataRecord dr;
200  dr.size = htonl((uint32_t)size);
201  dr.type = htonl(type);
202  dr.priority = htonl(priority);
203  dr.anonymity = htonl(anonymity);
204  dr.replication = htonl(replication);
205  dr.expiration = GNUNET_TIME_absolute_hton(expiration);
206  dr.key = *key;
207 
208  ssize_t len;
209  len = GNUNET_DISK_file_write(file_handle, &dr, sizeof(dr));
210  if (sizeof(dr) != len)
211  {
212  fprintf(stderr,
213  _("Short write to file: %zd bytes expecting %zd\n"),
214  len,
215  sizeof(dr));
216  ret = 1;
218  return;
219  }
220 
221  len = GNUNET_DISK_file_write(file_handle, data, size);
222  if (size != len)
223  {
224  fprintf(stderr,
225  _("Short write to file: %zd bytes expecting %zd\n"),
226  len,
227  size);
228  ret = 1;
230  return;
231  }
232 
233  record_count++;
234  do_get(uid + 1);
235 }
236 
237 
241 static void
242 do_get(const uint64_t next_uid)
243 {
244  GNUNET_assert(NULL == qe);
245  qe = GNUNET_DATASTORE_get_key(datastore,
246  next_uid,
247  false /* random */,
248  NULL /* key */,
250  0 /* queue_priority */,
251  1 /* max_queue_size */,
252  &get_cb,
253  NULL /* proc_cls */);
254  if (NULL == qe)
255  {
256  fprintf(stderr, _("Error queueing datastore GET operation\n"));
257  ret = 1;
259  }
260 }
261 
262 
266 static void
268 {
269  record_count = 0;
270 
271  if (NULL != file_name)
272  {
273  file_handle = GNUNET_DISK_file_open(file_name,
279  if (NULL == file_handle)
280  {
281  fprintf(stderr, _("Unable to open dump file: %s\n"), file_name);
282  ret = 1;
284  return;
285  }
286  }
287  else
288  {
289  file_handle = GNUNET_DISK_get_handle_from_int_fd(STDOUT_FILENO);
290  }
292  do_get(0);
293 }
294 
295 
309 static void
310 put_cb(void *cls,
311  int32_t success,
313  const char *msg)
314 {
315  qe = NULL;
316  if (GNUNET_SYSERR == success)
317  {
318  fprintf(stderr, _("Failed to store item: %s, aborting\n"), msg);
319  ret = 1;
321  return;
322  }
323 
324  struct DataRecord dr;
325  ssize_t len;
326 
327  len = GNUNET_DISK_file_read(file_handle, &dr, sizeof(dr));
328  if (0 == len)
329  {
330  fprintf(stderr, _("Inserted %" PRIu64 " records\n"), record_count);
331  ret = 0;
333  return;
334  }
335  else if (sizeof(dr) != len)
336  {
337  fprintf(stderr,
338  _("Short read from file: %zd bytes expecting %zd\n"),
339  len,
340  sizeof(dr));
341  ret = 1;
343  return;
344  }
345 
346  const size_t size = ntohl(dr.size);
347  uint8_t data[size];
348  len = GNUNET_DISK_file_read(file_handle, data, size);
349  if (size != len)
350  {
351  fprintf(stderr,
352  _("Short read from file: %zd bytes expecting %zd\n"),
353  len,
354  size);
355  ret = 1;
357  return;
358  }
359 
360  record_count++;
361  qe = GNUNET_DATASTORE_put(datastore,
362  0,
363  &dr.key,
364  size,
365  data,
366  ntohl(dr.type),
367  ntohl(dr.priority),
368  ntohl(dr.anonymity),
369  ntohl(dr.replication),
371  0,
372  1,
373  &put_cb,
374  NULL);
375  if (NULL == qe)
376  {
377  fprintf(stderr, _("Error queueing datastore PUT operation\n"));
378  ret = 1;
380  }
381 }
382 
383 
387 static void
389 {
390  record_count = 0;
391 
392  if (NULL != file_name)
393  {
394  file_handle = GNUNET_DISK_file_open(file_name,
397  if (NULL == file_handle)
398  {
399  fprintf(stderr, _("Unable to open dump file: %s\n"), file_name);
400  ret = 1;
402  return;
403  }
404  }
405  else
406  {
407  file_handle = GNUNET_DISK_get_handle_from_int_fd(STDIN_FILENO);
408  }
409 
410  uint8_t buf[MAGIC_LEN];
411  ssize_t len;
412 
413  len = GNUNET_DISK_file_read(file_handle, buf, MAGIC_LEN);
414  if (len != MAGIC_LEN || 0 != memcmp(buf, MAGIC_BYTES, MAGIC_LEN))
415  {
416  fprintf(stderr, _("Input file is not of a supported format\n"));
417  return;
418  }
420 }
421 
422 
431 static void
432 run(void *cls,
433  char *const *args,
434  const char *cfgfile,
435  const struct GNUNET_CONFIGURATION_Handle *cfg)
436 {
438  datastore = GNUNET_DATASTORE_connect(cfg);
439  if (NULL == datastore)
440  {
441  fprintf(stderr, _("Failed connecting to the datastore.\n"));
442  ret = 1;
444  return;
445  }
446  if (dump)
447  start_dump();
448  else if (insert)
449  start_insert();
450  else
451  {
452  fprintf(stderr,
453  _("Please choose at least one operation: %s, %s\n"),
454  "dump",
455  "insert");
456  ret = 1;
458  }
459 }
460 
461 
469 int
470 main(int argc, char *const *argv)
471 {
472  struct GNUNET_GETOPT_CommandLineOption options[] =
474  "dump",
475  gettext_noop(
476  "Dump all records from the datastore"),
477  &dump),
479  "insert",
480  gettext_noop(
481  "Insert records into the datastore"),
482  &insert),
484  "file",
485  "FILENAME",
486  gettext_noop("File to dump or insert"),
487  &file_name),
489 
490  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args(argc, argv, &argc, &argv))
491  return 2;
492 
493  if (GNUNET_OK !=
494  GNUNET_PROGRAM_run(argc,
495  argv,
496  "gnunet-datastore",
497  gettext_noop("Manipulate GNUnet datastore"),
498  options,
499  &run,
500  NULL))
501  ret = 1;
502  GNUNET_free((void *)argv);
503  return ret;
504 }
505 
506 /* end of gnunet-datastore.c */
static int ret
Global return value.
static struct GNUNET_TIME_Absolute min_expiration
Minimum time that content should have to not be discarded instantly (time stamp of any content that w...
Open the file for reading.
static struct GNUNET_DATASTORE_QueueEntry * qe
Current operation.
Create file if it doesn&#39;t exist.
struct GNUNET_MessageHeader * msg
Definition: 005.c:2
struct GNUNET_TIME_Absolute GNUNET_TIME_absolute_ntoh(struct GNUNET_TIME_AbsoluteNBO a)
Convert absolute time from network byte order.
Definition: time.c:671
int GNUNET_DISK_file_close(struct GNUNET_DISK_FileHandle *h)
Close an open file.
Definition: disk.c:1339
#define GNUNET_TIME_UNIT_ZERO_ABS
Absolute time zero.
Any type of block, used as a wildcard when searching.
GNUNET_BLOCK_Type
Blocks in the datastore and the datacache must have a unique type.
ssize_t GNUNET_DISK_file_read(const struct GNUNET_DISK_FileHandle *h, void *result, size_t len)
Read the contents of a binary file into a buffer.
Definition: disk.c:729
static struct GNUNET_DISK_FileHandle * file_handle
Dump file handle.
uint32_t replication
Desired replication level (NBO).
struct GNUNET_SCHEDULER_Task * GNUNET_SCHEDULER_add_shutdown(GNUNET_SCHEDULER_TaskCallback task, void *task_cls)
Schedule a new task to be run on shutdown, that is when a CTRL-C signal is received, or when GNUNET_SCHEDULER_shutdown() is being invoked.
Definition: scheduler.c:1284
int GNUNET_STRINGS_get_utf8_args(int argc, char *const *argv, int *u8argc, char *const **u8argv)
Returns utf-8 encoded arguments.
Definition: strings.c:1439
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_put(struct GNUNET_DATASTORE_Handle *h, uint32_t rid, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_ContinuationWithStatus cont, void *cont_cls)
Store an item in the datastore.
#define GNUNET_assert(cond)
Use this for fatal errors that cannot be handled.
static char * file_name
Dump file name.
static struct GNUNET_DATASTORE_Handle * datastore
Handle for datastore.
Nobody is allowed to do anything to the file.
uint32_t type
Type of the item (NBO) (actually an enum GNUNET_BLOCK_Type)
#define GNUNET_NO
Definition: gnunet_common.h:78
#define GNUNET_OK
Named constants for return values.
Definition: gnunet_common.h:75
Definition of a command line option.
struct GNUNET_HashCode key
Key under which the item can be found.
void GNUNET_SCHEDULER_shutdown(void)
Request the shutdown of a scheduler.
Definition: scheduler.c:517
Time for absolute time used by GNUnet, in microseconds and in network byte order. ...
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_OPTION_END
Definition: 002.c:13
#define GNUNET_NETWORK_STRUCT_BEGIN
Define as empty, GNUNET_PACKED should suffice, but this won&#39;t work on W32.
static void do_shutdown(void *cls)
#define _(String)
GNU gettext support macro.
Definition: platform.h:181
static uint64_t record_count
Record count.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_filename(char shortName, const char *name, const char *argumentHelp, const char *description, char **str)
Allow user to specify a filename (automatically path expanded).
Entry in our priority queue.
Definition: datastore_api.c:96
Truncate file if it exists.
ssize_t GNUNET_DISK_file_write(const struct GNUNET_DISK_FileHandle *h, const void *buffer, size_t n)
Write a buffer to a file.
Definition: disk.c:817
static void start_insert(void)
Begin inserting into the database.
static char buf[2048]
static GNUNET_NETWORK_STRUCT_END const size_t MAGIC_LEN
Length of our magic header.
static void start_dump(void)
Begin dumping the database.
A 512-bit hashcode.
uint32_t anonymity
Desired anonymity level (NBO).
static void get_cb(void *cls, const struct GNUNET_HashCode *key, size_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
Process a datum that was stored in the datastore.
uint32_t priority
Priority of the item (NBO).
static struct GNUNET_CONFIGURATION_Handle * cfg
Our configuration.
Definition: gnunet-arm.c:104
#define GNUNET_SYSERR
Definition: gnunet_common.h:76
Open the file for writing.
int main(int argc, char *const *argv)
The main function to manipulate datastores.
uint32_t size
Number of bytes in the item (NBO).
#define GNUNET_NETWORK_STRUCT_END
Define as empty, GNUNET_PACKED should suffice, but this won&#39;t work on W32;.
void GNUNET_DATASTORE_cancel(struct GNUNET_DATASTORE_QueueEntry *qe)
Cancel a datastore operation.
struct GNUNET_TIME_AbsoluteNBO expiration
Expiration time (NBO).
#define GNUNET_PACKED
gcc-ism to get packed structs.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_get_handle_from_int_fd(int fno)
Get a handle from a native integer FD.
Definition: disk.c:1369
configuration data
Definition: configuration.c:83
void GNUNET_DATASTORE_disconnect(struct GNUNET_DATASTORE_Handle *h, int drop)
Disconnect from the datastore service (and free associated resources).
Handle to the datastore service.
static void run(void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
Main function that will be run by the scheduler.
struct GNUNET_GETOPT_CommandLineOption GNUNET_GETOPT_option_flag(char shortName, const char *name, const char *description, int *val)
Allow user to specify a flag (which internally means setting an integer to 1/GNUNET_YES/GNUNET_OK.
int GNUNET_PROGRAM_run(int argc, char *const *argv, const char *binaryName, const char *binaryHelp, const struct GNUNET_GETOPT_CommandLineOption *options, GNUNET_PROGRAM_Main task, void *task_cls)
Run a standard GNUnet command startup sequence (initialize loggers and configuration, parse options).
Definition: program.c:367
Time for absolute times used by GNUnet, in microseconds.
#define GNUNET_YES
Definition: gnunet_common.h:77
static void do_get(const uint64_t next_uid)
Perform next GET operation.
struct GNUNET_DATASTORE_QueueEntry * GNUNET_DATASTORE_get_key(struct GNUNET_DATASTORE_Handle *h, uint64_t next_uid, bool random, const struct GNUNET_HashCode *key, enum GNUNET_BLOCK_Type type, unsigned int queue_priority, unsigned int max_queue_size, GNUNET_DATASTORE_DatumProcessor proc, void *proc_cls)
Get a result for a particular key from the datastore.
static int dump
Dump the database.
uint32_t data
The data value.
struct GNUNET_DISK_FileHandle * GNUNET_DISK_file_open(const char *fn, enum GNUNET_DISK_OpenFlags flags, enum GNUNET_DISK_AccessPermissions perm)
Open a file.
Definition: disk.c:1262
struct GNUNET_DATASTORE_Handle * GNUNET_DATASTORE_connect(const struct GNUNET_CONFIGURATION_Handle *cfg)
Connect to the datastore service.
struct GNUNET_TIME_AbsoluteNBO GNUNET_TIME_absolute_hton(struct GNUNET_TIME_Absolute a)
Convert absolute time to network byte order.
Definition: time.c:655
Handle used to access files (and pipes).
static int insert
Insert into the database.
static const uint8_t MAGIC_BYTES[16]
Magic header bytes.
static void put_cb(void *cls, int32_t success, struct GNUNET_TIME_Absolute min_expiration, const char *msg)
Continuation called to notify client about result of the operation.
#define GNUNET_free(ptr)
Wrapper around free.
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
#define gettext_noop(String)
Definition: gettext.h:69