gnunet-service-datastore.c File Reference

Management for the datastore for files stored on a GNUnet node. More...

#include "platform.h"
#include "gnunet_util_lib.h"
#include "gnunet_protocols.h"
#include "gnunet_statistics_service.h"
#include "gnunet_datastore_plugin.h"
#include "datastore.h"

Include dependency graph for gnunet-service-datastore.c:

Go to the source code of this file.

Data Structures
struct	DatastorePlugin
	Our datastore plugin. More...
struct	ReservationList
	Linked list of active reservations. More...

Macros
#define	MAX_PENDING   1024
	How many messages do we queue at most per client?
#define	MAX_BF_SIZE   ((uint32_t) (1LL << 31))
	Limit size of bloom filter to 2 GB.
#define	MAX_EXPIRE_DELAY    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
	How long are we at most keeping "expired" content past the expiration date in the database?
#define	MIN_EXPIRE_DELAY    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
	How fast are we allowed to query the database for deleting expired content? (1 item per second).
#define	MAX_STAT_SYNC_LAG   50
	After how many payload-changing operations do we sync our statistics?

Functions
static void	sync_stats ()
	Synchronize our utilization statistics with the statistics service.
static void	delete_expired (void *cls)
	Task that is used to remove expired entries from the datastore.
static int	expired_processor (void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
	Iterate over the expired items stored in the datastore.
static int	quota_processor (void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
	An iterator over a set of items stored in the datastore that deletes until we're happy with respect to our quota.
static void	manage_space (unsigned long long need)
	Manage available disk space by running tasks that will discard content if necessary.
static void	transmit_status (struct GNUNET_SERVICE_Client *client, int code, const char *msg)
	Transmit a status code to the client.
static int	transmit_item (void *cls, const struct GNUNET_HashCode *key, uint32_t size, const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, uint32_t anonymity, uint32_t replication, struct GNUNET_TIME_Absolute expiration, uint64_t uid)
	Function that will transmit the given datastore entry to the client.
static void	handle_reserve (void *cls, const struct ReserveMessage *msg)
	Handle RESERVE-message.
static void	handle_release_reserve (void *cls, const struct ReleaseReserveMessage *msg)
	Handle RELEASE_RESERVE-message.
static int	check_data (const struct DataMessage *dm)
	Check that the given message is a valid data message.
static void	put_continuation (void *cls, const struct GNUNET_HashCode *key, uint32_t size, int status, const char *msg)
	Put continuation.
static int	check_put (void *cls, const struct DataMessage *dm)
	Verify PUT-message.
static void	handle_put (void *cls, const struct DataMessage *dm)
	Handle PUT-message.
static void	handle_get (void *cls, const struct GetMessage *msg)
	Handle GNUNET_MESSAGE_TYPE_DATASTORE_GET-message.
static void	handle_get_key (void *cls, const struct GetKeyMessage *msg)
	Handle GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY-message.
static void	handle_get_replication (void *cls, const struct GNUNET_MessageHeader *message)
	Handle GET_REPLICATION-message.
static void	handle_get_zero_anonymity (void *cls, const struct GetZeroAnonymityMessage *msg)
	Handle GET_ZERO_ANONYMITY-message.
static void	remove_continuation (void *cls, const struct GNUNET_HashCode *key, uint32_t size, int status, const char *msg)
	Remove continuation.
static int	check_remove (void *cls, const struct DataMessage *dm)
	Verify REMOVE-message.
static void	handle_remove (void *cls, const struct DataMessage *dm)
	Handle REMOVE-message.
static void	handle_drop (void *cls, const struct GNUNET_MessageHeader *message)
	Handle DROP-message.
static void	disk_utilization_change_cb (void *cls, int delta)
	Function called by plugins to notify us about a change in their disk utilization.
static int	process_stat_in (void *cls, const char *subsystem, const char *name, uint64_t value, int is_persistent)
	Callback function to process statistic values.
static struct DatastorePlugin *	load_plugin ()
	Load the datastore plugin.
static void	unload_plugin (struct DatastorePlugin *plug)
	Function called when the service shuts down.
static void	begin_service ()
	Initialization complete, start operating the service.
static void	add_key_to_bloomfilter (void *cls, const struct GNUNET_HashCode *key, unsigned int count)
	Adds a given key to the bloomfilter in cls count times.
static void	process_stat_done (void *cls, int success)
	We finished receiving the statistic.
static void	stat_timeout (void *cls)
	Fetching stats took to long, run without.
static void	cleaning_task (void *cls)
	Task run during shutdown.
static void *	client_connect_cb (void *cls, struct GNUNET_SERVICE_Client *client, struct GNUNET_MQ_Handle *mq)
	Add a client to our list of active clients.
static void	client_disconnect_cb (void *cls, struct GNUNET_SERVICE_Client *client, void *app_ctx)
	Called whenever a client is disconnected.
static void	run (void *cls, const struct GNUNET_CONFIGURATION_Handle *c, struct GNUNET_SERVICE_Handle *serv)
	Process datastore requests.
	GNUNET_SERVICE_MAIN (GNUNET_OS_project_data_gnunet(), "datastore", GNUNET_SERVICE_OPTION_NONE, &run, &client_connect_cb, &client_disconnect_cb, NULL, GNUNET_MQ_hd_fixed_size(reserve, GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE, struct ReserveMessage, NULL), GNUNET_MQ_hd_fixed_size(release_reserve, GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE, struct ReleaseReserveMessage, NULL), GNUNET_MQ_hd_var_size(put, GNUNET_MESSAGE_TYPE_DATASTORE_PUT, struct DataMessage, NULL), GNUNET_MQ_hd_fixed_size(get, GNUNET_MESSAGE_TYPE_DATASTORE_GET, struct GetMessage, NULL), GNUNET_MQ_hd_fixed_size(get_key, GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY, struct GetKeyMessage, NULL), GNUNET_MQ_hd_fixed_size(get_replication, GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION, struct GNUNET_MessageHeader, NULL), GNUNET_MQ_hd_fixed_size(get_zero_anonymity, GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY, struct GetZeroAnonymityMessage, NULL), GNUNET_MQ_hd_var_size(remove, GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE, struct DataMessage, NULL), GNUNET_MQ_hd_fixed_size(drop, GNUNET_MESSAGE_TYPE_DATASTORE_DROP, struct GNUNET_MessageHeader, NULL), GNUNET_MQ_handler_end())
	Define "main" method using service macro.

Variables
static char *	quota_stat_name
	Name under which we store current space consumption.
static struct GNUNET_SCHEDULER_Task *	stat_timeout_task
	Task to timeout stat GET.
static struct DatastorePlugin *	plugin
	Our datastore plugin (NULL if not available).
static struct ReservationList *	reservations
	Linked list of space reservations made by clients.
static struct GNUNET_CONTAINER_BloomFilter *	filter
	Bloomfilter to quickly tell if we don't have the content.
static char *	plugin_name
	Name of our plugin.
static const struct GNUNET_CONFIGURATION_Handle *	cfg
	Our configuration.
static struct GNUNET_STATISTICS_Handle *	stats
	Handle for reporting statistics.
static unsigned long long	cache_size
	How much space are we using for the cache? (space available for insertions that will be instantly reclaimed by discarding less important content — or possibly whatever we just inserted into the "cache").
static unsigned long long	reserved
	How much space have we currently reserved?
static unsigned long long	payload
	How much data are we currently storing in the database?
static struct GNUNET_SCHEDULER_Task *	expired_kill_task
	Identity of the task that is used to delete expired content.
static struct GNUNET_TIME_Absolute	min_expiration
	Minimum time that content should have to not be discarded instantly (time stamp of any content that we've been discarding recently to stay below the quota).
static unsigned long long	quota
	How much space are we allowed to use?
static int	do_drop
	Should the database be dropped on exit?
static int	refresh_bf
	Should we refresh the BF when the DB is loaded?
static unsigned int	last_sync
	Number of updates that were made to the payload value since we last synchronized it with the statistics service.
static int	stats_worked
	Did we get an answer from statistics?
static int	cleaning_done
	Have we already cleaned up the TCCs and are hence no longer willing (or able) to transmit anything to anyone?
static struct GNUNET_STATISTICS_GetHandle *	stat_get
	Handle for pending get request.
static struct GNUNET_SERVICE_Handle *	service
	Handle to our server.

Detailed Description

Management for the datastore for files stored on a GNUnet node.

Author

Christian Grothoff

Definition in file gnunet-service-datastore.c.

Macro Definition Documentation

MAX_PENDING

#define MAX_PENDING   1024

How many messages do we queue at most per client?

Definition at line 37 of file gnunet-service-datastore.c.

MAX_BF_SIZE

#define MAX_BF_SIZE   ((uint32_t) (1LL << 31))

Limit size of bloom filter to 2 GB.

Definition at line 42 of file gnunet-service-datastore.c.

MAX_EXPIRE_DELAY

#define MAX_EXPIRE_DELAY    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)

How long are we at most keeping "expired" content past the expiration date in the database?

Definition at line 48 of file gnunet-service-datastore.c.

{
  struct GNUNET_DATASTORE_PluginFunctions *api;
 
  char *short_name;
 
  char *lib_name;
 
  struct GNUNET_DATASTORE_PluginEnvironment env;
};
 
 
struct ReservationList
{
  struct ReservationList *next;
 
  struct GNUNET_SERVICE_Client *client;
 
  uint64_t amount;
 
  uint64_t entries;
 
  int32_t rid;
};
 
 
static struct DatastorePlugin *plugin;
 
static struct ReservationList *reservations;
 
static struct GNUNET_CONTAINER_BloomFilter *filter;
 
static char *plugin_name;
 
static const struct GNUNET_CONFIGURATION_Handle *cfg;
 
static struct GNUNET_STATISTICS_Handle *stats;
 
static unsigned long long cache_size;
 
static unsigned long long reserved;
 
static unsigned long long payload;
 
static struct GNUNET_SCHEDULER_Task *expired_kill_task;
 
static struct GNUNET_TIME_Absolute min_expiration;
 
static unsigned long long quota;
 
static int do_drop;
 
static int refresh_bf;
 
static unsigned int last_sync;
 
static int stats_worked;
 
 
static void
sync_stats ()
{
  GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
  GNUNET_STATISTICS_set (stats,
                         "# utilization by current datastore",
                         payload,
                         GNUNET_NO);
  last_sync = 0;
}
 
 
static int cleaning_done;
 
static struct GNUNET_STATISTICS_GetHandle *stat_get;
 
static struct GNUNET_SERVICE_Handle *service;
 
static void
delete_expired (void *cls);
 
 
static int
expired_processor (void *cls,
                   const struct GNUNET_HashCode *key,
                   uint32_t size,
                   const void *data,
                   enum GNUNET_BLOCK_Type type,
                   uint32_t priority,
                   uint32_t anonymity,
                   uint32_t replication,
                   struct GNUNET_TIME_Absolute expiration,
                   uint64_t uid)
{
  struct GNUNET_TIME_Absolute now;
 
  if (NULL == key)
  {
    expired_kill_task =
      GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
                                                  GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                  &delete_expired,
                                                  NULL);
    return GNUNET_SYSERR;
  }
  now = GNUNET_TIME_absolute_get ();
  if (expiration.abs_value_us > now.abs_value_us)
  {
    /* finished processing */
    expired_kill_task =
      GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
                                                  GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                  &delete_expired,
                                                  NULL);
    return GNUNET_SYSERR;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Deleting content `%s' of type %u that expired %s ago\n",
              GNUNET_h2s (key),
              type,
              GNUNET_STRINGS_relative_time_to_string (
                GNUNET_TIME_absolute_get_difference (expiration, now),
                GNUNET_YES));
  min_expiration = now;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# bytes expired"),
                            size,
                            GNUNET_YES);
  GNUNET_CONTAINER_bloomfilter_remove (filter, key);
  expired_kill_task =
    GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY,
                                                GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                &delete_expired,
                                                NULL);
  return GNUNET_NO;
}
 
 
static void
delete_expired (void *cls)
{
  expired_kill_task = NULL;
  plugin->api->get_expiration (plugin->api->cls, &expired_processor, NULL);
}
 
 
static int
quota_processor (void *cls,
                 const struct GNUNET_HashCode *key,
                 uint32_t size,
                 const void *data,
                 enum GNUNET_BLOCK_Type type,
                 uint32_t priority,
                 uint32_t anonymity,
                 uint32_t replication,
                 struct GNUNET_TIME_Absolute expiration,
                 uint64_t uid)
{
  unsigned long long *need = cls;
 
  if (NULL == key)
    return GNUNET_SYSERR;
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %s prior to expiration (still trying to free another %llu bytes)\n",
    (unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD),
    (unsigned int) priority,
    GNUNET_h2s (key),
    type,
    GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
                                              expiration),
                                            GNUNET_YES),
    *need);
  if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need)
    *need = 0;
  else
    *need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
  if (priority > 0)
    min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS;
  else
    min_expiration = expiration;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# bytes purged (low-priority)"),
                            size,
                            GNUNET_YES);
  GNUNET_CONTAINER_bloomfilter_remove (filter, key);
  return GNUNET_NO;
}
 
 
static void
manage_space (unsigned long long need)
{
  unsigned long long last;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Asked to free up %llu bytes of cache space\n",
              need);
  last = 0;
  while ((need > 0) && (last != need))
  {
    last = need;
    plugin->api->get_expiration (plugin->api->cls, &quota_processor, &need);
  }
}
 
 
static void
transmit_status (struct GNUNET_SERVICE_Client *client,
                 int code,
                 const char *msg)
{
  struct GNUNET_MQ_Envelope *env;
  struct StatusMessage *sm;
  size_t slen;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Transmitting `%s' message with value %d and message `%s'\n",
              "STATUS",
              code,
              msg != NULL ? msg : "(none)");
  slen = (msg == NULL) ? 0 : strlen (msg) + 1;
  env = GNUNET_MQ_msg_extra (sm, slen, GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
  sm->status = htonl (code);
  sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration);
  GNUNET_memcpy (&sm[1], msg, slen);
  GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
}
 
 
static int
transmit_item (void *cls,
               const struct GNUNET_HashCode *key,
               uint32_t size,
               const void *data,
               enum GNUNET_BLOCK_Type type,
               uint32_t priority,
               uint32_t anonymity,
               uint32_t replication,
               struct GNUNET_TIME_Absolute expiration,
               uint64_t uid)
{
  struct GNUNET_SERVICE_Client *client = cls;
  struct GNUNET_MQ_Envelope *env;
  struct GNUNET_MessageHeader *end;
  struct DataMessage *dm;
 
  if (NULL == key)
  {
    /* transmit 'DATA_END' */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Transmitting DATA_END message\n");
    env = GNUNET_MQ_msg (end, GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END);
    GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
    return GNUNET_OK;
  }
  GNUNET_assert (sizeof(struct DataMessage) + size < GNUNET_MAX_MESSAGE_SIZE);
  env = GNUNET_MQ_msg_extra (dm, size, GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
  dm->rid = htonl (0);
  dm->size = htonl (size);
  dm->type = htonl (type);
  dm->priority = htonl (priority);
  dm->anonymity = htonl (anonymity);
  dm->replication = htonl (replication);
  dm->expiration = GNUNET_TIME_absolute_hton (expiration);
  dm->uid = GNUNET_htonll (uid);
  dm->key = *key;
  GNUNET_memcpy (&dm[1], data, size);
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Transmitting DATA message for `%s' of type %u with expiration %s (in: %s)\n",
    GNUNET_h2s (key),
    type,
    GNUNET_STRINGS_absolute_time_to_string (expiration),
    GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
                                              expiration),
                                            GNUNET_YES));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# results found"),
                            1,
                            GNUNET_NO);
  GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
  return GNUNET_OK;
}
 
 
static void
handle_reserve (void *cls, const struct ReserveMessage *msg)
{
  static int reservation_gen;
  struct GNUNET_SERVICE_Client *client = cls;
  struct ReservationList *e;
  unsigned long long used;
  unsigned long long req;
  uint64_t amount;
  uint32_t entries;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RESERVE request\n");
  amount = GNUNET_ntohll (msg->amount);
  entries = ntohl (msg->entries);
  used = payload + reserved;
  req =
    amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries;
  if (used + req > quota)
  {
    if (quota < used)
      used =
        quota;   /* cheat a bit for error message (to avoid negative numbers) */
    GNUNET_log (
      GNUNET_ERROR_TYPE_WARNING,
      _ (
        "Insufficient space (%llu bytes are available) to satisfy RESERVE request for %llu bytes\n"),
      quota - used,
      req);
    if (cache_size < req)
    {
      /* TODO: document this in the FAQ; essentially, if this
       * message happens, the insertion request could be blocked
       * by less-important content from migration because it is
       * larger than 1/8th of the overall available space, and
       * we only reserve 1/8th for "fresh" insertions */
      GNUNET_log (
        GNUNET_ERROR_TYPE_WARNING,
        _ (
          "The requested amount (%llu bytes) is larger than the cache size (%llu bytes)\n"),
        req,
        cache_size);
      transmit_status (client,
                       0,
                       gettext_noop (
                         "Insufficient space to satisfy request and "
                         "requested amount is larger than cache size"));
    }
    else
    {
      transmit_status (client,
                       0,
                       gettext_noop ("Insufficient space to satisfy request"));
    }
    GNUNET_SERVICE_client_continue (client);
    return;
  }
  reserved += req;
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# reserved"),
                         reserved,
                         GNUNET_NO);
  e = GNUNET_new (struct ReservationList);
  e->next = reservations;
  reservations = e;
  e->client = client;
  e->amount = amount;
  e->entries = entries;
  e->rid = ++reservation_gen;
  if (reservation_gen < 0)
    reservation_gen = 0; /* wrap around */
  transmit_status (client, e->rid, NULL);
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
handle_release_reserve (void *cls, const struct ReleaseReserveMessage *msg)
{
  struct GNUNET_SERVICE_Client *client = cls;
  struct ReservationList *pos;
  struct ReservationList *prev;
  struct ReservationList *next;
  int rid = ntohl (msg->rid);
  unsigned long long rem;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RELEASE_RESERVE request\n");
  next = reservations;
  prev = NULL;
  while (NULL != (pos = next))
  {
    next = pos->next;
    if (rid == pos->rid)
    {
      if (prev == NULL)
        reservations = next;
      else
        prev->next = next;
      rem =
        pos->amount
        + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * pos->entries;
      GNUNET_assert (reserved >= rem);
      reserved -= rem;
      GNUNET_STATISTICS_set (stats,
                             gettext_noop ("# reserved"),
                             reserved,
                             GNUNET_NO);
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Returning %llu remaining reserved bytes to storage pool\n",
                  rem);
      GNUNET_free (pos);
      transmit_status (client, GNUNET_OK, NULL);
      GNUNET_SERVICE_client_continue (client);
      return;
    }
    prev = pos;
  }
  GNUNET_break (0);
  transmit_status (client,
                   GNUNET_SYSERR,
                   gettext_noop ("Could not find matching reservation"));
  GNUNET_SERVICE_client_continue (client);
}
 
 
static int
check_data (const struct DataMessage *dm)
{
  uint16_t size;
  uint32_t dsize;
 
  size = ntohs (dm->header.size);
  dsize = ntohl (dm->size);
  if (size != dsize + sizeof(struct DataMessage))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}
 
 
static void
put_continuation (void *cls,
                  const struct GNUNET_HashCode *key,
                  uint32_t size,
                  int status,
                  const char *msg)
{
  struct GNUNET_SERVICE_Client *client = cls;
 
  if (GNUNET_OK == status)
  {
    GNUNET_STATISTICS_update (stats,
                              gettext_noop ("# bytes stored"),
                              size,
                              GNUNET_YES);
    GNUNET_CONTAINER_bloomfilter_add (filter, key);
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Successfully stored %u bytes under key `%s'\n",
                size,
                GNUNET_h2s (key));
  }
  transmit_status (client,
                   GNUNET_SYSERR == status ? GNUNET_SYSERR : GNUNET_OK,
                   msg);
  if (quota - reserved - cache_size < payload)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Need %llu bytes more space (%llu allowed, using %llu)\n"),
                (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
                (unsigned long long) (quota - reserved - cache_size),
                (unsigned long long) payload);
    manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
  }
}
 
 
static int
check_put (void *cls, const struct DataMessage *dm)
{
  if (GNUNET_OK != check_data (dm))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}
 
 
static void
handle_put (void *cls, const struct DataMessage *dm)
{
  struct GNUNET_SERVICE_Client *client = cls;
  int rid;
  struct ReservationList *pos;
  uint32_t size;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing PUT request for `%s' of type %u\n",
              GNUNET_h2s (&dm->key),
              (uint32_t) ntohl (dm->type));
  rid = ntohl (dm->rid);
  size = ntohl (dm->size);
  if (rid > 0)
  {
    pos = reservations;
    while ((NULL != pos) && (rid != pos->rid))
      pos = pos->next;
    GNUNET_break (pos != NULL);
    if (NULL != pos)
    {
      GNUNET_break (pos->entries > 0);
      GNUNET_break (pos->amount >= size);
      pos->entries--;
      pos->amount -= size;
      reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
      GNUNET_STATISTICS_set (stats,
                             gettext_noop ("# reserved"),
                             reserved,
                             GNUNET_NO);
    }
  }
  {
    bool absent =
      GNUNET_NO == GNUNET_CONTAINER_bloomfilter_test (filter, &dm->key);
    plugin->api->put (plugin->api->cls,
                      &dm->key,
                      absent,
                      ntohl (dm->size),
                      &dm[1],
                      ntohl (dm->type),
                      ntohl (dm->priority),
                      ntohl (dm->anonymity),
                      ntohl (dm->replication),
                      GNUNET_TIME_absolute_ntoh (dm->expiration),
                      &put_continuation,
                      client);
  }
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
handle_get (void *cls, const struct GetMessage *msg)
{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing GET request of type %u\n",
              (uint32_t) ntohl (msg->type));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# GET requests received"),
                            1,
                            GNUNET_NO);
  plugin->api->get_key (plugin->api->cls,
                        GNUNET_ntohll (msg->next_uid),
                        msg->random,
                        NULL,
                        ntohl (msg->type),
                        &transmit_item,
                        client);
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
handle_get_key (void *cls, const struct GetKeyMessage *msg)
{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing GET request for `%s' of type %u\n",
              GNUNET_h2s (&msg->key),
              (uint32_t) ntohl (msg->type));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# GET KEY requests received"),
                            1,
                            GNUNET_NO);
  if (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (filter, &msg->key))
  {
    /* don't bother database... */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Empty result set for GET request for `%s' (bloomfilter).\n",
                GNUNET_h2s (&msg->key));
    GNUNET_STATISTICS_update (stats,
                              gettext_noop (
                                "# requests filtered by bloomfilter"),
                              1,
                              GNUNET_NO);
    transmit_item (client,
                   NULL,
                   0,
                   NULL,
                   0,
                   0,
                   0,
                   0,
                   GNUNET_TIME_UNIT_ZERO_ABS,
                   0);
    GNUNET_SERVICE_client_continue (client);
    return;
  }
  plugin->api->get_key (plugin->api->cls,
                        GNUNET_ntohll (msg->next_uid),
                        msg->random,
                        &msg->key,
                        ntohl (msg->type),
                        &transmit_item,
                        client);
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
handle_get_replication (void *cls, const struct GNUNET_MessageHeader *message)
{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing GET_REPLICATION request\n");
  GNUNET_STATISTICS_update (stats,
                            gettext_noop (
                              "# GET REPLICATION requests received"),
                            1,
                            GNUNET_NO);
  plugin->api->get_replication (plugin->api->cls, &transmit_item, client);
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
handle_get_zero_anonymity (void *cls, const struct GetZeroAnonymityMessage *msg)
{
  struct GNUNET_SERVICE_Client *client = cls;
  enum GNUNET_BLOCK_Type type;
 
  type = ntohl (msg->type);
  if (type == GNUNET_BLOCK_TYPE_ANY)
  {
    GNUNET_break (0);
    GNUNET_SERVICE_client_drop (client);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing GET_ZERO_ANONYMITY request\n");
  GNUNET_STATISTICS_update (stats,
                            gettext_noop (
                              "# GET ZERO ANONYMITY requests received"),
                            1,
                            GNUNET_NO);
  plugin->api->get_zero_anonymity (plugin->api->cls,
                                   GNUNET_ntohll (msg->next_uid),
                                   type,
                                   &transmit_item,
                                   client);
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
remove_continuation (void *cls,
                     const struct GNUNET_HashCode *key,
                     uint32_t size,
                     int status,
                     const char *msg)
{
  struct GNUNET_SERVICE_Client *client = cls;
 
  if (GNUNET_SYSERR == status)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "REMOVE request failed: %s.\n", msg);
    transmit_status (client, GNUNET_NO, msg);
    return;
  }
  if (GNUNET_NO == status)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Content not found for REMOVE request.\n");
    transmit_status (client, GNUNET_NO, _ ("Content not found"));
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Item matches REMOVE request for key `%s'.\n",
              GNUNET_h2s (key));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# bytes removed (explicit request)"),
                            size,
                            GNUNET_YES);
  GNUNET_CONTAINER_bloomfilter_remove (filter, key);
  transmit_status (client, GNUNET_OK, NULL);
}
 
 
static int
check_remove (void *cls, const struct DataMessage *dm)
{
  if (GNUNET_OK != check_data (dm))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}
 
 
static void
handle_remove (void *cls, const struct DataMessage *dm)
{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# REMOVE requests received"),
                            1,
                            GNUNET_NO);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing REMOVE request for `%s'\n",
              GNUNET_h2s (&dm->key));
  plugin->api->remove_key (plugin->api->cls,
                           &dm->key,
                           ntohl (dm->size),
                           &dm[1],
                           &remove_continuation,
                           client);
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
handle_drop (void *cls,
             const struct GNUNET_MessageHeader *message)
{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing DROP request\n");
  do_drop = GNUNET_YES;
  GNUNET_SERVICE_client_continue (client);
}
 
 
static void
disk_utilization_change_cb (void *cls, int delta)
{
  if ((delta < 0) && (payload < -delta))
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_WARNING,
      _ (
        "Datastore payload must have been inaccurate (%lld < %lld). Recomputing it.\n"),
      (long long) payload,
      (long long) -delta);
    plugin->api->estimate_size (plugin->api->cls, &payload);
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("New payload: %lld\n"),
                (long long) payload);
    sync_stats ();
    return;
  }
  payload += delta;
  last_sync++;
  if (last_sync >= MAX_STAT_SYNC_LAG)
    sync_stats ();
}
 
 
static int
process_stat_in (void *cls,
                 const char *subsystem,
                 const char *name,
                 uint64_t value,
                 int is_persistent)
{
  GNUNET_assert (GNUNET_NO == stats_worked);
  stats_worked = GNUNET_YES;
  payload += value;
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Notification from statistics about existing payload (%llu), new payload is %llu\n",
    (unsigned long long) value,
    (unsigned long long) payload);
  return GNUNET_OK;
}
 
 
static struct DatastorePlugin *
load_plugin ()
{
  struct DatastorePlugin *ret;
  char *libname;
 
  ret = GNUNET_new (struct DatastorePlugin);
  ret->env.cfg = cfg;
  ret->env.duc = &disk_utilization_change_cb;
  ret->env.cls = NULL;
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _ ("Loading `%s' datastore plugin\n"),
              plugin_name);
  GNUNET_asprintf (&libname,
                   "libgnunet_plugin_datastore_%s",
                   plugin_name);
  ret->short_name = GNUNET_strdup (plugin_name);
  ret->lib_name = libname;
  ret->api = GNUNET_PLUGIN_load (GNUNET_OS_project_data_gnunet (),
                                 libname,
                                 &ret->env);
  if (NULL == ret->api)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("Failed to load datastore plugin for `%s'\n"),
                plugin_name);
    GNUNET_free (ret->short_name);
    GNUNET_free (libname);
    GNUNET_free (ret);
    return NULL;
  }
  return ret;
}
 
 
static void
unload_plugin (struct DatastorePlugin *plug)
{
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Datastore service is unloading plugin...\n");
  GNUNET_break (NULL == GNUNET_PLUGIN_unload (plug->lib_name, plug->api));
  GNUNET_free (plug->lib_name);
  GNUNET_free (plug->short_name);
  GNUNET_free (plug);
}
 
 
static void
begin_service ()
{
  GNUNET_SERVICE_resume (service);
  expired_kill_task =
    GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
                                        &delete_expired,
                                        NULL);
}
 
 
static void
add_key_to_bloomfilter (void *cls,
                        const struct GNUNET_HashCode *key,
                        unsigned int count)
{
  struct GNUNET_CONTAINER_BloomFilter *bf = cls;
 
  if (NULL == key)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Bloomfilter construction complete.\n"));
    begin_service ();
    return;
  }
 
  while (0 < count--)
    GNUNET_CONTAINER_bloomfilter_add (bf, key);
}
 
 
static void
process_stat_done (void *cls, int success)
{
  stat_get = NULL;
  if (NULL != stat_timeout_task)
  {
    GNUNET_SCHEDULER_cancel (stat_timeout_task);
    stat_timeout_task = NULL;
  }
  plugin = load_plugin ();
  if (NULL == plugin)
  {
    GNUNET_CONTAINER_bloomfilter_free (filter);
    filter = NULL;
    if (NULL != stats)
    {
      GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
      stats = NULL;
    }
    return;
  }
 
  if (GNUNET_NO == stats_worked)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Failed to obtain value from statistics service, recomputing it\n");
    plugin->api->estimate_size (plugin->api->cls, &payload);
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("New payload: %lld\n"),
                (long long) payload);
  }
 
  if (GNUNET_YES == refresh_bf)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Rebuilding bloomfilter.  Please be patient.\n"));
    if (NULL != plugin->api->get_keys)
    {
      plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);
      return;
    }
    else
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  _ (
                    "Plugin does not support get_keys function. Please fix!\n"))
      ;
    }
  }
  begin_service ();
}
 
 
static void
stat_timeout (void *cls)
{
  stat_timeout_task = NULL;
  GNUNET_STATISTICS_get_cancel (stat_get);
  process_stat_done (NULL, GNUNET_NO);
}
 
 
static void
cleaning_task (void *cls)
{
  cleaning_done = GNUNET_YES;
  if (NULL != expired_kill_task)
  {
    GNUNET_SCHEDULER_cancel (expired_kill_task);
    expired_kill_task = NULL;
  }
  if (GNUNET_YES == do_drop)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Dropping database!\n");
    plugin->api->drop (plugin->api->cls);
    payload = 0;
    last_sync++;
  }
  if (NULL != plugin)
  {
    unload_plugin (plugin);
    plugin = NULL;
  }
  if (NULL != filter)
  {
    GNUNET_CONTAINER_bloomfilter_free (filter);
    filter = NULL;
  }
  if (NULL != stat_get)
  {
    GNUNET_STATISTICS_get_cancel (stat_get);
    stat_get = NULL;
  }
  if (NULL != stat_timeout_task)
  {
    GNUNET_SCHEDULER_cancel (stat_timeout_task);
    stat_timeout_task = NULL;
  }
  GNUNET_free (plugin_name);
  plugin_name = NULL;
  if (last_sync > 0)
    sync_stats ();
  if (NULL != stats)
  {
    GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
    stats = NULL;
  }
  GNUNET_free (quota_stat_name);
  quota_stat_name = NULL;
}
 
 
static void *
client_connect_cb (void *cls,
                   struct GNUNET_SERVICE_Client *client,
                   struct GNUNET_MQ_Handle *mq)
{
  return client;
}
 
 
static void
client_disconnect_cb (void *cls,
                      struct GNUNET_SERVICE_Client *client,
                      void *app_ctx)
{
  struct ReservationList *pos;
  struct ReservationList *prev;
  struct ReservationList *next;
 
  GNUNET_assert (app_ctx == client);
  prev = NULL;
  pos = reservations;
  while (NULL != pos)
  {
    next = pos->next;
    if (pos->client == client)
    {
      if (NULL == prev)
        reservations = next;
      else
        prev->next = next;
      reserved -= pos->amount + pos->entries * GNUNET_DATASTORE_ENTRY_OVERHEAD;
      GNUNET_free (pos);
    }
    else
    {
      prev = pos;
    }
    pos = next;
  }
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# reserved"),
                         reserved,
                         GNUNET_NO);
}
 
 
static void
run (void *cls,
     const struct GNUNET_CONFIGURATION_Handle *c,
     struct GNUNET_SERVICE_Handle *serv)
{
  char *fn;
  char *pfn;
  unsigned int bf_size;
 
  service = serv;
  cfg = c;
  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg,
                                                          "DATASTORE",
                                                          "DATABASE",
                                                          &plugin_name))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                               "DATABASE",
                               "DATASTORE");
    return;
  }
  GNUNET_asprintf (&quota_stat_name,
                   _ ("# bytes used in file-sharing datastore `%s'"),
                   plugin_name);
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", &quota))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "QUOTA", "DATASTORE");
    return;
  }
  stats = GNUNET_STATISTICS_create ("datastore", cfg);
  GNUNET_STATISTICS_set (stats, gettext_noop ("# quota"), quota, GNUNET_NO);
  cache_size = quota / 8; /* Or should we make this an option? */
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# cache size"),
                         cache_size,
                         GNUNET_NO);
  if (quota / (32 * 1024LL) > MAX_BF_SIZE)
    bf_size = MAX_BF_SIZE;
  else
    bf_size =
      quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */
  fn = NULL;
  if ((GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                             "DATASTORE",
                                                             "BLOOMFILTER",
                                                             &fn)) ||
      (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn)))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Could not use specified filename `%s' for bloomfilter.\n"),
                NULL != fn ? fn : "");
    GNUNET_free (fn);
    fn = NULL;
  }
  if (NULL != fn)
  {
    GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
    if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
    {
      filter =
        GNUNET_CONTAINER_bloomfilter_load (pfn,
                                           bf_size,
                                           5);    /* approx. 3% false positives at max use */
      if (NULL == filter)
      {
        /* file exists but not valid, remove and try again, but refresh */
        if (0 != unlink (pfn))
        {
          /* failed to remove, run without file */
          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                      _ ("Failed to remove bogus bloomfilter file `%s'\n"),
                      pfn);
          GNUNET_free (pfn);
          pfn = NULL;
          filter = GNUNET_CONTAINER_bloomfilter_load (
            NULL,
            bf_size,
            5);         /* approx. 3% false positives at max use */
          refresh_bf = GNUNET_YES;
        }
        else
        {
          /* try again after remove */
          filter = GNUNET_CONTAINER_bloomfilter_load (
            pfn,
            bf_size,
            5);         /* approx. 3% false positives at max use */
          refresh_bf = GNUNET_YES;
          if (NULL == filter)
          {
            /* failed yet again, give up on using file */
            GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                        _ ("Failed to remove bogus bloomfilter file `%s'\n"),
                        pfn);
            GNUNET_free (pfn);
            pfn = NULL;
            filter = GNUNET_CONTAINER_bloomfilter_init (
              NULL,
              bf_size,
              5);           /* approx. 3% false positives at max use */
          }
        }
      }
      else
      {
        /* normal case: have an existing valid bf file, no need to refresh */
        refresh_bf = GNUNET_NO;
      }
    }
    else
    {
      filter =
        GNUNET_CONTAINER_bloomfilter_load (pfn,
                                           bf_size,
                                           5);    /* approx. 3% false positives at max use */
      refresh_bf = GNUNET_YES;
    }
    GNUNET_free (pfn);
  }
  else
  {
    filter =
      GNUNET_CONTAINER_bloomfilter_init (NULL,
                                         bf_size,
                                         5);  /* approx. 3% false positives at max use */
    refresh_bf = GNUNET_YES;
  }
  GNUNET_free (fn);
  if (NULL == filter)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("Failed to initialize bloomfilter.\n"));
    if (NULL != stats)
    {
      GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
      stats = NULL;
    }
    return;
  }
  GNUNET_SERVICE_suspend (service);
  stat_get = GNUNET_STATISTICS_get (stats,
                                    "datastore",
                                    quota_stat_name,
                                    &process_stat_done,
                                    &process_stat_in,
                                    NULL);
  if (NULL == stat_get)
    process_stat_done (NULL, GNUNET_SYSERR);
  else
    stat_timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
                                                      &stat_timeout,
                                                      NULL);
  GNUNET_SCHEDULER_add_shutdown (&cleaning_task, NULL);
}
 
 
GNUNET_SERVICE_MAIN (
  GNUNET_OS_project_data_gnunet(),
  "datastore",
  GNUNET_SERVICE_OPTION_NONE,
  &run,
  &client_connect_cb,
  &client_disconnect_cb,
  NULL,
  GNUNET_MQ_hd_fixed_size (reserve,
                           GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE,
                           struct ReserveMessage,
                           NULL),
  GNUNET_MQ_hd_fixed_size (release_reserve,
                           GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE,
                           struct ReleaseReserveMessage,
                           NULL),
  GNUNET_MQ_hd_var_size (put,
                         GNUNET_MESSAGE_TYPE_DATASTORE_PUT,
                         struct DataMessage,
                         NULL),
  GNUNET_MQ_hd_fixed_size (get,
                           GNUNET_MESSAGE_TYPE_DATASTORE_GET,
                           struct GetMessage,
                           NULL),
  GNUNET_MQ_hd_fixed_size (get_key,
                           GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY,
                           struct GetKeyMessage,
                           NULL),
  GNUNET_MQ_hd_fixed_size (get_replication,
                           GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION,
                           struct GNUNET_MessageHeader,
                           NULL),
  GNUNET_MQ_hd_fixed_size (get_zero_anonymity,
                           GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY,
                           struct GetZeroAnonymityMessage,
                           NULL),
  GNUNET_MQ_hd_var_size (remove,
                         GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE,
                         struct DataMessage,
                         NULL),
  GNUNET_MQ_hd_fixed_size (drop,
                           GNUNET_MESSAGE_TYPE_DATASTORE_DROP,
                           struct GNUNET_MessageHeader,
                           NULL),
  GNUNET_MQ_handler_end ());
 
 
/* end of gnunet-service-datastore.c */

MIN_EXPIRE_DELAY

#define MIN_EXPIRE_DELAY    GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)

How fast are we allowed to query the database for deleting expired content? (1 item per second).

Definition at line 55 of file gnunet-service-datastore.c.

MAX_STAT_SYNC_LAG

#define MAX_STAT_SYNC_LAG   50

After how many payload-changing operations do we sync our statistics?

Definition at line 72 of file gnunet-service-datastore.c.

Function Documentation

sync_stats()

static void sync_stats ( )

static

Synchronize our utilization statistics with the statistics service.

Definition at line 232 of file gnunet-service-datastore.c.

{
  GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
  GNUNET_STATISTICS_set (stats,
                         "# utilization by current datastore",
                         payload,
                         GNUNET_NO);
  last_sync = 0;
}

References GNUNET_NO, GNUNET_STATISTICS_set(), GNUNET_YES, last_sync, payload, quota_stat_name, and stats.

Referenced by cleaning_task(), and disk_utilization_change_cb().

Here is the call graph for this function:

Here is the caller graph for this function:

delete_expired()

static void delete_expired ( void *  cls )

static

Task that is used to remove expired entries from the datastore.

This task will schedule itself again automatically to always delete all expired content quickly.

Parameters

cls

not used

Definition at line 357 of file gnunet-service-datastore.c.

{
  expired_kill_task = NULL;
  plugin->api->get_expiration (plugin->api->cls, &expired_processor, NULL);
}

References DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, expired_kill_task, expired_processor(), GNUNET_DATASTORE_PluginFunctions::get_expiration, and plugin.

Referenced by begin_service(), and expired_processor().

Here is the call graph for this function:

Here is the caller graph for this function:

expired_processor()

static int expired_processor ( void *  cls, const struct GNUNET_HashCode *  key, uint32_t  size, const void *  data, enum GNUNET_BLOCK_Type  type, uint32_t  priority, uint32_t  anonymity, uint32_t  replication, struct GNUNET_TIME_Absolute  expiration, uint64_t  uid  )

static

Iterate over the expired items stored in the datastore.

Delete all expired items; once we have processed all expired items, re-schedule the "delete_expired" task.

Parameters

cls	not used
key	key for the content
size	number of bytes in data
data	content stored
type	type of the content
priority	priority of the content
anonymity	anonymity-level for the content
replication	replication-level for the content
expiration	expiration time for the content
uid	unique identifier for the datum; maybe 0 if no unique identifier is available

Returns

GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue (continue on call to "next", of course), GNUNET_NO to delete the item and continue (if supported)

Definition at line 293 of file gnunet-service-datastore.c.

{
  struct GNUNET_TIME_Absolute now;
 
  if (NULL == key)
  {
    expired_kill_task =
      GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
                                                  GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                  &delete_expired,
                                                  NULL);
    return GNUNET_SYSERR;
  }
  now = GNUNET_TIME_absolute_get ();
  if (expiration.abs_value_us > now.abs_value_us)
  {
    /* finished processing */
    expired_kill_task =
      GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
                                                  GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                  &delete_expired,
                                                  NULL);
    return GNUNET_SYSERR;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Deleting content `%s' of type %u that expired %s ago\n",
              GNUNET_h2s (key),
              type,
              GNUNET_STRINGS_relative_time_to_string (
                GNUNET_TIME_absolute_get_difference (expiration, now),
                GNUNET_YES));
  min_expiration = now;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# bytes expired"),
                            size,
                            GNUNET_YES);
  GNUNET_CONTAINER_bloomfilter_remove (filter, key);
  expired_kill_task =
    GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY,
                                                GNUNET_SCHEDULER_PRIORITY_IDLE,
                                                &delete_expired,
                                                NULL);
  return GNUNET_NO;
}

References GNUNET_TIME_Absolute::abs_value_us, delete_expired(), expiration, expired_kill_task, filter, gettext_noop, GNUNET_CONTAINER_bloomfilter_remove(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_h2s(), GNUNET_log, GNUNET_NO, GNUNET_SCHEDULER_add_delayed_with_priority(), GNUNET_SCHEDULER_PRIORITY_IDLE, GNUNET_STATISTICS_update(), GNUNET_STRINGS_relative_time_to_string(), GNUNET_SYSERR, GNUNET_TIME_absolute_get(), GNUNET_TIME_absolute_get_difference(), GNUNET_YES, key, MAX_EXPIRE_DELAY, min_expiration, MIN_EXPIRE_DELAY, size, stats, and type.

Referenced by delete_expired().

Here is the call graph for this function:

Here is the caller graph for this function:

quota_processor()

static int quota_processor ( void *  cls, const struct GNUNET_HashCode *  key, uint32_t  size, const void *  data, enum GNUNET_BLOCK_Type  type, uint32_t  priority, uint32_t  anonymity, uint32_t  replication, struct GNUNET_TIME_Absolute  expiration, uint64_t  uid  )

static

An iterator over a set of items stored in the datastore that deletes until we're happy with respect to our quota.

Parameters

cls	closure
key	key for the content
size	number of bytes in data
data	content stored
type	type of the content
priority	priority of the content
anonymity	anonymity-level for the content
replication	replication-level for the content
expiration	expiration time for the content
uid	unique identifier for the datum; maybe 0 if no unique identifier is available

Returns

GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue (continue on call to "next", of course), GNUNET_NO to delete the item and continue (if supported)

Definition at line 384 of file gnunet-service-datastore.c.

{
  unsigned long long *need = cls;
 
  if (NULL == key)
    return GNUNET_SYSERR;
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %s prior to expiration (still trying to free another %llu bytes)\n",
    (unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD),
    (unsigned int) priority,
    GNUNET_h2s (key),
    type,
    GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
                                              expiration),
                                            GNUNET_YES),
    *need);
  if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need)
    *need = 0;
  else
    *need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
  if (priority > 0)
    min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS;
  else
    min_expiration = expiration;
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# bytes purged (low-priority)"),
                            size,
                            GNUNET_YES);
  GNUNET_CONTAINER_bloomfilter_remove (filter, key);
  return GNUNET_NO;
}

References expiration, filter, gettext_noop, GNUNET_CONTAINER_bloomfilter_remove(), GNUNET_DATASTORE_ENTRY_OVERHEAD, GNUNET_ERROR_TYPE_DEBUG, GNUNET_h2s(), GNUNET_log, GNUNET_NO, GNUNET_STATISTICS_update(), GNUNET_STRINGS_relative_time_to_string(), GNUNET_SYSERR, GNUNET_TIME_absolute_get_remaining(), GNUNET_TIME_UNIT_FOREVER_ABS, GNUNET_YES, key, min_expiration, size, stats, and type.

Referenced by manage_space().

Here is the call graph for this function:

Here is the caller graph for this function:

manage_space()

static void manage_space ( unsigned long long  need )

static

Manage available disk space by running tasks that will discard content if necessary.

This function will be run whenever a request for "need" bytes of storage could only be satisfied by eating into the "cache" (and we want our cache space back).

Parameters

need	number of bytes of content that were placed into the "cache" (and hence the number of bytes that should be removed).

Definition at line 440 of file gnunet-service-datastore.c.

{
  unsigned long long last;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Asked to free up %llu bytes of cache space\n",
              need);
  last = 0;
  while ((need > 0) && (last != need))
  {
    last = need;
    plugin->api->get_expiration (plugin->api->cls, &quota_processor, &need);
  }
}

References DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, GNUNET_DATASTORE_PluginFunctions::get_expiration, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, plugin, and quota_processor().

Referenced by put_continuation().

Here is the call graph for this function:

Here is the caller graph for this function:

transmit_status()

static void transmit_status ( struct GNUNET_SERVICE_Client *  client, int  code, const char *  msg  )

static

Transmit a status code to the client.

Parameters

client	receiver of the response
code	status code
msg	optional error message (can be NULL)

Definition at line 464 of file gnunet-service-datastore.c.

{
  struct GNUNET_MQ_Envelope *env;
  struct StatusMessage *sm;
  size_t slen;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Transmitting `%s' message with value %d and message `%s'\n",
              "STATUS",
              code,
              msg != NULL ? msg : "(none)");
  slen = (msg == NULL) ? 0 : strlen (msg) + 1;
  env = GNUNET_MQ_msg_extra (sm, slen, GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
  sm->status = htonl (code);
  sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration);
  GNUNET_memcpy (&sm[1], msg, slen);
  GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
}

References env, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_DATASTORE_STATUS, GNUNET_MQ_msg_extra, GNUNET_MQ_send(), GNUNET_SERVICE_client_get_mq(), GNUNET_TIME_absolute_hton(), StatusMessage::min_expiration, min_expiration, msg, and StatusMessage::status.

Referenced by handle_release_reserve(), handle_reserve(), put_continuation(), and remove_continuation().

Here is the call graph for this function:

Here is the caller graph for this function:

transmit_item()

static int transmit_item ( void *  cls, const struct GNUNET_HashCode *  key, uint32_t  size, const void *  data, enum GNUNET_BLOCK_Type  type, uint32_t  priority, uint32_t  anonymity, uint32_t  replication, struct GNUNET_TIME_Absolute  expiration, uint64_t  uid  )

static

Function that will transmit the given datastore entry to the client.

Parameters

cls	closure, pointer to the client (of type struct GNUNET_SERVICE_Client).
key	key for the content
size	number of bytes in data
data	content stored
type	type of the content
priority	priority of the content
anonymity	anonymity-level for the content
replication	replication-level for the content
expiration	expiration time for the content
uid	unique identifier for the datum; maybe 0 if no unique identifier is available

Returns

GNUNET_SYSERR to abort the iteration, GNUNET_OK to continue, GNUNET_NO to delete the item and continue (if supported)

Definition at line 505 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
  struct GNUNET_MQ_Envelope *env;
  struct GNUNET_MessageHeader *end;
  struct DataMessage *dm;
 
  if (NULL == key)
  {
    /* transmit 'DATA_END' */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Transmitting DATA_END message\n");
    env = GNUNET_MQ_msg (end, GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END);
    GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
    return GNUNET_OK;
  }
  GNUNET_assert (sizeof(struct DataMessage) + size < GNUNET_MAX_MESSAGE_SIZE);
  env = GNUNET_MQ_msg_extra (dm, size, GNUNET_MESSAGE_TYPE_DATASTORE_DATA);
  dm->rid = htonl (0);
  dm->size = htonl (size);
  dm->type = htonl (type);
  dm->priority = htonl (priority);
  dm->anonymity = htonl (anonymity);
  dm->replication = htonl (replication);
  dm->expiration = GNUNET_TIME_absolute_hton (expiration);
  dm->uid = GNUNET_htonll (uid);
  dm->key = *key;
  GNUNET_memcpy (&dm[1], data, size);
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Transmitting DATA message for `%s' of type %u with expiration %s (in: %s)\n",
    GNUNET_h2s (key),
    type,
    GNUNET_STRINGS_absolute_time_to_string (expiration),
    GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_remaining (
                                              expiration),
                                            GNUNET_YES));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# results found"),
                            1,
                            GNUNET_NO);
  GNUNET_MQ_send (GNUNET_SERVICE_client_get_mq (client), env);
  return GNUNET_OK;
}

References anonymity, DataMessage::anonymity, data, end, env, expiration, DataMessage::expiration, gettext_noop, GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_h2s(), GNUNET_htonll(), GNUNET_log, GNUNET_MAX_MESSAGE_SIZE, GNUNET_memcpy, GNUNET_MESSAGE_TYPE_DATASTORE_DATA, GNUNET_MESSAGE_TYPE_DATASTORE_DATA_END, GNUNET_MQ_msg, GNUNET_MQ_msg_extra, GNUNET_MQ_send(), GNUNET_NO, GNUNET_OK, GNUNET_SERVICE_client_get_mq(), GNUNET_STATISTICS_update(), GNUNET_STRINGS_absolute_time_to_string(), GNUNET_STRINGS_relative_time_to_string(), GNUNET_TIME_absolute_get_remaining(), GNUNET_TIME_absolute_hton(), GNUNET_YES, key, DataMessage::key, DataMessage::priority, replication, DataMessage::replication, DataMessage::rid, size, DataMessage::size, stats, type, DataMessage::type, and DataMessage::uid.

Referenced by handle_get(), handle_get_key(), handle_get_replication(), and handle_get_zero_anonymity().

Here is the call graph for this function:

Here is the caller graph for this function:

handle_reserve()

static void handle_reserve ( void *  cls, const struct ReserveMessage *  msg  )

static

Handle RESERVE-message.

Parameters

cls	identification of the client
msg	the actual message

Static counter to produce reservation identifiers.

Definition at line 566 of file gnunet-service-datastore.c.

{
  static int reservation_gen;
  struct GNUNET_SERVICE_Client *client = cls;
  struct ReservationList *e;
  unsigned long long used;
  unsigned long long req;
  uint64_t amount;
  uint32_t entries;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RESERVE request\n");
  amount = GNUNET_ntohll (msg->amount);
  entries = ntohl (msg->entries);
  used = payload + reserved;
  req =
    amount + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * entries;
  if (used + req > quota)
  {
    if (quota < used)
      used =
        quota;   /* cheat a bit for error message (to avoid negative numbers) */
    GNUNET_log (
      GNUNET_ERROR_TYPE_WARNING,
      _ (
        "Insufficient space (%llu bytes are available) to satisfy RESERVE request for %llu bytes\n"),
      quota - used,
      req);
    if (cache_size < req)
    {
      /* TODO: document this in the FAQ; essentially, if this
       * message happens, the insertion request could be blocked
       * by less-important content from migration because it is
       * larger than 1/8th of the overall available space, and
       * we only reserve 1/8th for "fresh" insertions */
      GNUNET_log (
        GNUNET_ERROR_TYPE_WARNING,
        _ (
          "The requested amount (%llu bytes) is larger than the cache size (%llu bytes)\n"),
        req,
        cache_size);
      transmit_status (client,
                       0,
                       gettext_noop (
                         "Insufficient space to satisfy request and "
                         "requested amount is larger than cache size"));
    }
    else
    {
      transmit_status (client,
                       0,
                       gettext_noop ("Insufficient space to satisfy request"));
    }
    GNUNET_SERVICE_client_continue (client);
    return;
  }
  reserved += req;
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# reserved"),
                         reserved,
                         GNUNET_NO);
  e = GNUNET_new (struct ReservationList);
  e->next = reservations;
  reservations = e;
  e->client = client;
  e->amount = amount;
  e->entries = entries;
  e->rid = ++reservation_gen;
  if (reservation_gen < 0)
    reservation_gen = 0; /* wrap around */
  transmit_status (client, e->rid, NULL);
  GNUNET_SERVICE_client_continue (client);
}

References _, ReservationList::amount, cache_size, ReservationList::client, ReservationList::entries, gettext_noop, GNUNET_DATASTORE_ENTRY_OVERHEAD, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_WARNING, GNUNET_log, GNUNET_new, GNUNET_NO, GNUNET_ntohll(), GNUNET_SERVICE_client_continue(), GNUNET_STATISTICS_set(), msg, ReservationList::next, payload, quota, reservations, reserved, ReservationList::rid, stats, and transmit_status().

Here is the call graph for this function:

handle_release_reserve()

static void handle_release_reserve ( void *  cls, const struct ReleaseReserveMessage *  msg  )

static

Handle RELEASE_RESERVE-message.

Parameters

cls	identification of the client
msg	the actual message

Definition at line 650 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
  struct ReservationList *pos;
  struct ReservationList *prev;
  struct ReservationList *next;
  int rid = ntohl (msg->rid);
  unsigned long long rem;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing RELEASE_RESERVE request\n");
  next = reservations;
  prev = NULL;
  while (NULL != (pos = next))
  {
    next = pos->next;
    if (rid == pos->rid)
    {
      if (prev == NULL)
        reservations = next;
      else
        prev->next = next;
      rem =
        pos->amount
        + ((unsigned long long) GNUNET_DATASTORE_ENTRY_OVERHEAD) * pos->entries;
      GNUNET_assert (reserved >= rem);
      reserved -= rem;
      GNUNET_STATISTICS_set (stats,
                             gettext_noop ("# reserved"),
                             reserved,
                             GNUNET_NO);
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Returning %llu remaining reserved bytes to storage pool\n",
                  rem);
      GNUNET_free (pos);
      transmit_status (client, GNUNET_OK, NULL);
      GNUNET_SERVICE_client_continue (client);
      return;
    }
    prev = pos;
  }
  GNUNET_break (0);
  transmit_status (client,
                   GNUNET_SYSERR,
                   gettext_noop ("Could not find matching reservation"));
  GNUNET_SERVICE_client_continue (client);
}

References ReservationList::amount, ReservationList::client, ReservationList::entries, gettext_noop, GNUNET_assert, GNUNET_break, GNUNET_DATASTORE_ENTRY_OVERHEAD, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_log, GNUNET_NO, GNUNET_OK, GNUNET_SERVICE_client_continue(), GNUNET_STATISTICS_set(), GNUNET_SYSERR, msg, ReservationList::next, reservations, reserved, ReservationList::rid, stats, and transmit_status().

Here is the call graph for this function:

check_data()

static int check_data ( const struct DataMessage *  dm )

static

Check that the given message is a valid data message.

Parameters

dm	message to check

Returns

GNUNET_SYSERR is not well-formed, otherwise GNUNET_OK

Definition at line 705 of file gnunet-service-datastore.c.

{
  uint16_t size;
  uint32_t dsize;
 
  size = ntohs (dm->header.size);
  dsize = ntohl (dm->size);
  if (size != dsize + sizeof(struct DataMessage))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}

References GNUNET_break, GNUNET_OK, GNUNET_SYSERR, DataMessage::header, GNUNET_MessageHeader::size, size, and DataMessage::size.

Referenced by check_put(), and check_remove().

Here is the caller graph for this function:

put_continuation()

static void put_continuation ( void *  cls, const struct GNUNET_HashCode *  key, uint32_t  size, int  status, const char *  msg  )

static

Put continuation.

Parameters

cls	closure
key	key for the item stored
size	size of the item stored
status	GNUNET_OK if inserted, GNUNET_NO if updated, or #GNUNET_SYSERROR if error
msg	error message on error

Definition at line 732 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
 
  if (GNUNET_OK == status)
  {
    GNUNET_STATISTICS_update (stats,
                              gettext_noop ("# bytes stored"),
                              size,
                              GNUNET_YES);
    GNUNET_CONTAINER_bloomfilter_add (filter, key);
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Successfully stored %u bytes under key `%s'\n",
                size,
                GNUNET_h2s (key));
  }
  transmit_status (client,
                   GNUNET_SYSERR == status ? GNUNET_SYSERR : GNUNET_OK,
                   msg);
  if (quota - reserved - cache_size < payload)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Need %llu bytes more space (%llu allowed, using %llu)\n"),
                (unsigned long long) size + GNUNET_DATASTORE_ENTRY_OVERHEAD,
                (unsigned long long) (quota - reserved - cache_size),
                (unsigned long long) payload);
    manage_space (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
  }
}

References _, cache_size, filter, gettext_noop, GNUNET_CONTAINER_bloomfilter_add(), GNUNET_DATASTORE_ENTRY_OVERHEAD, GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_INFO, GNUNET_h2s(), GNUNET_log, GNUNET_OK, GNUNET_STATISTICS_update(), GNUNET_SYSERR, GNUNET_YES, key, manage_space(), msg, payload, quota, reserved, size, stats, status, and transmit_status().

Referenced by handle_put().

Here is the call graph for this function:

Here is the caller graph for this function:

check_put()

static int check_put ( void *  cls, const struct DataMessage *  dm  )

static

Verify PUT-message.

Parameters

cls	identification of the client
dm	the actual message

Returns

GNUNET_OK if dm is well-formed

Definition at line 775 of file gnunet-service-datastore.c.

{
  if (GNUNET_OK != check_data (dm))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}

References check_data(), GNUNET_break, GNUNET_OK, and GNUNET_SYSERR.

Here is the call graph for this function:

handle_put()

static void handle_put ( void *  cls, const struct DataMessage *  dm  )

static

Handle PUT-message.

Parameters

cls	identification of the client
dm	the actual message

Definition at line 793 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
  int rid;
  struct ReservationList *pos;
  uint32_t size;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing PUT request for `%s' of type %u\n",
              GNUNET_h2s (&dm->key),
              (uint32_t) ntohl (dm->type));
  rid = ntohl (dm->rid);
  size = ntohl (dm->size);
  if (rid > 0)
  {
    pos = reservations;
    while ((NULL != pos) && (rid != pos->rid))
      pos = pos->next;
    GNUNET_break (pos != NULL);
    if (NULL != pos)
    {
      GNUNET_break (pos->entries > 0);
      GNUNET_break (pos->amount >= size);
      pos->entries--;
      pos->amount -= size;
      reserved -= (size + GNUNET_DATASTORE_ENTRY_OVERHEAD);
      GNUNET_STATISTICS_set (stats,
                             gettext_noop ("# reserved"),
                             reserved,
                             GNUNET_NO);
    }
  }
  {
    bool absent =
      GNUNET_NO == GNUNET_CONTAINER_bloomfilter_test (filter, &dm->key);
    plugin->api->put (plugin->api->cls,
                      &dm->key,
                      absent,
                      ntohl (dm->size),
                      &dm[1],
                      ntohl (dm->type),
                      ntohl (dm->priority),
                      ntohl (dm->anonymity),
                      ntohl (dm->replication),
                      GNUNET_TIME_absolute_ntoh (dm->expiration),
                      &put_continuation,
                      client);
  }
  GNUNET_SERVICE_client_continue (client);
}

References ReservationList::amount, DataMessage::anonymity, DatastorePlugin::api, ReservationList::client, GNUNET_DATASTORE_PluginFunctions::cls, ReservationList::entries, DataMessage::expiration, filter, gettext_noop, GNUNET_break, GNUNET_CONTAINER_bloomfilter_test(), GNUNET_DATASTORE_ENTRY_OVERHEAD, GNUNET_ERROR_TYPE_DEBUG, GNUNET_h2s(), GNUNET_log, GNUNET_NO, GNUNET_SERVICE_client_continue(), GNUNET_STATISTICS_set(), GNUNET_TIME_absolute_ntoh(), DataMessage::key, ReservationList::next, plugin, DataMessage::priority, GNUNET_DATASTORE_PluginFunctions::put, put_continuation(), DataMessage::replication, reservations, reserved, DataMessage::rid, ReservationList::rid, size, DataMessage::size, stats, and DataMessage::type.

Here is the call graph for this function:

handle_get()

static void handle_get ( void *  cls, const struct GetMessage *  msg  )

static

Handle GNUNET_MESSAGE_TYPE_DATASTORE_GET-message.

Parameters

cls	identification of the client
msg	the actual message

Definition at line 852 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing GET request of type %u\n",
              (uint32_t) ntohl (msg->type));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# GET requests received"),
                            1,
                            GNUNET_NO);
  plugin->api->get_key (plugin->api->cls,
                        GNUNET_ntohll (msg->next_uid),
                        msg->random,
                        NULL,
                        ntohl (msg->type),
                        &transmit_item,
                        client);
  GNUNET_SERVICE_client_continue (client);
}

References DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, GNUNET_DATASTORE_PluginFunctions::get_key, gettext_noop, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_NO, GNUNET_ntohll(), GNUNET_SERVICE_client_continue(), GNUNET_STATISTICS_update(), msg, plugin, stats, transmit_item(), and GNUNET_MessageHeader::type.

Here is the call graph for this function:

handle_get_key()

static void handle_get_key ( void *  cls, const struct GetKeyMessage *  msg  )

static

Handle GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY-message.

Parameters

cls	closure
msg	the actual message

Definition at line 881 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing GET request for `%s' of type %u\n",
              GNUNET_h2s (&msg->key),
              (uint32_t) ntohl (msg->type));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# GET KEY requests received"),
                            1,
                            GNUNET_NO);
  if (GNUNET_YES != GNUNET_CONTAINER_bloomfilter_test (filter, &msg->key))
  {
    /* don't bother database... */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Empty result set for GET request for `%s' (bloomfilter).\n",
                GNUNET_h2s (&msg->key));
    GNUNET_STATISTICS_update (stats,
                              gettext_noop (
                                "# requests filtered by bloomfilter"),
                              1,
                              GNUNET_NO);
    transmit_item (client,
                   NULL,
                   0,
                   NULL,
                   0,
                   0,
                   0,
                   0,
                   GNUNET_TIME_UNIT_ZERO_ABS,
                   0);
    GNUNET_SERVICE_client_continue (client);
    return;
  }
  plugin->api->get_key (plugin->api->cls,
                        GNUNET_ntohll (msg->next_uid),
                        msg->random,
                        &msg->key,
                        ntohl (msg->type),
                        &transmit_item,
                        client);
  GNUNET_SERVICE_client_continue (client);
}

References DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, filter, GNUNET_DATASTORE_PluginFunctions::get_key, gettext_noop, GNUNET_CONTAINER_bloomfilter_test(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_h2s(), GNUNET_log, GNUNET_NO, GNUNET_ntohll(), GNUNET_SERVICE_client_continue(), GNUNET_STATISTICS_update(), GNUNET_TIME_UNIT_ZERO_ABS, GNUNET_YES, msg, plugin, stats, transmit_item(), and GNUNET_MessageHeader::type.

Here is the call graph for this function:

handle_get_replication()

static void handle_get_replication ( void *  cls, const struct GNUNET_MessageHeader *  message  )

static

Handle GET_REPLICATION-message.

Parameters

cls	identification of the client
message	the actual message

Definition at line 935 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing GET_REPLICATION request\n");
  GNUNET_STATISTICS_update (stats,
                            gettext_noop (
                              "# GET REPLICATION requests received"),
                            1,
                            GNUNET_NO);
  plugin->api->get_replication (plugin->api->cls, &transmit_item, client);
  GNUNET_SERVICE_client_continue (client);
}

References DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, GNUNET_DATASTORE_PluginFunctions::get_replication, gettext_noop, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_NO, GNUNET_SERVICE_client_continue(), GNUNET_STATISTICS_update(), plugin, stats, and transmit_item().

Here is the call graph for this function:

handle_get_zero_anonymity()

static void handle_get_zero_anonymity ( void *  cls, const struct GetZeroAnonymityMessage *  msg  )

static

Handle GET_ZERO_ANONYMITY-message.

Parameters

cls	client identification of the client
msg	the actual message

Definition at line 957 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
  enum GNUNET_BLOCK_Type type;
 
  type = ntohl (msg->type);
  if (type == GNUNET_BLOCK_TYPE_ANY)
  {
    GNUNET_break (0);
    GNUNET_SERVICE_client_drop (client);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing GET_ZERO_ANONYMITY request\n");
  GNUNET_STATISTICS_update (stats,
                            gettext_noop (
                              "# GET ZERO ANONYMITY requests received"),
                            1,
                            GNUNET_NO);
  plugin->api->get_zero_anonymity (plugin->api->cls,
                                   GNUNET_ntohll (msg->next_uid),
                                   type,
                                   &transmit_item,
                                   client);
  GNUNET_SERVICE_client_continue (client);
}

References DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, GNUNET_DATASTORE_PluginFunctions::get_zero_anonymity, gettext_noop, GNUNET_BLOCK_TYPE_ANY, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_NO, GNUNET_ntohll(), GNUNET_SERVICE_client_continue(), GNUNET_SERVICE_client_drop(), GNUNET_STATISTICS_update(), msg, plugin, stats, transmit_item(), type, and GNUNET_MessageHeader::type.

Here is the call graph for this function:

remove_continuation()

static void remove_continuation ( void *  cls, const struct GNUNET_HashCode *  key, uint32_t  size, int  status, const char *  msg  )

static

Remove continuation.

Parameters

cls	closure
key	key for the content
size	number of bytes in data
status	GNUNET_OK if removed, GNUNET_NO if not found, or #GNUNET_SYSERROR if error
msg	error message on error

Definition at line 996 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
 
  if (GNUNET_SYSERR == status)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "REMOVE request failed: %s.\n", msg);
    transmit_status (client, GNUNET_NO, msg);
    return;
  }
  if (GNUNET_NO == status)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Content not found for REMOVE request.\n");
    transmit_status (client, GNUNET_NO, _ ("Content not found"));
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Item matches REMOVE request for key `%s'.\n",
              GNUNET_h2s (key));
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# bytes removed (explicit request)"),
                            size,
                            GNUNET_YES);
  GNUNET_CONTAINER_bloomfilter_remove (filter, key);
  transmit_status (client, GNUNET_OK, NULL);
}

References _, filter, gettext_noop, GNUNET_CONTAINER_bloomfilter_remove(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_h2s(), GNUNET_log, GNUNET_NO, GNUNET_OK, GNUNET_STATISTICS_update(), GNUNET_SYSERR, GNUNET_YES, key, msg, size, stats, status, and transmit_status().

Referenced by handle_remove().

Here is the call graph for this function:

Here is the caller graph for this function:

check_remove()

static int check_remove ( void *  cls, const struct DataMessage *  dm  )

static

Verify REMOVE-message.

Parameters

cls	identification of the client
dm	the actual message

Returns

GNUNET_OK if dm is well-formed

Definition at line 1037 of file gnunet-service-datastore.c.

{
  if (GNUNET_OK != check_data (dm))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  return GNUNET_OK;
}

References check_data(), GNUNET_break, GNUNET_OK, and GNUNET_SYSERR.

Here is the call graph for this function:

handle_remove()

static void handle_remove ( void *  cls, const struct DataMessage *  dm  )

static

Handle REMOVE-message.

Parameters

cls	closure
dm	the actual message

Definition at line 1055 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_STATISTICS_update (stats,
                            gettext_noop ("# REMOVE requests received"),
                            1,
                            GNUNET_NO);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing REMOVE request for `%s'\n",
              GNUNET_h2s (&dm->key));
  plugin->api->remove_key (plugin->api->cls,
                           &dm->key,
                           ntohl (dm->size),
                           &dm[1],
                           &remove_continuation,
                           client);
  GNUNET_SERVICE_client_continue (client);
}

References DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, gettext_noop, GNUNET_ERROR_TYPE_DEBUG, GNUNET_h2s(), GNUNET_log, GNUNET_NO, GNUNET_SERVICE_client_continue(), GNUNET_STATISTICS_update(), DataMessage::key, plugin, remove_continuation(), GNUNET_DATASTORE_PluginFunctions::remove_key, DataMessage::size, and stats.

Here is the call graph for this function:

handle_drop()

static void handle_drop ( void *  cls, const struct GNUNET_MessageHeader *  message  )

static

Handle DROP-message.

Parameters

cls	identification of the client
message	the actual message

Definition at line 1083 of file gnunet-service-datastore.c.

{
  struct GNUNET_SERVICE_Client *client = cls;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Processing DROP request\n");
  do_drop = GNUNET_YES;
  GNUNET_SERVICE_client_continue (client);
}

References do_drop, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_SERVICE_client_continue(), and GNUNET_YES.

Here is the call graph for this function:

disk_utilization_change_cb()

static void disk_utilization_change_cb ( void *  cls, int  delta  )

static

Function called by plugins to notify us about a change in their disk utilization.

Parameters

cls	closure (NULL)
delta	change in disk utilization, 0 for "reset to empty"

Definition at line 1104 of file gnunet-service-datastore.c.

{
  if ((delta < 0) && (payload < -delta))
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_WARNING,
      _ (
        "Datastore payload must have been inaccurate (%lld < %lld). Recomputing it.\n"),
      (long long) payload,
      (long long) -delta);
    plugin->api->estimate_size (plugin->api->cls, &payload);
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("New payload: %lld\n"),
                (long long) payload);
    sync_stats ();
    return;
  }
  payload += delta;
  last_sync++;
  if (last_sync >= MAX_STAT_SYNC_LAG)
    sync_stats ();
}

References _, DatastorePlugin::api, GNUNET_DATASTORE_PluginFunctions::cls, delta, GNUNET_DATASTORE_PluginFunctions::estimate_size, GNUNET_ERROR_TYPE_INFO, GNUNET_ERROR_TYPE_WARNING, GNUNET_log, last_sync, MAX_STAT_SYNC_LAG, payload, plugin, and sync_stats().

Referenced by load_plugin().

Here is the call graph for this function:

Here is the caller graph for this function:

process_stat_in()

static int process_stat_in ( void *  cls, const char *  subsystem, const char *  name, uint64_t  value, int  is_persistent  )

static

Callback function to process statistic values.

Parameters

cls	closure (struct Plugin*)
subsystem	name of subsystem that created the statistic
name	the name of the datum
value	the current value
is_persistent	GNUNET_YES if the value is persistent, GNUNET_NO if not

Returns

GNUNET_OK to continue, GNUNET_SYSERR to abort iteration

Definition at line 1139 of file gnunet-service-datastore.c.

{
  GNUNET_assert (GNUNET_NO == stats_worked);
  stats_worked = GNUNET_YES;
  payload += value;
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Notification from statistics about existing payload (%llu), new payload is %llu\n",
    (unsigned long long) value,
    (unsigned long long) payload);
  return GNUNET_OK;
}

References GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_NO, GNUNET_OK, GNUNET_YES, payload, stats_worked, and value.

Referenced by run().

Here is the caller graph for this function:

load_plugin()

static struct DatastorePlugin * load_plugin ( )

static

Load the datastore plugin.

Definition at line 1161 of file gnunet-service-datastore.c.

{
  struct DatastorePlugin *ret;
  char *libname;
 
  ret = GNUNET_new (struct DatastorePlugin);
  ret->env.cfg = cfg;
  ret->env.duc = &disk_utilization_change_cb;
  ret->env.cls = NULL;
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              _ ("Loading `%s' datastore plugin\n"),
              plugin_name);
  GNUNET_asprintf (&libname,
                   "libgnunet_plugin_datastore_%s",
                   plugin_name);
  ret->short_name = GNUNET_strdup (plugin_name);
  ret->lib_name = libname;
  ret->api = GNUNET_PLUGIN_load (GNUNET_OS_project_data_gnunet (),
                                 libname,
                                 &ret->env);
  if (NULL == ret->api)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("Failed to load datastore plugin for `%s'\n"),
                plugin_name);
    GNUNET_free (ret->short_name);
    GNUNET_free (libname);
    GNUNET_free (ret);
    return NULL;
  }
  return ret;
}

References _, cfg, disk_utilization_change_cb(), GNUNET_asprintf(), GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_INFO, GNUNET_free, GNUNET_log, GNUNET_new, GNUNET_OS_project_data_gnunet(), GNUNET_PLUGIN_load(), GNUNET_strdup, plugin_name, and ret.

Referenced by process_stat_done().

Here is the call graph for this function:

Here is the caller graph for this function:

unload_plugin()

static void unload_plugin ( struct DatastorePlugin *  plug )

static

Function called when the service shuts down.

Unloads our datastore plugin.

Parameters

plug	plugin to unload

Definition at line 1202 of file gnunet-service-datastore.c.

{
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Datastore service is unloading plugin...\n");
  GNUNET_break (NULL == GNUNET_PLUGIN_unload (plug->lib_name, plug->api));
  GNUNET_free (plug->lib_name);
  GNUNET_free (plug->short_name);
  GNUNET_free (plug);
}

References DatastorePlugin::api, GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_log, GNUNET_PLUGIN_unload(), DatastorePlugin::lib_name, and DatastorePlugin::short_name.

Referenced by cleaning_task().

Here is the call graph for this function:

Here is the caller graph for this function:

begin_service()

static void begin_service ( )

static

Initialization complete, start operating the service.

Definition at line 1217 of file gnunet-service-datastore.c.

{
  GNUNET_SERVICE_resume (service);
  expired_kill_task =
    GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
                                        &delete_expired,
                                        NULL);
}

References delete_expired(), expired_kill_task, GNUNET_SCHEDULER_add_with_priority(), GNUNET_SCHEDULER_PRIORITY_IDLE, GNUNET_SERVICE_resume(), and service.

Referenced by add_key_to_bloomfilter(), and process_stat_done().

Here is the call graph for this function:

Here is the caller graph for this function:

add_key_to_bloomfilter()

static void add_key_to_bloomfilter ( void *  cls, const struct GNUNET_HashCode *  key, unsigned int  count  )

static

Adds a given key to the bloomfilter in cls count times.

Parameters

cls	the bloomfilter
key	key to add
count	number of times to add key

Definition at line 1235 of file gnunet-service-datastore.c.

{
  struct GNUNET_CONTAINER_BloomFilter *bf = cls;
 
  if (NULL == key)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Bloomfilter construction complete.\n"));
    begin_service ();
    return;
  }
 
  while (0 < count--)
    GNUNET_CONTAINER_bloomfilter_add (bf, key);
}

References _, begin_service(), GNUNET_CONTAINER_bloomfilter_add(), GNUNET_ERROR_TYPE_INFO, GNUNET_log, and key.

Referenced by process_stat_done().

Here is the call graph for this function:

Here is the caller graph for this function:

process_stat_done()

static void process_stat_done ( void *  cls, int  success  )

static

We finished receiving the statistic.

Initialize the plugin; if loading the statistic failed, run the estimator.

Parameters

cls	NULL
success	GNUNET_NO if we failed to read the stat

Definition at line 1262 of file gnunet-service-datastore.c.

{
  stat_get = NULL;
  if (NULL != stat_timeout_task)
  {
    GNUNET_SCHEDULER_cancel (stat_timeout_task);
    stat_timeout_task = NULL;
  }
  plugin = load_plugin ();
  if (NULL == plugin)
  {
    GNUNET_CONTAINER_bloomfilter_free (filter);
    filter = NULL;
    if (NULL != stats)
    {
      GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
      stats = NULL;
    }
    return;
  }
 
  if (GNUNET_NO == stats_worked)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Failed to obtain value from statistics service, recomputing it\n");
    plugin->api->estimate_size (plugin->api->cls, &payload);
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("New payload: %lld\n"),
                (long long) payload);
  }
 
  if (GNUNET_YES == refresh_bf)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                _ ("Rebuilding bloomfilter.  Please be patient.\n"));
    if (NULL != plugin->api->get_keys)
    {
      plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);
      return;
    }
    else
    {
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  _ (
                    "Plugin does not support get_keys function. Please fix!\n"))
      ;
    }
  }
  begin_service ();
}

References _, add_key_to_bloomfilter(), DatastorePlugin::api, begin_service(), GNUNET_DATASTORE_PluginFunctions::cls, GNUNET_DATASTORE_PluginFunctions::estimate_size, filter, GNUNET_DATASTORE_PluginFunctions::get_keys, GNUNET_CONTAINER_bloomfilter_free(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_INFO, GNUNET_log, GNUNET_NO, GNUNET_SCHEDULER_cancel(), GNUNET_STATISTICS_destroy(), GNUNET_YES, load_plugin(), payload, plugin, refresh_bf, stat_get, stat_timeout_task, stats, and stats_worked.

Referenced by run(), and stat_timeout().

Here is the call graph for this function:

Here is the caller graph for this function:

stat_timeout()

static void stat_timeout ( void *  cls )

static

Fetching stats took to long, run without.

Parameters

cls

NULL

Definition at line 1320 of file gnunet-service-datastore.c.

{
  stat_timeout_task = NULL;
  GNUNET_STATISTICS_get_cancel (stat_get);
  process_stat_done (NULL, GNUNET_NO);
}

References GNUNET_NO, GNUNET_STATISTICS_get_cancel(), process_stat_done(), stat_get, and stat_timeout_task.

Referenced by run().

Here is the call graph for this function:

Here is the caller graph for this function:

cleaning_task()

static void cleaning_task ( void *  cls )

static

Task run during shutdown.

Definition at line 1332 of file gnunet-service-datastore.c.

{
  cleaning_done = GNUNET_YES;
  if (NULL != expired_kill_task)
  {
    GNUNET_SCHEDULER_cancel (expired_kill_task);
    expired_kill_task = NULL;
  }
  if (GNUNET_YES == do_drop)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Dropping database!\n");
    plugin->api->drop (plugin->api->cls);
    payload = 0;
    last_sync++;
  }
  if (NULL != plugin)
  {
    unload_plugin (plugin);
    plugin = NULL;
  }
  if (NULL != filter)
  {
    GNUNET_CONTAINER_bloomfilter_free (filter);
    filter = NULL;
  }
  if (NULL != stat_get)
  {
    GNUNET_STATISTICS_get_cancel (stat_get);
    stat_get = NULL;
  }
  if (NULL != stat_timeout_task)
  {
    GNUNET_SCHEDULER_cancel (stat_timeout_task);
    stat_timeout_task = NULL;
  }
  GNUNET_free (plugin_name);
  plugin_name = NULL;
  if (last_sync > 0)
    sync_stats ();
  if (NULL != stats)
  {
    GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
    stats = NULL;
  }
  GNUNET_free (quota_stat_name);
  quota_stat_name = NULL;
}

References DatastorePlugin::api, cleaning_done, GNUNET_DATASTORE_PluginFunctions::cls, do_drop, GNUNET_DATASTORE_PluginFunctions::drop, expired_kill_task, filter, GNUNET_CONTAINER_bloomfilter_free(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_free, GNUNET_log, GNUNET_SCHEDULER_cancel(), GNUNET_STATISTICS_destroy(), GNUNET_STATISTICS_get_cancel(), GNUNET_YES, last_sync, payload, plugin, plugin_name, quota_stat_name, stat_get, stat_timeout_task, stats, sync_stats(), and unload_plugin().

Referenced by run().

Here is the call graph for this function:

Here is the caller graph for this function:

client_connect_cb()

static void * client_connect_cb ( void *  cls, struct GNUNET_SERVICE_Client *  client, struct GNUNET_MQ_Handle *  mq  )

static

Add a client to our list of active clients.

Parameters

cls	NULL
client	client to add
mq	message queue for client

Returns

client

Definition at line 1391 of file gnunet-service-datastore.c.

{
  return client;
}

client_disconnect_cb()

static void client_disconnect_cb ( void *  cls, struct GNUNET_SERVICE_Client *  client, void *  app_ctx  )

static

Called whenever a client is disconnected.

Frees our resources associated with that client.

Parameters

cls	closure
client	identification of the client
app_ctx	must match client

Definition at line 1408 of file gnunet-service-datastore.c.

{
  struct ReservationList *pos;
  struct ReservationList *prev;
  struct ReservationList *next;
 
  GNUNET_assert (app_ctx == client);
  prev = NULL;
  pos = reservations;
  while (NULL != pos)
  {
    next = pos->next;
    if (pos->client == client)
    {
      if (NULL == prev)
        reservations = next;
      else
        prev->next = next;
      reserved -= pos->amount + pos->entries * GNUNET_DATASTORE_ENTRY_OVERHEAD;
      GNUNET_free (pos);
    }
    else
    {
      prev = pos;
    }
    pos = next;
  }
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# reserved"),
                         reserved,
                         GNUNET_NO);
}

References ReservationList::amount, ReservationList::client, ReservationList::entries, gettext_noop, GNUNET_assert, GNUNET_DATASTORE_ENTRY_OVERHEAD, GNUNET_free, GNUNET_NO, GNUNET_STATISTICS_set(), ReservationList::next, reservations, reserved, and stats.

Here is the call graph for this function:

run()

static void run ( void *  cls, const struct GNUNET_CONFIGURATION_Handle *  c, struct GNUNET_SERVICE_Handle *  serv  )

static

Process datastore requests.

Parameters

cls	closure
serv	the initialized service
c	configuration to use

Definition at line 1452 of file gnunet-service-datastore.c.

{
  char *fn;
  char *pfn;
  unsigned int bf_size;
 
  service = serv;
  cfg = c;
  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg,
                                                          "DATASTORE",
                                                          "DATABASE",
                                                          &plugin_name))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                               "DATABASE",
                               "DATASTORE");
    return;
  }
  GNUNET_asprintf (&quota_stat_name,
                   _ ("# bytes used in file-sharing datastore `%s'"),
                   plugin_name);
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", &quota))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "QUOTA", "DATASTORE");
    return;
  }
  stats = GNUNET_STATISTICS_create ("datastore", cfg);
  GNUNET_STATISTICS_set (stats, gettext_noop ("# quota"), quota, GNUNET_NO);
  cache_size = quota / 8; /* Or should we make this an option? */
  GNUNET_STATISTICS_set (stats,
                         gettext_noop ("# cache size"),
                         cache_size,
                         GNUNET_NO);
  if (quota / (32 * 1024LL) > MAX_BF_SIZE)
    bf_size = MAX_BF_SIZE;
  else
    bf_size =
      quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */
  fn = NULL;
  if ((GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                             "DATASTORE",
                                                             "BLOOMFILTER",
                                                             &fn)) ||
      (GNUNET_OK != GNUNET_DISK_directory_create_for_file (fn)))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                _ ("Could not use specified filename `%s' for bloomfilter.\n"),
                NULL != fn ? fn : "");
    GNUNET_free (fn);
    fn = NULL;
  }
  if (NULL != fn)
  {
    GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
    if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
    {
      filter =
        GNUNET_CONTAINER_bloomfilter_load (pfn,
                                           bf_size,
                                           5);    /* approx. 3% false positives at max use */
      if (NULL == filter)
      {
        /* file exists but not valid, remove and try again, but refresh */
        if (0 != unlink (pfn))
        {
          /* failed to remove, run without file */
          GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                      _ ("Failed to remove bogus bloomfilter file `%s'\n"),
                      pfn);
          GNUNET_free (pfn);
          pfn = NULL;
          filter = GNUNET_CONTAINER_bloomfilter_load (
            NULL,
            bf_size,
            5);         /* approx. 3% false positives at max use */
          refresh_bf = GNUNET_YES;
        }
        else
        {
          /* try again after remove */
          filter = GNUNET_CONTAINER_bloomfilter_load (
            pfn,
            bf_size,
            5);         /* approx. 3% false positives at max use */
          refresh_bf = GNUNET_YES;
          if (NULL == filter)
          {
            /* failed yet again, give up on using file */
            GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                        _ ("Failed to remove bogus bloomfilter file `%s'\n"),
                        pfn);
            GNUNET_free (pfn);
            pfn = NULL;
            filter = GNUNET_CONTAINER_bloomfilter_init (
              NULL,
              bf_size,
              5);           /* approx. 3% false positives at max use */
          }
        }
      }
      else
      {
        /* normal case: have an existing valid bf file, no need to refresh */
        refresh_bf = GNUNET_NO;
      }
    }
    else
    {
      filter =
        GNUNET_CONTAINER_bloomfilter_load (pfn,
                                           bf_size,
                                           5);    /* approx. 3% false positives at max use */
      refresh_bf = GNUNET_YES;
    }
    GNUNET_free (pfn);
  }
  else
  {
    filter =
      GNUNET_CONTAINER_bloomfilter_init (NULL,
                                         bf_size,
                                         5);  /* approx. 3% false positives at max use */
    refresh_bf = GNUNET_YES;
  }
  GNUNET_free (fn);
  if (NULL == filter)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("Failed to initialize bloomfilter.\n"));
    if (NULL != stats)
    {
      GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
      stats = NULL;
    }
    return;
  }
  GNUNET_SERVICE_suspend (service);
  stat_get = GNUNET_STATISTICS_get (stats,
                                    "datastore",
                                    quota_stat_name,
                                    &process_stat_done,
                                    &process_stat_in,
                                    NULL);
  if (NULL == stat_get)
    process_stat_done (NULL, GNUNET_SYSERR);
  else
    stat_timeout_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_SECONDS,
                                                      &stat_timeout,
                                                      NULL);
  GNUNET_SCHEDULER_add_shutdown (&cleaning_task, NULL);
}

References _, cache_size, cfg, cleaning_task(), filter, gettext_noop, GNUNET_asprintf(), GNUNET_CONFIGURATION_get_value_filename(), GNUNET_CONFIGURATION_get_value_size(), GNUNET_CONFIGURATION_get_value_string(), GNUNET_CONTAINER_bloomfilter_init(), GNUNET_CONTAINER_bloomfilter_load(), GNUNET_DISK_directory_create_for_file(), GNUNET_DISK_file_test(), GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_WARNING, GNUNET_free, GNUNET_log, GNUNET_log_config_missing(), GNUNET_NO, GNUNET_OK, GNUNET_SCHEDULER_add_delayed(), GNUNET_SCHEDULER_add_shutdown(), GNUNET_SERVICE_suspend(), GNUNET_STATISTICS_create(), GNUNET_STATISTICS_destroy(), GNUNET_STATISTICS_get(), GNUNET_STATISTICS_set(), GNUNET_SYSERR, GNUNET_TIME_UNIT_SECONDS, GNUNET_YES, MAX_BF_SIZE, plugin_name, process_stat_done(), process_stat_in(), quota, quota_stat_name, refresh_bf, service, stat_get, stat_timeout(), stat_timeout_task, and stats.

Here is the call graph for this function:

GNUNET_SERVICE_MAIN()

GNUNET_SERVICE_MAIN	(	GNUNET_OS_project_data_gnunet()	,
		"datastore"	,
		GNUNET_SERVICE_OPTION_NONE	,
		&	run,
		&	client_connect_cb,
		&	client_disconnect_cb,
		NULL	,
		GNUNET_MQ_hd_fixed_size(reserve, GNUNET_MESSAGE_TYPE_DATASTORE_RESERVE, struct ReserveMessage, NULL)	,
		GNUNET_MQ_hd_fixed_size(release_reserve, GNUNET_MESSAGE_TYPE_DATASTORE_RELEASE_RESERVE, struct ReleaseReserveMessage, NULL)	,
		GNUNET_MQ_hd_var_size(put, GNUNET_MESSAGE_TYPE_DATASTORE_PUT, struct DataMessage, NULL)	,
		GNUNET_MQ_hd_fixed_size(get, GNUNET_MESSAGE_TYPE_DATASTORE_GET, struct GetMessage, NULL)	,
		GNUNET_MQ_hd_fixed_size(get_key, GNUNET_MESSAGE_TYPE_DATASTORE_GET_KEY, struct GetKeyMessage, NULL)	,
		GNUNET_MQ_hd_fixed_size(get_replication, GNUNET_MESSAGE_TYPE_DATASTORE_GET_REPLICATION, struct GNUNET_MessageHeader, NULL)	,
		GNUNET_MQ_hd_fixed_size(get_zero_anonymity, GNUNET_MESSAGE_TYPE_DATASTORE_GET_ZERO_ANONYMITY, struct GetZeroAnonymityMessage, NULL)	,
		GNUNET_MQ_hd_var_size(remove, GNUNET_MESSAGE_TYPE_DATASTORE_REMOVE, struct DataMessage, NULL)	,
		GNUNET_MQ_hd_fixed_size(drop, GNUNET_MESSAGE_TYPE_DATASTORE_DROP, struct GNUNET_MessageHeader, NULL)	,
		GNUNET_MQ_handler_end()
	)

Define "main" method using service macro.

Variable Documentation

quota_stat_name

char* quota_stat_name

static

Name under which we store current space consumption.

Definition at line 61 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), run(), and sync_stats().

stat_timeout_task

struct GNUNET_SCHEDULER_Task* stat_timeout_task

static

Task to timeout stat GET.

Definition at line 66 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), GNUNET_HOSTLIST_client_start(), process_stat_done(), run(), and stat_timeout().

plugin

struct DatastorePlugin* plugin

static

Our datastore plugin (NULL if not available).

Definition at line 139 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), delete_expired(), disk_utilization_change_cb(), handle_get(), handle_get_key(), handle_get_replication(), handle_get_zero_anonymity(), handle_put(), handle_remove(), manage_space(), and process_stat_done().

reservations

struct ReservationList* reservations

static

Linked list of space reservations made by clients.

Definition at line 144 of file gnunet-service-datastore.c.

Referenced by client_disconnect_cb(), handle_put(), handle_release_reserve(), and handle_reserve().

filter

struct GNUNET_CONTAINER_BloomFilter* filter

static

Bloomfilter to quickly tell if we don't have the content.

Definition at line 149 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), expired_processor(), get_audiobin(), GNUNET_GNSRECORD_normalize_record_set(), GNUNET_NAMESTORE_records_lookup2(), GNUNET_NAMESTORE_zone_iteration_start2(), GNUNET_NAMESTORE_zone_monitor_start2(), handle_get_key(), handle_put(), main(), process_stat_done(), put_continuation(), quota_processor(), records_lookup(), remove_continuation(), run(), and send_lookup_response_with_filter().

plugin_name

char* plugin_name

static

Name of our plugin.

Definition at line 154 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), load_plugin(), and run().

cfg

const struct GNUNET_CONFIGURATION_Handle* cfg

static

Our configuration.

Definition at line 159 of file gnunet-service-datastore.c.

Referenced by load_plugin(), and run().

stats

struct GNUNET_STATISTICS_Handle* stats

static

Handle for reporting statistics.

Definition at line 164 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), client_disconnect_cb(), expired_processor(), handle_get(), handle_get_key(), handle_get_replication(), handle_get_zero_anonymity(), handle_put(), handle_release_reserve(), handle_remove(), handle_reserve(), process_stat_done(), put_continuation(), quota_processor(), remove_continuation(), run(), sync_stats(), and transmit_item().

cache_size

unsigned long long cache_size

static

How much space are we using for the cache? (space available for insertions that will be instantly reclaimed by discarding less important content — or possibly whatever we just inserted into the "cache").

Definition at line 172 of file gnunet-service-datastore.c.

Referenced by handle_reserve(), put_continuation(), and run().

reserved

unsigned long long reserved

static

How much space have we currently reserved?

Definition at line 177 of file gnunet-service-datastore.c.

Referenced by client_disconnect_cb(), handle_put(), handle_release_reserve(), handle_reserve(), and put_continuation().

payload

unsigned long long payload

static

How much data are we currently storing in the database?

Definition at line 183 of file gnunet-service-datastore.c.

Referenced by block_create_ecdsa(), block_create_eddsa(), block_decrypt_ecdsa(), block_decrypt_eddsa(), cleaning_task(), disk_utilization_change_cb(), GNUNET_TRANSPORT_TESTING_transport_communicator_send(), GNUNET_TUN_calculate_icmp_checksum(), GNUNET_TUN_calculate_tcp4_checksum(), GNUNET_TUN_calculate_tcp6_checksum(), GNUNET_TUN_calculate_udp4_checksum(), GNUNET_TUN_calculate_udp6_checksum(), GSC_KX_encrypt_and_transmit(), handle_alices_cryptodata_message(), handle_alices_cryptodata_message(), handle_bobs_cryptodata_message(), handle_bobs_cryptodata_multipart(), handle_client_response(), handle_icmp_remote(), handle_icmp_service(), handle_local_data(), handle_reserve(), handle_stun(), handle_tcp_remote(), handle_udp_remote(), OIDC_build_authz_code(), parse(), prepare_ipv4_packet(), prepare_ipv6_packet(), process_stat_done(), process_stat_in(), put_continuation(), queue_send_msg(), route_packet(), send_alices_cryptodata_message(), send_alices_cryptodata_message(), send_icmp_packet_via_tun(), send_reply(), send_tcp_packet_via_tun(), send_udp_packet_via_tun(), sync_stats(), transmit_bobs_cryptodata_message(), and transmit_bobs_cryptodata_message_multipart().

expired_kill_task

struct GNUNET_SCHEDULER_Task* expired_kill_task

static

Identity of the task that is used to delete expired content.

Definition at line 189 of file gnunet-service-datastore.c.

Referenced by begin_service(), cleaning_task(), delete_expired(), and expired_processor().

min_expiration

struct GNUNET_TIME_Absolute min_expiration

static

Minimum time that content should have to not be discarded instantly (time stamp of any content that we've been discarding recently to stay below the quota).

FOREVER if we had to expire content with non-zero priority.

Definition at line 197 of file gnunet-service-datastore.c.

Referenced by expired_processor(), put_migration_continuation(), quota_processor(), and transmit_status().

quota

unsigned long long quota

static

How much space are we allowed to use?

Definition at line 202 of file gnunet-service-datastore.c.

Referenced by GNUNET_DATACACHE_create(), handle_reserve(), put_continuation(), and run().

do_drop

int do_drop

static

Should the database be dropped on exit?

Definition at line 207 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), and handle_drop().

refresh_bf

int refresh_bf

static

Should we refresh the BF when the DB is loaded?

Definition at line 212 of file gnunet-service-datastore.c.

Referenced by process_stat_done(), and run().

last_sync

unsigned int last_sync

static

Number of updates that were made to the payload value since we last synchronized it with the statistics service.

Definition at line 219 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), disk_utilization_change_cb(), and sync_stats().

stats_worked

int stats_worked

static

Did we get an answer from statistics?

Definition at line 224 of file gnunet-service-datastore.c.

Referenced by process_stat_done(), and process_stat_in().

cleaning_done

int cleaning_done

static

Have we already cleaned up the TCCs and are hence no longer willing (or able) to transmit anything to anyone?

Definition at line 247 of file gnunet-service-datastore.c.

Referenced by cleaning_task().

stat_get

struct GNUNET_STATISTICS_GetHandle* stat_get

static

Handle for pending get request.

Definition at line 252 of file gnunet-service-datastore.c.

Referenced by cleaning_task(), process_stat_done(), run(), and stat_timeout().

service

struct GNUNET_SERVICE_Handle* service

static

Handle to our server.

Definition at line 257 of file gnunet-service-datastore.c.

Referenced by begin_service(), and run().