download methods More...

#include "platform.h"
#include "gnunet_constants.h"
#include "gnunet_fs_service.h"
#include "fs_api.h"
#include "fs_tree.h"

Include dependency graph for fs_download.c:

Data Structures
struct	ProcessResultClosure
	Closure for iterator processing results. More...

Macros
#define	GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX

Functions
static int	is_recursive_download (struct GNUNET_FS_DownloadContext *dc)
	Determine if the given download (options and meta data) should cause use to try to do a recursive download. More...

static uint64_t	compute_disk_offset (uint64_t fsize, uint64_t off, unsigned int depth)
	We're storing the IBLOCKS after the DBLOCKS on disk (so that we only have to truncate the file once we're done). More...

void	GNUNET_FS_download_make_status_ (struct GNUNET_FS_ProgressInfo pi, struct GNUNET_FS_DownloadContext dc)
	Fill in all of the generic fields for a download event and call the callback. More...

static int	process_result_with_request (void cls, const struct GNUNET_HashCode key, void *value)
	Iterator over entries in the pending requests in the 'active' map for the reply that we just got. More...

static int	encrypt_existing_match (struct GNUNET_FS_DownloadContext dc, const struct ContentHashKey chk, struct DownloadRequest dr, const char block, size_t len, int do_store)
	We've found a matching block without downloading it. More...

static void	try_reconnect (struct GNUNET_FS_DownloadContext *dc)
	We've lost our connection with the FS service. More...

static void	trigger_recursive_download (void cls, const char filename, const struct GNUNET_FS_Uri uri, const struct GNUNET_FS_MetaData meta, size_t length, const void *data)
	We found an entry in a directory. More...

static void	full_recursive_download (struct GNUNET_FS_DownloadContext *dc)
	We're done downloading a directory. More...

static void	check_completed (struct GNUNET_FS_DownloadContext *dc)
	Check if all child-downloads have completed (or trigger them if necessary) and once we're completely done, signal completion (and possibly recurse to parent). More...

static void	try_match_block (struct GNUNET_FS_DownloadContext dc, struct DownloadRequest dr, const char *data, size_t data_len)
	We got a block of plaintext data (from the meta data). More...

static int	match_full_data (void cls, const char plugin_name, enum EXTRACTOR_MetaType type, enum EXTRACTOR_MetaFormat format, const char data_mime_type, const char data, size_t data_len)
	Type of a function that libextractor calls for each meta data item found. More...

static void	propagate_up (struct DownloadRequest *dr)
	Set the state of the given download request to BRS_DOWNLOAD_UP and propagate it up the tree. More...

static void	try_top_down_reconstruction (struct GNUNET_FS_DownloadContext dc, struct DownloadRequest dr)
	Try top-down reconstruction. More...

static int	retry_entry (void cls, const struct GNUNET_HashCode key, void *entry)
	Add entries to the message queue. More...

static void	schedule_block_download (struct GNUNET_FS_DownloadContext dc, struct DownloadRequest dr)
	Schedule the download of the specified block in the tree. More...

void	GNUNET_FS_free_download_request_ (struct DownloadRequest *dr)
	(recursively) free download request structure More...

static int	check_put (void cls, const struct ClientPutMessage cm)
	Type of a function to call when we check the PUT message from the service. More...

static void	handle_put (void cls, const struct ClientPutMessage cm)
	Type of a function to call when we receive a message from the service. More...

static void	download_mq_error_handler (void *cls, enum GNUNET_MQ_Error error)
	Generic error handler, called with the appropriate error code and the same closure specified at the creation of the message queue. More...

static void	do_reconnect (void *cls)
	Reconnect to the FS service and transmit our queries NOW. More...

static void	activate_fs_download (void *cls)
	We're allowed to ask the FS service for our blocks. More...

static void	deactivate_fs_download (void *cls)
	We must stop to ask the FS service for our blocks. More...

static struct DownloadRequest *	create_download_request (struct DownloadRequest *parent, unsigned int chk_idx, unsigned int depth, uint64_t dr_offset, uint64_t file_start_offset, uint64_t desired_length)
	(recursively) Create a download request structure. More...

static void	reconstruct_cont (void *cls)
	Continuation after a possible attempt to reconstruct the current IBlock from the existing file. More...

static void	get_next_block (void *cls)
	Task requesting the next block from the tree encoder. More...

static void	reconstruct_cb (void cls, const struct ContentHashKey chk, uint64_t offset, unsigned int depth, enum GNUNET_BLOCK_Type type, const void *block, uint16_t block_size)
	Function called asking for the current (encoded) block to be processed. More...

static size_t	fh_reader (void cls, uint64_t offset, size_t max, void buf, char **emsg)
	Function called by the tree encoder to obtain a block of plaintext data (for the lowest level of the tree). More...

void	GNUNET_FS_download_start_task_ (void *cls)
	Task that creates the initial (top-level) download request for the file. More...

void	GNUNET_FS_download_signal_suspend_ (void *cls)
	Create SUSPEND event for the given download operation and then clean up our state (without stop signal). More...

struct GNUNET_FS_DownloadContext *	create_download_context (struct GNUNET_FS_Handle h, const struct GNUNET_FS_Uri uri, const struct GNUNET_FS_MetaData meta, const char filename, const char tempname, uint64_t offset, uint64_t length, uint32_t anonymity, enum GNUNET_FS_DownloadOptions options, void cctx)
	Helper function to setup the download context. More...

struct GNUNET_FS_DownloadContext *	GNUNET_FS_download_start (struct GNUNET_FS_Handle h, const struct GNUNET_FS_Uri uri, const struct GNUNET_FS_MetaData meta, const char filename, const char tempname, uint64_t offset, uint64_t length, uint32_t anonymity, enum GNUNET_FS_DownloadOptions options, void cctx, struct GNUNET_FS_DownloadContext *parent)
	Download parts of a file. More...

struct GNUNET_FS_DownloadContext *	GNUNET_FS_download_start_from_search (struct GNUNET_FS_Handle h, struct GNUNET_FS_SearchResult sr, const char filename, const char tempname, uint64_t offset, uint64_t length, uint32_t anonymity, enum GNUNET_FS_DownloadOptions options, void *cctx)
	Download parts of a file based on a search result. More...

void	GNUNET_FS_download_start_downloading_ (struct GNUNET_FS_DownloadContext *dc)
	Start the downloading process (by entering the queue). More...

void	GNUNET_FS_download_suspend (struct GNUNET_FS_DownloadContext *dc)
	Suspend a download. More...

void	GNUNET_FS_download_resume (struct GNUNET_FS_DownloadContext *dc)
	Resume a suspended download. More...

void	GNUNET_FS_download_stop (struct GNUNET_FS_DownloadContext *dc, int do_delete)
	Stop a download (aborts if download is incomplete). More...

Detailed Description

download methods

Author: Christian Grothoff

Definition in file fs_download.c.

Macro Definition Documentation

◆ GNUNET_FS_URI_CHK_PREFIX

#define GNUNET_FS_URI_CHK_PREFIX GNUNET_FS_URI_PREFIX GNUNET_FS_URI_CHK_INFIX

Definition at line 837 of file fs_download.c.

Function Documentation

◆ is_recursive_download()

static int is_recursive_download ( struct GNUNET_FS_DownloadContext * dc )

static

Determine if the given download (options and meta data) should cause use to try to do a recursive download.

Definition at line 38 of file fs_download.c.

{
  return (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE)) &&
         ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (dc->meta)) ||
          ((NULL == dc->meta) &&
           ((NULL == dc->filename) ||
            ((strlen (dc->filename) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
             (NULL != strstr (dc->filename + strlen (dc->filename)
                              - strlen (GNUNET_FS_DIRECTORY_EXT),
                              GNUNET_FS_DIRECTORY_EXT))))));
}

References dc, GNUNET_FS_DownloadContext::filename, GNUNET_FS_DIRECTORY_EXT, GNUNET_FS_DOWNLOAD_OPTION_RECURSIVE, GNUNET_FS_meta_data_test_for_directory(), GNUNET_YES, GNUNET_FS_DownloadContext::meta, and GNUNET_FS_DownloadContext::options.

Referenced by check_completed(), create_download_context(), and process_result_with_request().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ compute_disk_offset()

static uint64_t compute_disk_offset	(	uint64_t	fsize,
		uint64_t	off,
		unsigned int	depth
	)

static

We're storing the IBLOCKS after the DBLOCKS on disk (so that we only have to truncate the file once we're done).

Given the offset of a block (with respect to the DBLOCKS) and its depth, return the offset where we would store this block in the file.

Parameters

fsize	overall file size
off	offset of the block in the file
depth	depth of the block in the tree, 0 for DBLOCK

Returns: off for DBLOCKS (depth == treedepth), otherwise an offset past the end of the file that does not overlap with the range for any other block

Definition at line 68 of file fs_download.c.

{
  unsigned int i;
  uint64_t lsize; /* what is the size of all IBlocks for depth "i"? */
  uint64_t loff; /* where do IBlocks for depth "i" start? */
  unsigned int ioff; /* which IBlock corresponds to "off" at depth "i"? */
 
  if (0 == depth)
    return off;
  /* first IBlocks start at the end of file, rounded up
   * to full DBLOCK_SIZE */
  loff = ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * DBLOCK_SIZE;
  lsize =
    ((fsize + DBLOCK_SIZE - 1) / DBLOCK_SIZE) * sizeof(struct ContentHashKey);
  GNUNET_assert (0 == (off % DBLOCK_SIZE));
  ioff = (off / DBLOCK_SIZE);
  for (i = 1; i < depth; i++)
  {
    loff += lsize;
    lsize = (lsize + CHK_PER_INODE - 1) / CHK_PER_INODE;
    GNUNET_assert (lsize > 0);
    GNUNET_assert (0 == (ioff % CHK_PER_INODE));
    ioff /= CHK_PER_INODE;
  }
  return loff + ioff * sizeof(struct ContentHashKey);
}

References CHK_PER_INODE, DBLOCK_SIZE, and GNUNET_assert.

Referenced by process_result_with_request(), and try_top_down_reconstruction().

Here is the caller graph for this function:

◆ GNUNET_FS_download_make_status_()

void GNUNET_FS_download_make_status_	(	struct GNUNET_FS_ProgressInfo *	pi,
		struct GNUNET_FS_DownloadContext *	dc
	)

Fill in all of the generic fields for a download event and call the callback.

Parameters

pi	structure to fill in
dc	overall download context

Definition at line 104 of file fs_download.c.

{
  pi->value.download.dc = dc;
  pi->value.download.cctx = dc->client_info;
  pi->value.download.pctx =
    (NULL == dc->parent) ? NULL : dc->parent->client_info;
  pi->value.download.sctx =
    (NULL == dc->search) ? NULL : dc->search->client_info;
  pi->value.download.uri = dc->uri;
  pi->value.download.filename = dc->filename;
  pi->value.download.size = dc->length;
  /* FIXME: Fix duration calculation to account for pauses */
  pi->value.download.duration =
    GNUNET_TIME_absolute_get_duration (dc->start_time);
  pi->value.download.completed = dc->completed;
  pi->value.download.anonymity = dc->anonymity;
  pi->value.download.eta =
    GNUNET_TIME_calculate_eta (dc->start_time, dc->completed, dc->length);
  pi->value.download.is_active = (NULL == dc->mq) ? GNUNET_NO : GNUNET_YES;
  pi->fsh = dc->h;
  if (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
    dc->client_info = dc->h->upcb (dc->h->upcb_cls, pi);
  else
    dc->client_info = GNUNET_FS_search_probe_progress_ (NULL, pi);
}

Referenced by activate_fs_download(), check_completed(), deactivate_fs_download(), GNUNET_FS_download_resume(), GNUNET_FS_download_signal_suspend_(), GNUNET_FS_download_start_task_(), GNUNET_FS_download_stop(), process_result_with_request(), reconstruct_cb(), search_result_stop(), signal_download_resume(), and try_match_block().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ process_result_with_request()

static int process_result_with_request	(	void *	cls,
		const struct GNUNET_HashCode *	key,
		void *	value
	)

static

Iterator over entries in the pending requests in the 'active' map for the reply that we just got.

Parameters

cls	closure (our `struct ProcessResultClosure`)
key	query for the given value / request
value	value in the hash map (a `struct DownloadRequest`)

Returns: GNUNET_YES (we should continue to iterate); unless serious error

Definition at line 1009 of file fs_download.c.

{
  struct ProcessResultClosure *prc = cls;
  struct DownloadRequest *dr = value;
  struct GNUNET_FS_DownloadContext *dc = prc->dc;
  struct DownloadRequest *drc;
  struct GNUNET_DISK_FileHandle *fh = NULL;
  struct GNUNET_CRYPTO_SymmetricSessionKey skey;
  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
  char pt[prc->size];
  struct GNUNET_FS_ProgressInfo pi;
  uint64_t off;
  size_t bs;
  size_t app;
  int i;
  struct ContentHashKey *chkarr;
 
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Received %u byte block `%s' matching pending request at depth %u and offset %llu/%llu\n",
    (unsigned int) prc->size,
    GNUNET_h2s (key),
    dr->depth,
    (unsigned long long) dr->offset,
    (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length));
  bs = GNUNET_FS_tree_calculate_block_size (GNUNET_ntohll (
                                              dc->uri->data.chk.file_length),
                                            dr->offset,
                                            dr->depth);
  if (prc->size != bs)
  {
    GNUNET_asprintf (
      &dc->emsg,
      "Internal error or bogus download URI (expected %llu bytes at depth %u and offset %llu/%llu, got %llu bytes)",
      (unsigned long long) bs,
      dr->depth,
      (unsigned long long) dr->offset,
      (unsigned long long) GNUNET_ntohll (dc->uri->data.chk.file_length),
      (unsigned long long) prc->size);
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "%s\n", dc->emsg);
    while (NULL != dr->parent)
    {
      dr->state = BRS_ERROR;
      dr = dr->parent;
    }
    dr->state = BRS_ERROR;
    goto signal_error;
  }
 
  (void) GNUNET_CONTAINER_multihashmap_remove (dc->active, &prc->query, dr);
  GNUNET_CRYPTO_hash_to_aes_key (&dr->chk.key, &skey, &iv);
  if (-1 ==
      GNUNET_CRYPTO_symmetric_decrypt (prc->data, prc->size, &skey, &iv, pt))
  {
    GNUNET_break (0);
    dc->emsg = GNUNET_strdup (_ ("internal error decrypting content"));
    goto signal_error;
  }
  off = compute_disk_offset (GNUNET_ntohll (dc->uri->data.chk.file_length),
                             dr->offset,
                             dr->depth);
  /* save to disk */
  if ((GNUNET_YES == prc->do_store) &&
      ((NULL != dc->filename) || (is_recursive_download (dc))) &&
      ((dr->depth == dc->treedepth) ||
       (0 == (dc->options & GNUNET_FS_DOWNLOAD_NO_TEMPORARIES))))
  {
    fh = GNUNET_DISK_file_open (NULL != dc->filename ? dc->filename
                                : dc->temp_filename,
                                GNUNET_DISK_OPEN_READWRITE
                                | GNUNET_DISK_OPEN_CREATE,
                                GNUNET_DISK_PERM_USER_READ
                                | GNUNET_DISK_PERM_USER_WRITE
                                | GNUNET_DISK_PERM_GROUP_READ
                                | GNUNET_DISK_PERM_OTHER_READ);
    if (NULL == fh)
    {
      GNUNET_asprintf (&dc->emsg,
                       _ ("Download failed: could not open file `%s': %s"),
                       dc->filename,
                       strerror (errno));
      goto signal_error;
    }
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Saving decrypted block to disk at offset %llu\n",
                (unsigned long long) off);
    if ((off != GNUNET_DISK_file_seek (fh, off, GNUNET_DISK_SEEK_SET)))
    {
      GNUNET_asprintf (&dc->emsg,
                       _ ("Failed to seek to offset %llu in file `%s': %s"),
                       (unsigned long long) off,
                       dc->filename,
                       strerror (errno));
      goto signal_error;
    }
    if (prc->size != GNUNET_DISK_file_write (fh, pt, prc->size))
    {
      GNUNET_asprintf (
        &dc->emsg,
        _ ("Failed to write block of %u bytes at offset %llu in file `%s': %s"),
        (unsigned int) prc->size,
        (unsigned long long) off,
        dc->filename,
        strerror (errno));
      goto signal_error;
    }
    GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (fh));
    fh = NULL;
  }
 
  if (0 == dr->depth)
  {
    /* DBLOCK, update progress and try recursion if applicable */
    app = prc->size;
    if (dr->offset < dc->offset)
    {
      /* starting offset begins in the middle of pt,
       * do not count first bytes as progress */
      GNUNET_assert (app > (dc->offset - dr->offset));
      app -= (dc->offset - dr->offset);
    }
    if (dr->offset + prc->size > dc->offset + dc->length)
    {
      /* end of block is after relevant range,
      * do not count last bytes as progress */
      GNUNET_assert (app >
                     (dr->offset + prc->size) - (dc->offset + dc->length));
      app -= (dr->offset + prc->size) - (dc->offset + dc->length);
    }
    dc->completed += app;
 
    /* do recursive download if option is set and either meta data
     * says it is a directory or if no meta data is given AND filename
     * ends in '.gnd' (top-level case) */
    if (is_recursive_download (dc))
      GNUNET_FS_directory_list_contents (prc->size,
                                         pt,
                                         off,
                                         &trigger_recursive_download,
                                         dc);
  }
  GNUNET_assert (dc->completed <= dc->length);
  dr->state = BRS_DOWNLOAD_DOWN;
  pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
  pi.value.download.specifics.progress.data = pt;
  pi.value.download.specifics.progress.offset = dr->offset;
  pi.value.download.specifics.progress.data_len = prc->size;
  pi.value.download.specifics.progress.depth = dr->depth;
  pi.value.download.specifics.progress.respect_offered = prc->respect_offered;
  pi.value.download.specifics.progress.num_transmissions =
    prc->num_transmissions;
  if (prc->last_transmission.abs_value_us !=
      GNUNET_TIME_UNIT_FOREVER_ABS.abs_value_us)
    pi.value.download.specifics.progress.block_download_duration =
      GNUNET_TIME_absolute_get_duration (prc->last_transmission);
  else
    pi.value.download.specifics.progress.block_download_duration =
      GNUNET_TIME_UNIT_ZERO; /* found locally */
  GNUNET_FS_download_make_status_ (&pi, dc);
  if (0 == dr->depth)
    propagate_up (dr);
 
  if (dc->completed == dc->length)
  {
    /* download completed, signal */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Download completed, truncating file to desired length %llu\n",
                (unsigned long long) GNUNET_ntohll (
                  dc->uri->data.chk.file_length));
    /* truncate file to size (since we store IBlocks at the end) */
    if (NULL != dc->filename)
    {
      if (0 != truncate (dc->filename,
                         GNUNET_ntohll (dc->uri->data.chk.file_length)))
        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                                  "truncate",
                                  dc->filename);
    }
    GNUNET_assert (0 == dr->depth);
    check_completed (dc);
  }
  if (0 == dr->depth)
  {
    /* bottom of the tree, no child downloads possible, just sync */
    GNUNET_FS_download_sync_ (dc);
    return GNUNET_YES;
  }
 
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Triggering downloads of children (this block was at depth %u and offset %llu)\n",
    dr->depth,
    (unsigned long long) dr->offset);
  GNUNET_assert (0 == (prc->size % sizeof(struct ContentHashKey)));
  chkarr = (struct ContentHashKey *) pt;
  for (i = dr->num_children - 1; i >= 0; i--)
  {
    drc = dr->children[i];
    switch (drc->state)
    {
    case BRS_INIT:
      if ((drc->chk_idx + 1) * sizeof(struct ContentHashKey) > prc->size)
      {
        /* 'chkarr' does not have enough space for this chk_idx;
           internal error! */
        GNUNET_break (0);
        GNUNET_assert (0);
        dc->emsg = GNUNET_strdup (_ ("internal error decoding tree"));
        goto signal_error;
      }
      drc->chk = chkarr[drc->chk_idx];
      drc->state = BRS_CHK_SET;
      if (GNUNET_YES == dc->issue_requests)
        schedule_block_download (dc, drc);
      break;
 
    case BRS_RECONSTRUCT_DOWN:
      GNUNET_assert (0);
      break;
 
    case BRS_RECONSTRUCT_META_UP:
      GNUNET_assert (0);
      break;
 
    case BRS_RECONSTRUCT_UP:
      GNUNET_assert (0);
      break;
 
    case BRS_CHK_SET:
      GNUNET_assert (0);
      break;
 
    case BRS_DOWNLOAD_DOWN:
      GNUNET_assert (0);
      break;
 
    case BRS_DOWNLOAD_UP:
      GNUNET_assert (0);
      break;
 
    case BRS_ERROR:
      GNUNET_assert (0);
      break;
 
    default:
      GNUNET_assert (0);
      break;
    }
  }
  GNUNET_FS_download_sync_ (dc);
  return GNUNET_YES;
 
signal_error:
  if (NULL != fh)
    GNUNET_DISK_file_close (fh);
  pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
  pi.value.download.specifics.error.message = dc->emsg;
  GNUNET_FS_download_make_status_ (&pi, dc);
  GNUNET_MQ_destroy (dc->mq);
  dc->mq = NULL;
  GNUNET_FS_free_download_request_ (dc->top_request);
  dc->top_request = NULL;
  if (NULL != dc->job_queue)
  {
    GNUNET_FS_dequeue_ (dc->job_queue);
    dc->job_queue = NULL;
  }
  GNUNET_FS_download_sync_ (dc);
  return GNUNET_NO;
}

Referenced by encrypt_existing_match(), and handle_put().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ encrypt_existing_match()

static int encrypt_existing_match	(	struct GNUNET_FS_DownloadContext *	dc,
		const struct ContentHashKey *	chk,
		struct DownloadRequest *	dr,
		const char *	block,
		size_t	len,
		int	do_store
	)

static

We've found a matching block without downloading it.

Encrypt it and pass it to our "receive" function as if we had received it from the network.

Parameters

dc	download in question
chk	request this relates to
dr	request details
block	plaintext data matching request
len	number of bytes in block
do_store	should we still store the block on disk?

Returns: GNUNET_OK on success

Definition at line 213 of file fs_download.c.

{
  struct ProcessResultClosure prc;
  char enc[len];
  struct GNUNET_CRYPTO_SymmetricSessionKey sk;
  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
  struct GNUNET_HashCode query;
 
  GNUNET_CRYPTO_hash_to_aes_key (&chk->key, &sk, &iv);
  if (-1 == GNUNET_CRYPTO_symmetric_encrypt (block, len, &sk, &iv, enc))
  {
    GNUNET_break (0);
    return GNUNET_SYSERR;
  }
  GNUNET_CRYPTO_hash (enc, len, &query);
  if (0 != memcmp (&query, &chk->query, sizeof(struct GNUNET_HashCode)))
  {
    GNUNET_break_op (0);
    return GNUNET_SYSERR;
  }
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Matching %u byte block for `%s' at offset %llu already present, no need for download!\n",
    (unsigned int) len,
    dc->filename,
    (unsigned long long) dr->offset);
  /* already got it! */
  prc.dc = dc;
  prc.data = enc;
  prc.size = len;
  prc.type = (0 == dr->depth) ? GNUNET_BLOCK_TYPE_FS_DBLOCK
             : GNUNET_BLOCK_TYPE_FS_IBLOCK;
  prc.query = chk->query;
  prc.do_store = do_store;
  prc.last_transmission = GNUNET_TIME_UNIT_FOREVER_ABS;
  process_result_with_request (&prc, &chk->key, dr);
  return GNUNET_OK;
}

References ProcessResultClosure::data, dc, ProcessResultClosure::dc, DownloadRequest::depth, ProcessResultClosure::do_store, enc, GNUNET_FS_DownloadContext::filename, GNUNET_BLOCK_TYPE_FS_DBLOCK, GNUNET_BLOCK_TYPE_FS_IBLOCK, GNUNET_break, GNUNET_break_op, GNUNET_CRYPTO_hash(), GNUNET_CRYPTO_hash_to_aes_key(), GNUNET_CRYPTO_symmetric_encrypt(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_OK, GNUNET_SYSERR, GNUNET_TIME_UNIT_FOREVER_ABS, ContentHashKey::key, ProcessResultClosure::last_transmission, DownloadRequest::offset, process_result_with_request(), ContentHashKey::query, ProcessResultClosure::query, ProcessResultClosure::size, and ProcessResultClosure::type.

Referenced by try_top_down_reconstruction().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ try_reconnect()

static void try_reconnect ( struct GNUNET_FS_DownloadContext * dc )

static

We've lost our connection with the FS service.

Re-establish it and re-transmit all of our pending requests.

Parameters

dc	download context that is having trouble

Definition at line 1397 of file fs_download.c.

{
  if (NULL != dc->mq)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Moving all requests back to pending list\n");
    GNUNET_MQ_destroy (dc->mq);
    dc->mq = NULL;
  }
  if (0 == dc->reconnect_backoff.rel_value_us)
    dc->reconnect_backoff = GNUNET_TIME_UNIT_MILLISECONDS;
  else
    dc->reconnect_backoff = GNUNET_TIME_STD_BACKOFF (dc->reconnect_backoff);
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Will try to reconnect in %s\n",
              GNUNET_STRINGS_relative_time_to_string (dc->reconnect_backoff,
                                                      GNUNET_YES));
  GNUNET_break (NULL != dc->job_queue);
  dc->task =
    GNUNET_SCHEDULER_add_delayed (dc->reconnect_backoff, &do_reconnect, dc);
}

References dc, do_reconnect(), GNUNET_break, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_MQ_destroy(), GNUNET_SCHEDULER_add_delayed(), GNUNET_STRINGS_relative_time_to_string(), GNUNET_TIME_STD_BACKOFF, GNUNET_TIME_UNIT_MILLISECONDS, GNUNET_YES, GNUNET_FS_DownloadContext::job_queue, GNUNET_FS_DownloadContext::mq, GNUNET_FS_DownloadContext::reconnect_backoff, GNUNET_TIME_Relative::rel_value_us, and GNUNET_FS_DownloadContext::task.

Referenced by do_reconnect(), and download_mq_error_handler().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ trigger_recursive_download()

static void trigger_recursive_download	(	void *	cls,
		const char *	filename,
		const struct GNUNET_FS_Uri *	uri,
		const struct GNUNET_FS_MetaData *	meta,
		size_t	length,
		const void *	data
	)

static

We found an entry in a directory.

Check if the respective child already exists and if not create the respective child download.

Parameters

cls	the parent download
filename	name of the file in the directory
uri	URI of the file (CHK or LOC)
meta	meta data of the file
length	number of bytes in data
data	contents of the file (or NULL if they were not inlined)

Definition at line 851 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_FS_DownloadContext *cpos;
  char *temp_name;
  char *fn;
  char *us;
  char *ext;
  char *dn;
  char *pos;
  char *full_name;
  char *sfn;
 
  if (NULL == uri)
    return; /* entry for the directory itself */
  cpos = dc->child_head;
  while (NULL != cpos)
  {
    if ((GNUNET_FS_uri_test_equal (uri, cpos->uri)) ||
        ((NULL != filename) && (0 == strcmp (cpos->filename, filename))))
      break;
    cpos = cpos->next;
  }
  if (NULL != cpos)
    return; /* already exists */
  fn = NULL;
  if (NULL == filename)
  {
    fn = GNUNET_FS_meta_data_suggest_filename (meta);
    if (NULL == fn)
    {
      us = GNUNET_FS_uri_to_string (uri);
      fn = GNUNET_strdup (&us[strlen (GNUNET_FS_URI_CHK_PREFIX)]);
      GNUNET_free (us);
    }
    else if ('.' == fn[0])
    {
      ext = fn;
      us = GNUNET_FS_uri_to_string (uri);
      GNUNET_asprintf (&fn,
                       "%s%s",
                       &us[strlen (GNUNET_FS_URI_CHK_PREFIX)],
                       ext);
      GNUNET_free (ext);
      GNUNET_free (us);
    }
    /* change '\' to '/' (this should have happened
     * during insertion, but malicious peers may
     * not have done this) */
    while (NULL != (pos = strstr (fn, "\\")))
      *pos = '/';
    /* remove '../' everywhere (again, well-behaved
     * peers don't do this, but don't trust that
     * we did not get something nasty) */
    while (NULL != (pos = strstr (fn, "../")))
    {
      pos[0] = '_';
      pos[1] = '_';
      pos[2] = '_';
    }
    filename = fn;
  }
  if (NULL == dc->filename)
  {
    full_name = NULL;
  }
  else
  {
    dn = GNUNET_strdup (dc->filename);
    GNUNET_break (
      (strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
      (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT),
                       GNUNET_FS_DIRECTORY_EXT)));
    sfn = GNUNET_strdup (filename);
    while ((strlen (sfn) > 0) && ('/' == filename[strlen (sfn) - 1]))
      sfn[strlen (sfn) - 1] = '\0';
    if ((strlen (dn) >= strlen (GNUNET_FS_DIRECTORY_EXT)) &&
        (NULL != strstr (dn + strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT),
                         GNUNET_FS_DIRECTORY_EXT)))
      dn[strlen (dn) - strlen (GNUNET_FS_DIRECTORY_EXT)] = '\0';
    if ((GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (meta)) &&
        ((strlen (filename) < strlen (GNUNET_FS_DIRECTORY_EXT)) ||
         (NULL == strstr (filename + strlen (filename)
                          - strlen (GNUNET_FS_DIRECTORY_EXT),
                          GNUNET_FS_DIRECTORY_EXT))))
    {
      GNUNET_asprintf (&full_name,
                       "%s%s%s%s",
                       dn,
                       DIR_SEPARATOR_STR,
                       sfn,
                       GNUNET_FS_DIRECTORY_EXT);
    }
    else
    {
      GNUNET_asprintf (&full_name, "%s%s%s", dn, DIR_SEPARATOR_STR, sfn);
    }
    GNUNET_free (sfn);
    GNUNET_free (dn);
  }
  if ((NULL != full_name) &&
      (GNUNET_OK != GNUNET_DISK_directory_create_for_file (full_name)))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ (
                  "Failed to create directory for recursive download of `%s'\n"),
                full_name);
    GNUNET_free (full_name);
    GNUNET_free (fn);
    return;
  }
 
  temp_name = NULL;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Triggering recursive download of size %llu with %u bytes MD\n",
              (unsigned long long) GNUNET_FS_uri_chk_get_file_size (uri),
              (unsigned int) GNUNET_FS_meta_data_get_serialized_size (
                meta));
  GNUNET_FS_download_start (dc->h,
                            uri,
                            meta,
                            full_name,
                            temp_name,
                            0,
                            GNUNET_FS_uri_chk_get_file_size (uri),
                            dc->anonymity,
                            dc->options,
                            NULL,
                            dc);
  GNUNET_free (full_name);
  GNUNET_free (temp_name);
  GNUNET_free (fn);
}

References _, GNUNET_FS_DownloadContext::anonymity, GNUNET_FS_DownloadContext::child_head, dc, DIR_SEPARATOR_STR, filename, GNUNET_FS_DownloadContext::filename, GNUNET_asprintf(), GNUNET_break, GNUNET_DISK_directory_create_for_file(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_ERROR, GNUNET_free, GNUNET_FS_DIRECTORY_EXT, GNUNET_FS_download_start(), GNUNET_FS_meta_data_get_serialized_size(), GNUNET_FS_meta_data_suggest_filename(), GNUNET_FS_meta_data_test_for_directory(), GNUNET_FS_uri_chk_get_file_size(), GNUNET_FS_URI_CHK_PREFIX, GNUNET_FS_uri_test_equal(), GNUNET_FS_uri_to_string(), GNUNET_log, GNUNET_OK, GNUNET_strdup, GNUNET_YES, GNUNET_FS_DownloadContext::h, meta, GNUNET_FS_DownloadContext::next, GNUNET_FS_DownloadContext::options, uri, and GNUNET_FS_DownloadContext::uri.

Referenced by full_recursive_download(), and process_result_with_request().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ full_recursive_download()

static void full_recursive_download ( struct GNUNET_FS_DownloadContext * dc )

static

We're done downloading a directory.

Open the file and trigger all of the (remaining) child downloads.

Parameters

dc	context of download that just completed

Definition at line 296 of file fs_download.c.

{
  size_t size;
  uint64_t size64;
  void *data;
  struct GNUNET_DISK_FileHandle *h;
  struct GNUNET_DISK_MapHandle *m;
 
  size64 = GNUNET_FS_uri_chk_get_file_size (dc->uri);
  size = (size_t) size64;
  if (size64 != (uint64_t) size)
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_ERROR,
      _ (
        "Recursive downloads of directories larger than 4 GB are not supported on 32-bit systems\n"));
    return;
  }
  if (NULL != dc->filename)
  {
    h = GNUNET_DISK_file_open (dc->filename,
                               GNUNET_DISK_OPEN_READ,
                               GNUNET_DISK_PERM_NONE);
  }
  else
  {
    GNUNET_assert (NULL != dc->temp_filename);
    h = GNUNET_DISK_file_open (dc->temp_filename,
                               GNUNET_DISK_OPEN_READ,
                               GNUNET_DISK_PERM_NONE);
  }
  if (NULL == h)
    return; /* oops */
  data = GNUNET_DISK_file_map (h, &m, GNUNET_DISK_MAP_TYPE_READ, size);
  if (NULL == data)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                _ ("Directory too large for system address space\n"));
  }
  else
  {
    if (GNUNET_OK !=
        GNUNET_FS_directory_list_contents (size,
                                           data,
                                           0,
                                           &trigger_recursive_download,
                                           dc))
    {
      GNUNET_log (
        GNUNET_ERROR_TYPE_WARNING,
        _ (
          "Failed to access full directory contents of `%s' for recursive download\n"),
        dc->filename);
    }
    GNUNET_DISK_file_unmap (m);
  }
  GNUNET_DISK_file_close (h);
  if (NULL == dc->filename)
  {
    if (0 != unlink (dc->temp_filename))
      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                                "unlink",
                                dc->temp_filename);
    GNUNET_free (dc->temp_filename);
    dc->temp_filename = NULL;
  }
}

References _, data, dc, GNUNET_FS_DownloadContext::filename, GNUNET_assert, GNUNET_DISK_file_close(), GNUNET_DISK_file_map(), GNUNET_DISK_file_open(), GNUNET_DISK_file_unmap(), GNUNET_DISK_MAP_TYPE_READ, GNUNET_DISK_OPEN_READ, GNUNET_DISK_PERM_NONE, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_WARNING, GNUNET_free, GNUNET_FS_directory_list_contents(), GNUNET_FS_uri_chk_get_file_size(), GNUNET_log, GNUNET_log_strerror_file, GNUNET_OK, h, m, size, GNUNET_FS_DownloadContext::temp_filename, trigger_recursive_download(), and GNUNET_FS_DownloadContext::uri.

Referenced by check_completed().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ check_completed()

static void check_completed ( struct GNUNET_FS_DownloadContext * dc )

static

Check if all child-downloads have completed (or trigger them if necessary) and once we're completely done, signal completion (and possibly recurse to parent).

This function MUST be called when the download of a file itself is done or when the download of a file is done and then later a direct child download has completed (and hence this download may complete itself).

Parameters

dc	download to check for completion of children

Definition at line 376 of file fs_download.c.

{
  struct GNUNET_FS_ProgressInfo pi;
  struct GNUNET_FS_DownloadContext *pos;
 
  /* first, check if we need to download children */
  if (is_recursive_download (dc))
    full_recursive_download (dc);
  /* then, check if children are done already */
  for (pos = dc->child_head; NULL != pos; pos = pos->next)
  {
    if ((NULL == pos->emsg) && (pos->completed < pos->length))
      return;   /* not done yet */
    if ((NULL != pos->child_head) && (pos->has_finished != GNUNET_YES))
      return;   /* not transitively done yet */
  }
  /* All of our children are done, so mark this download done */
  dc->has_finished = GNUNET_YES;
  if (NULL != dc->job_queue)
  {
    GNUNET_FS_dequeue_ (dc->job_queue);
    dc->job_queue = NULL;
  }
  if (NULL != dc->task)
  {
    GNUNET_SCHEDULER_cancel (dc->task);
    dc->task = NULL;
  }
  if (NULL != dc->rfh)
  {
    GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
    dc->rfh = NULL;
  }
  GNUNET_FS_download_sync_ (dc);
 
  /* signal completion */
  pi.status = GNUNET_FS_STATUS_DOWNLOAD_COMPLETED;
  GNUNET_FS_download_make_status_ (&pi, dc);
 
  /* let parent know */
  if (NULL != dc->parent)
    check_completed (dc->parent);
}

Referenced by check_completed(), GNUNET_FS_download_start_task_(), process_result_with_request(), reconstruct_cb(), and try_match_block().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ try_match_block()

static void try_match_block	(	struct GNUNET_FS_DownloadContext *	dc,
		struct DownloadRequest *	dr,
		const char *	data,
		size_t	data_len
	)

static

We got a block of plaintext data (from the meta data).

Try it for upward reconstruction of the data. On success, the top-level block will move to state BRS_DOWNLOAD_UP.

Parameters

dc	context for the download
dr	download request to match against
data	plaintext data, starting from the beginning of the file
data_len	number of bytes in data

Definition at line 432 of file fs_download.c.

{
  struct GNUNET_FS_ProgressInfo pi;
  unsigned int i;
  char enc[DBLOCK_SIZE];
  struct ContentHashKey chks[CHK_PER_INODE];
  struct ContentHashKey in_chk;
  struct GNUNET_CRYPTO_SymmetricSessionKey sk;
  struct GNUNET_CRYPTO_SymmetricInitializationVector iv;
  size_t dlen;
  struct DownloadRequest *drc;
  struct GNUNET_DISK_FileHandle *fh;
  int complete;
  const char *fn;
  const char *odata;
  size_t odata_len;
 
  odata = data;
  odata_len = data_len;
  if (BRS_DOWNLOAD_UP == dr->state)
    return;
  if (dr->depth > 0)
  {
    if ((dc->offset > 0) ||
        (dc->length < GNUNET_ntohll (dc->uri->data.chk.file_length)))
    {
      /* NOTE: this test is not tight, but should suffice; the issue
         here is that 'dr->num_children' may inherently only specify a
         smaller range than what is in the original file;
         thus, reconstruction of (some) inner blocks will fail.
         FIXME: we might eventually want to write a tighter test to
         maximize the circumstances under which we do succeed with
         IBlock reconstruction. (need good tests though). */return;
    }
    complete = GNUNET_YES;
    for (i = 0; i < dr->num_children; i++)
    {
      drc = dr->children[i];
      try_match_block (dc, drc, data, data_len);
      if (drc->state != BRS_RECONSTRUCT_META_UP)
        complete = GNUNET_NO;
      else
        chks[i] = drc->chk;
    }
    if (GNUNET_YES != complete)
      return;
    data = (const char *) chks;
    dlen = dr->num_children * sizeof(struct ContentHashKey);
  }
  else
  {
    if (dr->offset > data_len)
      return;   /* oops */
    dlen = GNUNET_MIN (data_len - dr->offset, DBLOCK_SIZE);
  }
  GNUNET_CRYPTO_hash (&data[dr->offset], dlen, &in_chk.key);
  GNUNET_CRYPTO_hash_to_aes_key (&in_chk.key, &sk, &iv);
  if (-1 ==
      GNUNET_CRYPTO_symmetric_encrypt (&data[dr->offset], dlen, &sk, &iv, enc))
  {
    GNUNET_break (0);
    return;
  }
  GNUNET_CRYPTO_hash (enc, dlen, &in_chk.query);
  switch (dr->state)
  {
  case BRS_INIT:
    dr->chk = in_chk;
    dr->state = BRS_RECONSTRUCT_META_UP;
    break;
 
  case BRS_CHK_SET:
    if (0 != memcmp (&in_chk, &dr->chk, sizeof(struct ContentHashKey)))
    {
      /* other peer provided bogus meta data */
      GNUNET_break_op (0);
      break;
    }
    /* write block to disk */
    fn = (NULL != dc->filename) ? dc->filename : dc->temp_filename;
    if (NULL != fn)
    {
      fh = GNUNET_DISK_file_open (fn,
                                  GNUNET_DISK_OPEN_READWRITE
                                  | GNUNET_DISK_OPEN_CREATE
                                  | GNUNET_DISK_OPEN_TRUNCATE,
                                  GNUNET_DISK_PERM_USER_READ
                                  | GNUNET_DISK_PERM_USER_WRITE
                                  | GNUNET_DISK_PERM_GROUP_READ
                                  | GNUNET_DISK_PERM_OTHER_READ);
      if (NULL == fh)
      {
        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "open", fn);
        GNUNET_asprintf (&dc->emsg,
                         _ ("Failed to open file `%s' for writing"),
                         fn);
        GNUNET_DISK_file_close (fh);
        dr->state = BRS_ERROR;
        pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
        pi.value.download.specifics.error.message = dc->emsg;
        GNUNET_FS_download_make_status_ (&pi, dc);
        return;
      }
      if (data_len != GNUNET_DISK_file_write (fh, odata, odata_len))
      {
        GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "write", fn);
        GNUNET_asprintf (&dc->emsg,
                         _ ("Failed to open file `%s' for writing"),
                         fn);
        GNUNET_DISK_file_close (fh);
        dr->state = BRS_ERROR;
        pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
        pi.value.download.specifics.error.message = dc->emsg;
        GNUNET_FS_download_make_status_ (&pi, dc);
        return;
      }
      GNUNET_DISK_file_close (fh);
    }
    /* signal success */
    dr->state = BRS_DOWNLOAD_UP;
    dc->completed = dc->length;
    GNUNET_FS_download_sync_ (dc);
    pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
    pi.value.download.specifics.progress.data = data;
    pi.value.download.specifics.progress.offset = 0;
    pi.value.download.specifics.progress.data_len = dlen;
    pi.value.download.specifics.progress.depth = 0;
    pi.value.download.specifics.progress.respect_offered = 0;
    pi.value.download.specifics.progress.block_download_duration =
      GNUNET_TIME_UNIT_ZERO;
    GNUNET_FS_download_make_status_ (&pi, dc);
    if ((NULL != dc->filename) &&
        (0 != truncate (dc->filename,
                        GNUNET_ntohll (dc->uri->data.chk.file_length))))
      GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                                "truncate",
                                dc->filename);
    check_completed (dc);
    break;
 
  default:
    /* how did we get here? */
    GNUNET_break (0);
    break;
  }
}

References _, BRS_CHK_SET, BRS_DOWNLOAD_UP, BRS_ERROR, BRS_INIT, BRS_RECONSTRUCT_META_UP, check_completed(), DownloadRequest::children, GNUNET_FS_Uri::chk, DownloadRequest::chk, CHK_PER_INODE, GNUNET_FS_DownloadContext::completed, data, GNUNET_FS_Uri::data, DBLOCK_SIZE, dc, DownloadRequest::depth, GNUNET_FS_ProgressInfo::download, GNUNET_FS_DownloadContext::emsg, enc, FileIdentifier::file_length, GNUNET_FS_DownloadContext::filename, GNUNET_asprintf(), GNUNET_break, GNUNET_break_op, GNUNET_CRYPTO_hash(), GNUNET_CRYPTO_hash_to_aes_key(), GNUNET_CRYPTO_symmetric_encrypt(), GNUNET_DISK_file_close(), GNUNET_DISK_file_open(), GNUNET_DISK_file_write(), GNUNET_DISK_OPEN_CREATE, GNUNET_DISK_OPEN_READWRITE, GNUNET_DISK_OPEN_TRUNCATE, GNUNET_DISK_PERM_GROUP_READ, GNUNET_DISK_PERM_OTHER_READ, GNUNET_DISK_PERM_USER_READ, GNUNET_DISK_PERM_USER_WRITE, GNUNET_ERROR_TYPE_ERROR, GNUNET_ERROR_TYPE_WARNING, GNUNET_FS_download_make_status_(), GNUNET_FS_download_sync_(), GNUNET_FS_STATUS_DOWNLOAD_ERROR, GNUNET_FS_STATUS_DOWNLOAD_PROGRESS, GNUNET_log_strerror_file, GNUNET_MIN, GNUNET_NO, GNUNET_ntohll(), GNUNET_TIME_UNIT_ZERO, GNUNET_YES, ContentHashKey::key, GNUNET_FS_DownloadContext::length, DownloadRequest::num_children, DownloadRequest::offset, GNUNET_FS_DownloadContext::offset, ContentHashKey::query, DownloadRequest::state, GNUNET_FS_ProgressInfo::status, GNUNET_FS_DownloadContext::temp_filename, try_match_block(), GNUNET_FS_DownloadContext::uri, and GNUNET_FS_ProgressInfo::value.

Referenced by match_full_data(), and try_match_block().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ match_full_data()

static int match_full_data	(	void *	cls,
		const char *	plugin_name,
		enum EXTRACTOR_MetaType	type,
		enum EXTRACTOR_MetaFormat	format,
		const char *	data_mime_type,
		const char *	data,
		size_t	data_len
	)

static

Type of a function that libextractor calls for each meta data item found.

If we find full data meta data, call 'try_match_block' on it.

Parameters

cls	our 'struct GNUNET_FS_DownloadContext*'
plugin_name	name of the plugin that produced this value; special values can be used (e.g. '<zlib>' for zlib being used in the main libextractor library and yielding meta data).
type	libextractor-type describing the meta data
format	basic format information about data
data_mime_type	mime-type of data (not of the original file); can be NULL (if mime-type is not known)
data	actual meta-data found
data_len	number of bytes in data

Returns: 0 to continue extracting, 1 to abort

Definition at line 602 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
 
  if (EXTRACTOR_METATYPE_GNUNET_FULL_DATA != type)
    return 0;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Found %u bytes of FD!\n",
              (unsigned int) data_len);
  if (GNUNET_FS_uri_chk_get_file_size (dc->uri) != data_len)
  {
    GNUNET_break_op (0);
    return 1;   /* bogus meta data */
  }
  try_match_block (dc, dc->top_request, data, data_len);
  return 1;
}

References data, dc, GNUNET_break_op, GNUNET_ERROR_TYPE_DEBUG, GNUNET_FS_uri_chk_get_file_size(), GNUNET_log, GNUNET_FS_DownloadContext::top_request, try_match_block(), type, and GNUNET_FS_DownloadContext::uri.

Referenced by GNUNET_FS_download_start_task_().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ propagate_up()

static void propagate_up ( struct DownloadRequest * dr )

static

Set the state of the given download request to BRS_DOWNLOAD_UP and propagate it up the tree.

Parameters

dr	download request that is done

Definition at line 634 of file fs_download.c.

{
  unsigned int i;
 
  do
  {
    dr->state = BRS_DOWNLOAD_UP;
    dr = dr->parent;
    if (NULL == dr)
      break;
    for (i = 0; i < dr->num_children; i++)
      if (dr->children[i]->state != BRS_DOWNLOAD_UP)
        break;
  }
  while (i == dr->num_children);
}

References BRS_DOWNLOAD_UP, DownloadRequest::children, DownloadRequest::num_children, DownloadRequest::parent, and DownloadRequest::state.

Referenced by process_result_with_request(), and try_top_down_reconstruction().

Here is the caller graph for this function:

◆ try_top_down_reconstruction()

static void try_top_down_reconstruction	(	struct GNUNET_FS_DownloadContext *	dc,
		struct DownloadRequest *	dr
	)

static

Try top-down reconstruction.

Before, the given request node must have the state BRS_CHK_SET. Afterwards, more nodes may have that state or advanced to BRS_DOWNLOAD_DOWN or even BRS_DOWNLOAD_UP. It is also possible to get BRS_ERROR on the top level.

Parameters

dc	overall download this block belongs to
dr	block to reconstruct

Definition at line 663 of file fs_download.c.

{
  uint64_t off;
  char block[DBLOCK_SIZE];
  struct GNUNET_HashCode key;
  uint64_t total;
  size_t len;
  unsigned int i;
  struct DownloadRequest *drc;
  uint64_t child_block_size;
  const struct ContentHashKey *chks;
  int up_done;
 
  GNUNET_assert (NULL != dc->rfh);
  GNUNET_assert (BRS_CHK_SET == dr->state);
  total = GNUNET_FS_uri_chk_get_file_size (dc->uri);
  GNUNET_assert (dr->depth < dc->treedepth);
  len = GNUNET_FS_tree_calculate_block_size (total, dr->offset, dr->depth);
  GNUNET_assert (len <= DBLOCK_SIZE);
  off = compute_disk_offset (total, dr->offset, dr->depth);
  if (dc->old_file_size < off + len)
    return; /* failure */
  if (off != GNUNET_DISK_file_seek (dc->rfh, off, GNUNET_DISK_SEEK_SET))
  {
    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "seek", dc->filename);
    return;   /* failure */
  }
  if (len != GNUNET_DISK_file_read (dc->rfh, block, len))
  {
    GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "read", dc->filename);
    return;   /* failure */
  }
  GNUNET_CRYPTO_hash (block, len, &key);
  if (0 != memcmp (&key, &dr->chk.key, sizeof(struct GNUNET_HashCode)))
    return; /* mismatch */
  if (GNUNET_OK !=
      encrypt_existing_match (dc, &dr->chk, dr, block, len, GNUNET_NO))
  {
    /* hash matches but encrypted block does not, really bad */
    dr->state = BRS_ERROR;
    /* propagate up */
    while (NULL != dr->parent)
    {
      dr = dr->parent;
      dr->state = BRS_ERROR;
    }
    return;
  }
  /* block matches */
  dr->state = BRS_DOWNLOAD_DOWN;
 
  /* set CHKs for children */
  up_done = GNUNET_YES;
  chks = (const struct ContentHashKey *) block;
  for (i = 0; i < dr->num_children; i++)
  {
    drc = dr->children[i];
    GNUNET_assert (drc->offset >= dr->offset);
    child_block_size = GNUNET_FS_tree_compute_tree_size (drc->depth);
    GNUNET_assert (0 == (drc->offset - dr->offset) % child_block_size);
    if (BRS_INIT == drc->state)
    {
      drc->state = BRS_CHK_SET;
      drc->chk = chks[drc->chk_idx];
      try_top_down_reconstruction (dc, drc);
    }
    if (BRS_DOWNLOAD_UP != drc->state)
      up_done = GNUNET_NO;   /* children not all done */
  }
  if (GNUNET_YES == up_done)
    propagate_up (dr); /* children all done (or no children...) */
}

References BRS_CHK_SET, BRS_DOWNLOAD_DOWN, BRS_DOWNLOAD_UP, BRS_ERROR, BRS_INIT, DownloadRequest::children, DownloadRequest::chk, DownloadRequest::chk_idx, compute_disk_offset(), DBLOCK_SIZE, dc, DownloadRequest::depth, encrypt_existing_match(), GNUNET_FS_DownloadContext::filename, GNUNET_assert, GNUNET_CRYPTO_hash(), GNUNET_DISK_file_read(), GNUNET_DISK_file_seek(), GNUNET_DISK_SEEK_SET, GNUNET_ERROR_TYPE_WARNING, GNUNET_FS_tree_calculate_block_size(), GNUNET_FS_tree_compute_tree_size(), GNUNET_FS_uri_chk_get_file_size(), GNUNET_log_strerror_file, GNUNET_NO, GNUNET_OK, GNUNET_YES, key, ContentHashKey::key, DownloadRequest::num_children, DownloadRequest::offset, GNUNET_FS_DownloadContext::old_file_size, DownloadRequest::parent, propagate_up(), GNUNET_FS_DownloadContext::rfh, DownloadRequest::state, GNUNET_FS_DownloadContext::treedepth, try_top_down_reconstruction(), and GNUNET_FS_DownloadContext::uri.

Referenced by GNUNET_FS_download_start_task_(), and try_top_down_reconstruction().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ retry_entry()

static int retry_entry	(	void *	cls,
		const struct GNUNET_HashCode *	key,
		void *	entry
	)

static

Add entries to the message queue.

Parameters

cls	our download context
key	unused
entry	entry of type `struct DownloadRequest`

Returns: GNUNET_OK

Definition at line 747 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct DownloadRequest *dr = entry;
  struct SearchMessage *sm;
  struct GNUNET_MQ_Envelope *env;
 
  env = GNUNET_MQ_msg (sm, GNUNET_MESSAGE_TYPE_FS_START_SEARCH);
  if (0 != (dc->options & GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY))
    sm->options = htonl (GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY);
  else
    sm->options = htonl (GNUNET_FS_SEARCH_OPTION_NONE);
  if (0 == dr->depth)
    sm->type = htonl (GNUNET_BLOCK_TYPE_FS_DBLOCK);
  else
    sm->type = htonl (GNUNET_BLOCK_TYPE_FS_IBLOCK);
  sm->anonymity_level = htonl (dc->anonymity);
  sm->target = dc->target;
  sm->query = dr->chk.query;
  GNUNET_MQ_send (dc->mq, env);
  return GNUNET_OK;
}

References GNUNET_FS_DownloadContext::anonymity, SearchMessage::anonymity_level, DownloadRequest::chk, dc, DownloadRequest::depth, env, GNUNET_BLOCK_TYPE_FS_DBLOCK, GNUNET_BLOCK_TYPE_FS_IBLOCK, GNUNET_FS_DOWNLOAD_OPTION_LOOPBACK_ONLY, GNUNET_FS_SEARCH_OPTION_LOOPBACK_ONLY, GNUNET_FS_SEARCH_OPTION_NONE, GNUNET_MESSAGE_TYPE_FS_START_SEARCH, GNUNET_MQ_msg, GNUNET_MQ_send(), GNUNET_OK, GNUNET_FS_DownloadContext::mq, SearchMessage::options, GNUNET_FS_DownloadContext::options, ContentHashKey::query, SearchMessage::query, SearchMessage::target, GNUNET_FS_DownloadContext::target, and SearchMessage::type.

Referenced by do_reconnect(), and schedule_block_download().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ schedule_block_download()

static void schedule_block_download	(	struct GNUNET_FS_DownloadContext *	dc,
		struct DownloadRequest *	dr
	)

static

Schedule the download of the specified block in the tree.

Parameters

dc	overall download this block belongs to
dr	request to schedule

Definition at line 778 of file fs_download.c.

{
  unsigned int i;
 
  switch (dr->state)
  {
  case BRS_INIT:
    GNUNET_assert (0);
    break;
 
  case BRS_RECONSTRUCT_DOWN:
    GNUNET_assert (0);
    break;
 
  case BRS_RECONSTRUCT_META_UP:
    GNUNET_assert (0);
    break;
 
  case BRS_RECONSTRUCT_UP:
    GNUNET_assert (0);
    break;
 
  case BRS_CHK_SET:
    /* normal case, start download */
    break;
 
  case BRS_DOWNLOAD_DOWN:
    for (i = 0; i < dr->num_children; i++)
      schedule_block_download (dc, dr->children[i]);
    return;
 
  case BRS_DOWNLOAD_UP:
    /* We're done! */
    return;
 
  case BRS_ERROR:
    GNUNET_break (0);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Scheduling download at offset %llu and depth %u for `%s'\n",
              (unsigned long long) dr->offset,
              dr->depth,
              GNUNET_h2s (&dr->chk.query));
  if (GNUNET_NO != GNUNET_CONTAINER_multihashmap_contains_value (dc->active,
                                                                 &dr->chk.query,
                                                                 dr))
    return; /* already active */
  GNUNET_CONTAINER_multihashmap_put (dc->active,
                                     &dr->chk.query,
                                     dr,
                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
  if (NULL == dc->mq)
    return; /* download not active */
  retry_entry (dc, &dr->chk.query, dr);
}

References GNUNET_FS_DownloadContext::active, BRS_CHK_SET, BRS_DOWNLOAD_DOWN, BRS_DOWNLOAD_UP, BRS_ERROR, BRS_INIT, BRS_RECONSTRUCT_DOWN, BRS_RECONSTRUCT_META_UP, BRS_RECONSTRUCT_UP, DownloadRequest::children, DownloadRequest::chk, dc, DownloadRequest::depth, GNUNET_assert, GNUNET_break, GNUNET_CONTAINER_multihashmap_contains_value(), GNUNET_CONTAINER_multihashmap_put(), GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE, GNUNET_ERROR_TYPE_DEBUG, GNUNET_h2s(), GNUNET_log, GNUNET_NO, GNUNET_FS_DownloadContext::mq, DownloadRequest::num_children, DownloadRequest::offset, ContentHashKey::query, retry_entry(), schedule_block_download(), and DownloadRequest::state.

Referenced by GNUNET_FS_download_start_task_(), process_result_with_request(), reconstruct_cont(), and schedule_block_download().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ GNUNET_FS_free_download_request_()

void GNUNET_FS_free_download_request_ ( struct DownloadRequest * dr )

(recursively) free download request structure

Parameters

dr	request to free

Definition at line 997 of file fs_download.c.

{
  if (NULL == dr)
    return;
  for (unsigned int i = 0; i < dr->num_children; i++)
    GNUNET_FS_free_download_request_ (dr->children[i]);
  GNUNET_free (dr->children);
  GNUNET_free (dr);
}

References DownloadRequest::children, GNUNET_free, GNUNET_FS_free_download_request_(), and DownloadRequest::num_children.

Referenced by free_download_context(), GNUNET_FS_download_signal_suspend_(), GNUNET_FS_download_stop(), GNUNET_FS_free_download_request_(), process_result_with_request(), and read_download_request().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ check_put()

static int check_put	(	void *	cls,
		const struct ClientPutMessage *	cm
	)

static

Type of a function to call when we check the PUT message from the service.

Parameters

cls	closure
msg	message received

Definition at line 1291 of file fs_download.c.

{
  /* any varsize length is OK */
  return GNUNET_OK;
}

References GNUNET_OK.

◆ handle_put()

static void handle_put	(	void *	cls,
		const struct ClientPutMessage *	cm
	)

static

Type of a function to call when we receive a message from the service.

Parameters

cls	closure
msg	message received

Definition at line 1306 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  uint16_t msize = ntohs (cm->header.size) - sizeof(*cm);
  struct ProcessResultClosure prc;
 
  prc.dc = dc;
  prc.data = &cm[1];
  prc.last_transmission = GNUNET_TIME_absolute_ntoh (cm->last_transmission);
  prc.size = msize;
  prc.type = ntohl (cm->type);
  prc.do_store = GNUNET_YES;
  prc.respect_offered = ntohl (cm->respect_offered);
  prc.num_transmissions = ntohl (cm->num_transmissions);
  GNUNET_CRYPTO_hash (prc.data, msize, &prc.query);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Received result for query `%s' from FS service\n",
              GNUNET_h2s (&prc.query));
  GNUNET_CONTAINER_multihashmap_get_multiple (dc->active,
                                              &prc.query,
                                              &process_result_with_request,
                                              &prc);
}

Here is the call graph for this function:

◆ download_mq_error_handler()

static void download_mq_error_handler	(	void *	cls,
		enum GNUNET_MQ_Error	error
	)

static

Generic error handler, called with the appropriate error code and the same closure specified at the creation of the message queue.

Not every message queue implementation supports an error handler.

Parameters

cls	closure with the `struct GNUNET_FS_DownloadContext *`
error	error code

Definition at line 1340 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
 
  if (NULL != dc->mq)
  {
    GNUNET_MQ_destroy (dc->mq);
    dc->mq = NULL;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Transmitting download request failed, trying to reconnect\n");
  try_reconnect (dc);
}

References dc, GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_MQ_destroy(), GNUNET_FS_DownloadContext::mq, and try_reconnect().

Referenced by do_reconnect().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ do_reconnect()

static void do_reconnect ( void * cls )

static

Reconnect to the FS service and transmit our queries NOW.

Parameters

cls	our download context

Definition at line 1361 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_MQ_MessageHandler handlers[] =
  { GNUNET_MQ_hd_var_size (put,
                           GNUNET_MESSAGE_TYPE_FS_PUT,
                           struct ClientPutMessage,
                           dc),
    GNUNET_MQ_handler_end () };
 
  dc->task = NULL;
  dc->mq = GNUNET_CLIENT_connect (dc->h->cfg,
                                  "fs",
                                  handlers,
                                  &download_mq_error_handler,
                                  dc);
  if (NULL == dc->mq)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Connecting to `%s'-service failed, will try again.\n",
                "FS");
    try_reconnect (dc);
    return;
  }
  GNUNET_CONTAINER_multihashmap_iterate (dc->active, &retry_entry, dc);
}

References GNUNET_FS_DownloadContext::active, GNUNET_FS_Handle::cfg, dc, download_mq_error_handler(), GNUNET_CLIENT_connect(), GNUNET_CONTAINER_multihashmap_iterate(), GNUNET_ERROR_TYPE_WARNING, GNUNET_log, GNUNET_MESSAGE_TYPE_FS_PUT, GNUNET_MQ_handler_end, GNUNET_MQ_hd_var_size, GNUNET_FS_DownloadContext::h, handlers, GNUNET_FS_DownloadContext::mq, retry_entry(), GNUNET_FS_DownloadContext::task, and try_reconnect().

Referenced by activate_fs_download(), and try_reconnect().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ activate_fs_download()

static void activate_fs_download ( void * cls )

static

We're allowed to ask the FS service for our blocks.

Start the download.

Parameters

cls	the 'struct GNUNET_FS_DownloadContext'

Definition at line 1427 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_FS_ProgressInfo pi;
 
  GNUNET_assert (NULL == dc->mq);
  GNUNET_assert (NULL != dc->active);
  do_reconnect (dc);
  if (NULL != dc->mq)
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download activated\n");
  pi.status = GNUNET_FS_STATUS_DOWNLOAD_ACTIVE;
  GNUNET_FS_download_make_status_ (&pi, dc);
}

References GNUNET_FS_DownloadContext::active, dc, do_reconnect(), GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_FS_download_make_status_(), GNUNET_FS_STATUS_DOWNLOAD_ACTIVE, GNUNET_log, GNUNET_FS_DownloadContext::mq, and GNUNET_FS_ProgressInfo::status.

Referenced by GNUNET_FS_download_resume(), and GNUNET_FS_download_start_downloading_().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ deactivate_fs_download()

static void deactivate_fs_download ( void * cls )

static

We must stop to ask the FS service for our blocks.

Pause the download.

Parameters

cls	the `struct GNUNET_FS_DownloadContext`

Definition at line 1448 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_FS_ProgressInfo pi;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download deactivated\n");
  if (NULL != dc->mq)
  {
    GNUNET_MQ_destroy (dc->mq);
    dc->mq = NULL;
  }
  pi.status = GNUNET_FS_STATUS_DOWNLOAD_INACTIVE;
  GNUNET_FS_download_make_status_ (&pi, dc);
}

References dc, GNUNET_ERROR_TYPE_DEBUG, GNUNET_FS_download_make_status_(), GNUNET_FS_STATUS_DOWNLOAD_INACTIVE, GNUNET_log, GNUNET_MQ_destroy(), GNUNET_FS_DownloadContext::mq, and GNUNET_FS_ProgressInfo::status.

Referenced by GNUNET_FS_download_resume(), GNUNET_FS_download_start_downloading_(), and GNUNET_FS_download_suspend().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ create_download_request()

static struct DownloadRequest * create_download_request	(	struct DownloadRequest *	parent,
		unsigned int	chk_idx,
		unsigned int	depth,
		uint64_t	dr_offset,
		uint64_t	file_start_offset,
		uint64_t	desired_length
	)

static

(recursively) Create a download request structure.

Parameters

parent	parent of the current entry
chk_idx	index of the chk for this block in the parent block
depth	depth of the current entry, 0 are the DBLOCKs, top level block is 'dc->treedepth - 1'
dr_offset	offset in the original file this block maps to (as in, offset of the first byte of the first DBLOCK in the subtree rooted in the returned download request tree)
file_start_offset	desired starting offset for the download in the original file; requesting tree should not contain DBLOCKs prior to the file_start_offset
desired_length	desired number of bytes the user wanted to access (from file_start_offset). Resulting tree should not contain DBLOCKs after file_start_offset + file_length.

Returns: download request tree for the given range of DBLOCKs at the specified depth

Definition at line 1484 of file fs_download.c.

{
  struct DownloadRequest *dr;
  unsigned int i;
  unsigned int head_skip;
  uint64_t child_block_size;
 
  dr = GNUNET_new (struct DownloadRequest);
  dr->parent = parent;
  dr->depth = depth;
  dr->offset = dr_offset;
  dr->chk_idx = chk_idx;
  if (0 == depth)
    return dr;
  child_block_size = GNUNET_FS_tree_compute_tree_size (depth - 1);
 
  /* calculate how many blocks at this level are not interesting
   * from the start (rounded down), either because of the requested
   * file offset or because this IBlock is further along */
  if (dr_offset < file_start_offset)
  {
    head_skip = (file_start_offset - dr_offset) / child_block_size;
  }
  else
  {
    head_skip = 0;
  }
 
  /* calculate index of last block at this level that is interesting (rounded up) */
  dr->num_children =
    (file_start_offset + desired_length - dr_offset) / child_block_size;
  if (dr->num_children * child_block_size <
      file_start_offset + desired_length - dr_offset)
    dr->num_children++; /* round up */
  GNUNET_assert (dr->num_children > head_skip);
  dr->num_children -= head_skip;
  if (dr->num_children > CHK_PER_INODE)
    dr->num_children = CHK_PER_INODE; /* cap at max */
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Block at offset %llu and depth %u has %u children\n",
              (unsigned long long) dr_offset,
              depth,
              dr->num_children);
 
  /* now we can get the total number of *interesting* children for this block */
 
  /* why else would we have gotten here to begin with? (that'd be a bad logic error) */
  GNUNET_assert (dr->num_children > 0);
 
  dr->children = GNUNET_new_array (dr->num_children, struct DownloadRequest *);
  for (i = 0; i < dr->num_children; i++)
  {
    dr->children[i] =
      create_download_request (dr,
                               i + head_skip,
                               depth - 1,
                               dr_offset + (i + head_skip) * child_block_size,
                               file_start_offset,
                               desired_length);
  }
  return dr;
}

References DownloadRequest::children, DownloadRequest::chk_idx, CHK_PER_INODE, create_download_request(), DownloadRequest::depth, GNUNET_assert, GNUNET_ERROR_TYPE_DEBUG, GNUNET_FS_tree_compute_tree_size(), GNUNET_log, GNUNET_new, GNUNET_new_array, DownloadRequest::num_children, DownloadRequest::offset, and DownloadRequest::parent.

Referenced by create_download_request(), and GNUNET_FS_download_start_task_().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ reconstruct_cont()

static void reconstruct_cont ( void * cls )

static

Continuation after a possible attempt to reconstruct the current IBlock from the existing file.

Parameters

cls	the 'struct ReconstructContext'

Definition at line 1560 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
 
  /* clean up state from tree encoder */
  if (NULL != dc->task)
  {
    GNUNET_SCHEDULER_cancel (dc->task);
    dc->task = NULL;
  }
  if (NULL != dc->rfh)
  {
    GNUNET_break (GNUNET_OK == GNUNET_DISK_file_close (dc->rfh));
    dc->rfh = NULL;
  }
  /* start "normal" download */
  dc->issue_requests = GNUNET_YES;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting normal download\n");
  schedule_block_download (dc, dc->top_request);
}

References dc, GNUNET_break, GNUNET_DISK_file_close(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_log, GNUNET_OK, GNUNET_SCHEDULER_cancel(), GNUNET_YES, GNUNET_FS_DownloadContext::issue_requests, GNUNET_FS_DownloadContext::rfh, schedule_block_download(), GNUNET_FS_DownloadContext::task, and GNUNET_FS_DownloadContext::top_request.

Referenced by GNUNET_FS_download_start_task_().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ get_next_block()

static void get_next_block ( void * cls )

static

Task requesting the next block from the tree encoder.

Parameters

cls	the 'struct GNUJNET_FS_DownloadContext' we're processing

Definition at line 1588 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
 
  dc->task = NULL;
  GNUNET_FS_tree_encoder_next (dc->te);
}

References dc, GNUNET_FS_tree_encoder_next(), GNUNET_FS_DownloadContext::task, and GNUNET_FS_DownloadContext::te.

Referenced by GNUNET_FS_download_start_task_(), and reconstruct_cb().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ reconstruct_cb()

static void reconstruct_cb	(	void *	cls,
		const struct ContentHashKey *	chk,
		uint64_t	offset,
		unsigned int	depth,
		enum GNUNET_BLOCK_Type	type,
		const void *	block,
		uint16_t	block_size
	)

static

Function called asking for the current (encoded) block to be processed.

After processing the client should either call "GNUNET_FS_tree_encode_next" or (on error) "GNUNET_FS_tree_encode_finish".

This function checks if the content on disk matches the expected content based on the URI.

Parameters

cls	closure
chk	content hash key for the block
offset	offset of the block
depth	depth of the block, 0 for DBLOCK
type	type of the block (IBLOCK or DBLOCK)
block	the (encrypted) block
block_size	size of block (in bytes)

Definition at line 1615 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_FS_ProgressInfo pi;
  struct DownloadRequest *dr;
  uint64_t blen;
  unsigned int chld;
 
  /* find corresponding request entry */
  dr = dc->top_request;
  while (dr->depth > depth)
  {
    GNUNET_assert (dr->num_children > 0);
    blen = GNUNET_FS_tree_compute_tree_size (dr->depth - 1);
    chld = (offset - dr->offset) / blen;
    if (chld < dr->children[0]->chk_idx)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Block %u < %u irrelevant for our range\n",
                  chld,
                  dr->children[0]->chk_idx);
      dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
      return;     /* irrelevant block */
    }
    if (chld > dr->children[dr->num_children - 1]->chk_idx)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Block %u > %u irrelevant for our range\n",
                  chld,
                  dr->children[dr->num_children - 1]->chk_idx);
      dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
      return;     /* irrelevant block */
    }
    dr = dr->children[chld - dr->children[0]->chk_idx];
  }
  GNUNET_log (
    GNUNET_ERROR_TYPE_DEBUG,
    "Matched TE block with request at offset %llu and depth %u in state %d\n",
    (unsigned long long) dr->offset,
    dr->depth,
    dr->state);
  /* FIXME: this code needs more testing and might
     need to handle more states... */
  switch (dr->state)
  {
  case BRS_INIT:
    break;
 
  case BRS_RECONSTRUCT_DOWN:
    break;
 
  case BRS_RECONSTRUCT_META_UP:
    break;
 
  case BRS_RECONSTRUCT_UP:
    break;
 
  case BRS_CHK_SET:
    if (0 == memcmp (chk, &dr->chk, sizeof(struct ContentHashKey)))
    {
      GNUNET_log (
        GNUNET_ERROR_TYPE_DEBUG,
        "Reconstruction succeeded, can use block at offset %llu, depth %u\n",
        (unsigned long long) offset,
        depth);
      /* block matches, hence tree below matches;
       * this request is done! */
      dr->state = BRS_DOWNLOAD_UP;
      (void) GNUNET_CONTAINER_multihashmap_remove (dc->active,
                                                   &dr->chk.query,
                                                   dr);
      /* calculate how many bytes of payload this block
       * corresponds to */
      blen = GNUNET_FS_tree_compute_tree_size (dr->depth);
      /* how many of those bytes are in the requested range? */
      blen = GNUNET_MIN (blen, dc->length + dc->offset - dr->offset);
      /* signal progress */
      dc->completed += blen;
      pi.status = GNUNET_FS_STATUS_DOWNLOAD_PROGRESS;
      pi.value.download.specifics.progress.data = NULL;
      pi.value.download.specifics.progress.offset = offset;
      pi.value.download.specifics.progress.data_len = 0;
      pi.value.download.specifics.progress.depth = 0;
      pi.value.download.specifics.progress.respect_offered = 0;
      pi.value.download.specifics.progress.block_download_duration =
        GNUNET_TIME_UNIT_ZERO;
      GNUNET_FS_download_make_status_ (&pi, dc);
      /* FIXME: duplicated code from 'process_result_with_request - refactor */
      if (dc->completed == dc->length)
      {
        /* download completed, signal */
        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                    "Download completed, truncating file to desired length %llu\n",
                    (unsigned long long) GNUNET_ntohll (
                      dc->uri->data.chk.file_length));
        /* truncate file to size (since we store IBlocks at the end) */
        if (NULL != dc->filename)
        {
          if (0 != truncate (dc->filename,
                             GNUNET_ntohll (dc->uri->data.chk.file_length)))
            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                                      "truncate",
                                      dc->filename);
        }
      }
    }
    else
      GNUNET_log (
        GNUNET_ERROR_TYPE_DEBUG,
        "Reconstruction failed, need to download block at offset %llu, depth %u\n",
        (unsigned long long) offset,
        depth);
    break;
 
  case BRS_DOWNLOAD_DOWN:
    break;
 
  case BRS_DOWNLOAD_UP:
    break;
 
  case BRS_ERROR:
    break;
 
  default:
    GNUNET_assert (0);
    break;
  }
  dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
  if ((dr == dc->top_request) && (dr->state == BRS_DOWNLOAD_UP))
    check_completed (dc);
}

References GNUNET_FS_DownloadContext::active, BRS_CHK_SET, BRS_DOWNLOAD_DOWN, BRS_DOWNLOAD_UP, BRS_ERROR, BRS_INIT, BRS_RECONSTRUCT_DOWN, BRS_RECONSTRUCT_META_UP, BRS_RECONSTRUCT_UP, check_completed(), DownloadRequest::children, GNUNET_FS_Uri::chk, DownloadRequest::chk, DownloadRequest::chk_idx, GNUNET_FS_DownloadContext::completed, GNUNET_FS_Uri::data, dc, DownloadRequest::depth, GNUNET_FS_ProgressInfo::download, FileIdentifier::file_length, GNUNET_FS_DownloadContext::filename, get_next_block(), GNUNET_assert, GNUNET_CONTAINER_multihashmap_remove(), GNUNET_ERROR_TYPE_DEBUG, GNUNET_ERROR_TYPE_WARNING, GNUNET_FS_download_make_status_(), GNUNET_FS_STATUS_DOWNLOAD_PROGRESS, GNUNET_FS_tree_compute_tree_size(), GNUNET_log, GNUNET_log_strerror_file, GNUNET_MIN, GNUNET_ntohll(), GNUNET_SCHEDULER_add_now(), GNUNET_TIME_UNIT_ZERO, GNUNET_FS_DownloadContext::length, DownloadRequest::num_children, DownloadRequest::offset, GNUNET_FS_DownloadContext::offset, ContentHashKey::query, DownloadRequest::state, GNUNET_FS_ProgressInfo::status, GNUNET_FS_DownloadContext::task, GNUNET_FS_DownloadContext::top_request, GNUNET_FS_DownloadContext::uri, and GNUNET_FS_ProgressInfo::value.

Referenced by GNUNET_FS_download_start_task_().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ fh_reader()

static size_t fh_reader	(	void *	cls,
		uint64_t	offset,
		size_t	max,
		void *	buf,
		char **	emsg
	)

static

Function called by the tree encoder to obtain a block of plaintext data (for the lowest level of the tree).

Parameters

cls	our 'struct ReconstructContext'
offset	identifies which block to get
max	(maximum) number of bytes to get; returning fewer will also cause errors
buf	where to copy the plaintext buffer
emsg	location to store an error message (on error)

Returns: number of bytes copied to buf, 0 on error

Definition at line 1767 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_DISK_FileHandle *fh = dc->rfh;
  ssize_t ret;
 
  if (NULL != emsg)
    *emsg = NULL;
  if (offset != GNUNET_DISK_file_seek (fh, offset, GNUNET_DISK_SEEK_SET))
  {
    if (NULL != emsg)
      *emsg = GNUNET_strdup (strerror (errno));
    return 0;
  }
  ret = GNUNET_DISK_file_read (fh, buf, max);
  if (ret < 0)
  {
    if (NULL != emsg)
      *emsg = GNUNET_strdup (strerror (errno));
    return 0;
  }
  return ret;
}

References dc, GNUNET_DISK_file_read(), GNUNET_DISK_file_seek(), GNUNET_DISK_SEEK_SET, GNUNET_strdup, max, ret, and GNUNET_FS_DownloadContext::rfh.

Referenced by GNUNET_FS_download_start_task_().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ GNUNET_FS_download_start_task_()

void GNUNET_FS_download_start_task_ ( void * cls )

Task that creates the initial (top-level) download request for the file.

Parameters

cls	the 'struct GNUNET_FS_DownloadContext'

Definition at line 1799 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_FS_ProgressInfo pi;
  struct GNUNET_DISK_FileHandle *fh;
 
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Start task running...\n");
  dc->task = NULL;
  if (0 == dc->length)
  {
    /* no bytes required! */
    if (NULL != dc->filename)
    {
      fh = GNUNET_DISK_file_open (dc->filename,
                                  GNUNET_DISK_OPEN_READWRITE
                                  | GNUNET_DISK_OPEN_CREATE
                                  | ((0 ==
                                      GNUNET_FS_uri_chk_get_file_size (dc->uri))
                                     ? GNUNET_DISK_OPEN_TRUNCATE
                                     : 0),
                                  GNUNET_DISK_PERM_USER_READ
                                  | GNUNET_DISK_PERM_USER_WRITE
                                  | GNUNET_DISK_PERM_GROUP_READ
                                  | GNUNET_DISK_PERM_OTHER_READ);
      GNUNET_DISK_file_close (fh);
    }
    GNUNET_FS_download_sync_ (dc);
    pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
    pi.value.download.specifics.start.meta = dc->meta;
    GNUNET_FS_download_make_status_ (&pi, dc);
    check_completed (dc);
    return;
  }
  if (NULL != dc->emsg)
    return;
  if (NULL == dc->top_request)
  {
    dc->top_request = create_download_request (NULL,
                                               0,
                                               dc->treedepth - 1,
                                               0,
                                               dc->offset,
                                               dc->length);
    dc->top_request->state = BRS_CHK_SET;
    dc->top_request->chk = (dc->uri->type == GNUNET_FS_URI_CHK)
                           ? dc->uri->data.chk.chk
                           : dc->uri->data.loc.fi.chk;
    /* signal start */
    GNUNET_FS_download_sync_ (dc);
    if (NULL != dc->search)
      GNUNET_FS_search_result_sync_ (dc->search);
    pi.status = GNUNET_FS_STATUS_DOWNLOAD_START;
    pi.value.download.specifics.start.meta = dc->meta;
    GNUNET_FS_download_make_status_ (&pi, dc);
  }
  GNUNET_FS_download_start_downloading_ (dc);
  /* attempt reconstruction from disk */
  if (GNUNET_YES == GNUNET_DISK_file_test (dc->filename))
    dc->rfh = GNUNET_DISK_file_open (dc->filename,
                                     GNUNET_DISK_OPEN_READ,
                                     GNUNET_DISK_PERM_NONE);
  if (dc->top_request->state == BRS_CHK_SET)
  {
    if (NULL != dc->rfh)
    {
      /* first, try top-down */
      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                  "Trying top-down reconstruction for `%s'\n",
                  dc->filename);
      try_top_down_reconstruction (dc, dc->top_request);
      switch (dc->top_request->state)
      {
      case BRS_CHK_SET:
        break;       /* normal */
 
      case BRS_DOWNLOAD_DOWN:
        break;       /* normal, some blocks already down */
 
      case BRS_DOWNLOAD_UP:
        /* already done entirely, party! */
        if (NULL != dc->rfh)
        {
          /* avoid hanging on to file handle longer than
           * necessary */
          GNUNET_DISK_file_close (dc->rfh);
          dc->rfh = NULL;
        }
        return;
 
      case BRS_ERROR:
        GNUNET_asprintf (&dc->emsg, _ ("Invalid URI"));
        GNUNET_FS_download_sync_ (dc);
        pi.status = GNUNET_FS_STATUS_DOWNLOAD_ERROR;
        pi.value.download.specifics.error.message = dc->emsg;
        GNUNET_FS_download_make_status_ (&pi, dc);
        return;
 
      default:
        GNUNET_assert (0);
        break;
      }
    }
  }
  /* attempt reconstruction from meta data */
  if ((GNUNET_FS_uri_chk_get_file_size (dc->uri) <= MAX_INLINE_SIZE) &&
      (NULL != dc->meta))
  {
    GNUNET_log (
      GNUNET_ERROR_TYPE_DEBUG,
      "Trying to find embedded meta data for download of size %llu with %u bytes MD\n",
      (unsigned long long) GNUNET_FS_uri_chk_get_file_size (dc->uri),
      (unsigned int) GNUNET_FS_meta_data_get_serialized_size (dc->meta));
    GNUNET_FS_meta_data_iterate (dc->meta, &match_full_data, dc);
    if (BRS_DOWNLOAD_UP == dc->top_request->state)
    {
      if (NULL != dc->rfh)
      {
        /* avoid hanging on to file handle longer than
         * necessary */
        GNUNET_DISK_file_close (dc->rfh);
        dc->rfh = NULL;
      }
      return;     /* finished, status update was already done for us */
    }
  }
  if (NULL != dc->rfh)
  {
    /* finally, actually run bottom-up */
    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                "Trying bottom-up reconstruction of file `%s'\n",
                dc->filename);
    dc->te =
      GNUNET_FS_tree_encoder_create (dc->h,
                                     GNUNET_FS_uri_chk_get_file_size (dc->uri),
                                     dc,
                                     &fh_reader,
                                     &reconstruct_cb,
                                     NULL,
                                     &reconstruct_cont);
    dc->task = GNUNET_SCHEDULER_add_now (&get_next_block, dc);
  }
  else
  {
    /* simple, top-level download */
    dc->issue_requests = GNUNET_YES;
    schedule_block_download (dc, dc->top_request);
  }
  if (BRS_DOWNLOAD_UP == dc->top_request->state)
    check_completed (dc);
}

Referenced by create_download_context(), and deserialize_download().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ GNUNET_FS_download_signal_suspend_()

void GNUNET_FS_download_signal_suspend_ ( void * cls )

Create SUSPEND event for the given download operation and then clean up our state (without stop signal).

Parameters

cls	the `struct GNUNET_FS_DownloadContext` to signal for

Definition at line 1952 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc = cls;
  struct GNUNET_FS_ProgressInfo pi;
 
  if (NULL != dc->top)
    GNUNET_FS_end_top (dc->h, dc->top);
  while (NULL != dc->child_head)
    GNUNET_FS_download_signal_suspend_ (dc->child_head);
  if (NULL != dc->search)
  {
    dc->search->download = NULL;
    dc->search = NULL;
  }
  if (NULL != dc->job_queue)
  {
    GNUNET_FS_dequeue_ (dc->job_queue);
    dc->job_queue = NULL;
  }
  if (NULL != dc->parent)
    GNUNET_CONTAINER_DLL_remove (dc->parent->child_head,
                                 dc->parent->child_tail,
                                 dc);
  if (NULL != dc->task)
  {
    GNUNET_SCHEDULER_cancel (dc->task);
    dc->task = NULL;
  }
  pi.status = GNUNET_FS_STATUS_DOWNLOAD_SUSPEND;
  GNUNET_FS_download_make_status_ (&pi, dc);
  if (NULL != dc->te)
  {
    GNUNET_FS_tree_encoder_finish (dc->te, NULL);
    dc->te = NULL;
  }
  if (NULL != dc->rfh)
  {
    GNUNET_DISK_file_close (dc->rfh);
    dc->rfh = NULL;
  }
  GNUNET_FS_free_download_request_ (dc->top_request);
  if (NULL != dc->active)
  {
    GNUNET_CONTAINER_multihashmap_destroy (dc->active);
    dc->active = NULL;
  }
  GNUNET_free (dc->filename);
  GNUNET_FS_meta_data_destroy (dc->meta);
  GNUNET_FS_uri_destroy (dc->uri);
  GNUNET_free (dc->temp_filename);
  GNUNET_free (dc->serialization);
  GNUNET_assert (NULL == dc->job_queue);
  GNUNET_free (dc);
}

Referenced by deserialize_download(), GNUNET_FS_download_signal_suspend_(), GNUNET_FS_download_start(), search_result_stop(), and search_result_suspend().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ create_download_context()

struct GNUNET_FS_DownloadContext * create_download_context	(	struct GNUNET_FS_Handle *	h,
		const struct GNUNET_FS_Uri *	uri,
		const struct GNUNET_FS_MetaData *	meta,
		const char *	filename,
		const char *	tempname,
		uint64_t	offset,
		uint64_t	length,
		uint32_t	anonymity,
		enum GNUNET_FS_DownloadOptions	options,
		void *	cctx
	)

Helper function to setup the download context.

Parameters

h	handle to the file sharing subsystem
uri	the URI of the file (determines what to download); CHK or LOC URI
meta	known metadata for the file (can be NULL)
filename	where to store the file, maybe NULL (then no file is created on disk and data must be grabbed from the callbacks)
tempname	where to store temporary file data, not used if filename is non-NULL; can be NULL (in which case we will pick a name if needed); the temporary file may already exist, in which case we will try to use the data that is there and if it is not what is desired, will overwrite it
offset	at what offset should we start the download (typically 0)
length	how many bytes should be downloaded starting at offset
anonymity	anonymity level to use for the download
options	various options
cctx	initial value for the client context for this download

Returns: context that can be used to control this download

Definition at line 2028 of file fs_download.c.

{
  struct GNUNET_FS_DownloadContext *dc;
 
  GNUNET_assert (GNUNET_FS_uri_test_chk (uri) || GNUNET_FS_uri_test_loc (uri));
  if ((offset + length < offset) ||
      (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)))
  {
    GNUNET_break (0);
    return NULL;
  }
  dc = GNUNET_new (struct GNUNET_FS_DownloadContext);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Starting download %p, %u bytes at offset %llu\n",
              dc,
              (unsigned int) length,
              (unsigned long long) offset);
  dc->h = h;
  dc->uri = GNUNET_FS_uri_dup (uri);
  dc->meta = GNUNET_FS_meta_data_duplicate (meta);
  dc->client_info = cctx;
  dc->start_time = GNUNET_TIME_absolute_get ();
  if (NULL != filename)
  {
    dc->filename = GNUNET_strdup (filename);
    if (GNUNET_YES == GNUNET_DISK_file_test (filename))
      GNUNET_break (GNUNET_OK == GNUNET_DISK_file_size (filename,
                                                        &dc->old_file_size,
                                                        GNUNET_YES,
                                                        GNUNET_YES));
  }
  if (GNUNET_FS_uri_test_loc (dc->uri))
    GNUNET_assert (GNUNET_OK ==
                   GNUNET_FS_uri_loc_get_peer_identity (dc->uri, &dc->target));
  dc->offset = offset;
  dc->length = length;
  dc->anonymity = anonymity;
  dc->options = options;
  dc->active =
    GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE),
                                          GNUNET_NO);
  dc->treedepth =
    GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size (dc->uri));
  if ((NULL == filename) && (is_recursive_download (dc)))
  {
    if (NULL != tempname)
      dc->temp_filename = GNUNET_strdup (tempname);
    else
      dc->temp_filename = GNUNET_DISK_mktemp ("gnunet-directory-download-tmp");
  }
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Starting download `%s' of %llu bytes with tree depth %u\n",
              filename,
              (unsigned long long) length,
              dc->treedepth);
  GNUNET_assert (NULL == dc->job_queue);
  dc->task = GNUNET_SCHEDULER_add_now (&GNUNET_FS_download_start_task_, dc);
  return dc;
}

Referenced by GNUNET_FS_download_start(), and GNUNET_FS_download_start_from_search().

Here is the call graph for this function:

Here is the caller graph for this function:

◆ GNUNET_FS_download_start_downloading_()

void GNUNET_FS_download_start_downloading_ ( struct GNUNET_FS_DownloadContext * dc )

Start the downloading process (by entering the queue).

Parameters

dc	our download context

Definition at line 2183 of file fs_download.c.

{
  if (dc->completed == dc->length)
    return;
  if (NULL != dc->mq)
    return; /* already running */
  GNUNET_assert (NULL == dc->job_queue);
  GNUNET_assert (NULL == dc->task);
  GNUNET_assert (NULL != dc->active);
  dc->job_queue =
    GNUNET_FS_queue_ (dc->h,
                      &activate_fs_download,
                      &deactivate_fs_download,
                      dc,
                      (dc->length + DBLOCK_SIZE - 1) / DBLOCK_SIZE,
                      (0 == (dc->options & GNUNET_FS_DOWNLOAD_IS_PROBE))
                      ? GNUNET_FS_QUEUE_PRIORITY_NORMAL
                      : GNUNET_FS_QUEUE_PRIORITY_PROBE);
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "Download %p put into queue as job %p\n",
              dc,
              dc->job_queue);
}

Referenced by GNUNET_FS_download_start_task_().

Here is the call graph for this function:

Here is the caller graph for this function:

Data Structures

Macros

Functions

Detailed Description

Macro Definition Documentation

◆ GNUNET_FS_URI_CHK_PREFIX

Function Documentation

◆ is_recursive_download()

◆ compute_disk_offset()

◆ GNUNET_FS_download_make_status_()

◆ process_result_with_request()

◆ encrypt_existing_match()

◆ try_reconnect()

◆ trigger_recursive_download()

◆ full_recursive_download()

◆ check_completed()

◆ try_match_block()

◆ match_full_data()

◆ propagate_up()

◆ try_top_down_reconstruction()

◆ retry_entry()

◆ schedule_block_download()

◆ GNUNET_FS_free_download_request_()

◆ check_put()

◆ handle_put()

◆ download_mq_error_handler()

◆ do_reconnect()

◆ activate_fs_download()

◆ deactivate_fs_download()

◆ create_download_request()

◆ reconstruct_cont()

◆ get_next_block()

◆ reconstruct_cb()

◆ fh_reader()

◆ GNUNET_FS_download_start_task_()

◆ GNUNET_FS_download_signal_suspend_()

◆ create_download_context()

◆ GNUNET_FS_download_start_downloading_()