gnunet-publish.c

Go to the documentation of this file.

/*
     This file is part of GNUnet.
     Copyright (C) 2001-2013 GNUnet e.V.

     GNUnet is free software: you can redistribute it and/or modify it
     under the terms of the GNU Affero General Public License as published
     by the Free Software Foundation, either version 3 of the License,
     or (at your option) any later version.

     GNUnet is distributed in the hope that it will be useful, but
     WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Affero General Public License for more details.

     You should have received a copy of the GNU Affero General Public License
     along with this program.  If not, see <http://www.gnu.org/licenses/>.

     SPDX-License-Identifier: AGPL3.0-or-later
 */
#include "platform.h"
#include "gnunet_fs_service.h"
#include "gnunet_identity_service.h"

static int ret;

static unsigned int verbose;

static const struct GNUNET_CONFIGURATION_Handle *cfg;

static struct GNUNET_FS_Handle *ctx;

static struct GNUNET_FS_PublishContext *pc;

static struct GNUNET_CONTAINER_MetaData *meta;

static struct GNUNET_FS_Uri *topKeywords;

static struct GNUNET_FS_BlockOptions bo = { { 0LL }, 1, 365, 1 };

static char *uri_string;

static struct GNUNET_FS_Uri *uri;

static char *next_id;

static char *this_id;

static char *pseudonym;

static int do_insert;

static int disable_extractor;

static int do_simulate;

static int extract_only;

static int enable_creation_time;

static struct GNUNET_FS_DirScanner *ds;

static struct GNUNET_IDENTITY_Ego *namespace;

static struct GNUNET_IDENTITY_Handle *identity;


static void
do_stop_task (void *cls)
{
  struct GNUNET_FS_PublishContext *p;

  if (NULL != ds)
  {
    GNUNET_FS_directory_scan_abort (ds);
    ds = NULL;
  }
  if (NULL != identity)
  {
    GNUNET_IDENTITY_disconnect (identity);
    identity = NULL;
  }
  if (NULL != pc)
  {
    p = pc;
    pc = NULL;
    GNUNET_FS_publish_stop (p);
  }
  if (NULL != ctx)
  {
    GNUNET_FS_stop (ctx);
    ctx = NULL;
  }
  if (NULL != meta)
  {
    GNUNET_CONTAINER_meta_data_destroy (meta);
    meta = NULL;
  }
  if (NULL != uri)
  {
    GNUNET_FS_uri_destroy (uri);
    uri = NULL;
  }
}


static void *
progress_cb (void *cls, const struct GNUNET_FS_ProgressInfo *info)
{
  const char *s;
  char *suri;

  switch (info->status)
  {
  case GNUNET_FS_STATUS_PUBLISH_START:
    break;

  case GNUNET_FS_STATUS_PUBLISH_PROGRESS:
    if (verbose)
    {
      s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.eta,
                                                  GNUNET_YES);
      fprintf (stdout,
               _ ("Publishing `%s' at %llu/%llu (%s remaining)\n"),
               info->value.publish.filename,
               (unsigned long long) info->value.publish.completed,
               (unsigned long long) info->value.publish.size,
               s);
    }
    break;

  case GNUNET_FS_STATUS_PUBLISH_PROGRESS_DIRECTORY:
    if (verbose)
    {
      s = GNUNET_STRINGS_relative_time_to_string (info->value.publish.specifics
                                                  .progress_directory.eta,
                                                  GNUNET_YES);
      fprintf (stdout,
               _ ("Publishing `%s' at %llu/%llu (%s remaining)\n"),
               info->value.publish.filename,
               (unsigned long long)
               info->value.publish.specifics.progress_directory.completed,
               (unsigned long long)
               info->value.publish.specifics.progress_directory.total,
               s);
    }
    break;

  case GNUNET_FS_STATUS_PUBLISH_ERROR:
    fprintf (stderr,
             _ ("Error publishing: %s.\n"),
             info->value.publish.specifics.error.message);
    ret = 1;
    GNUNET_SCHEDULER_shutdown ();
    break;

  case GNUNET_FS_STATUS_PUBLISH_COMPLETED:
    fprintf (stdout,
             _ ("Publishing `%s' done.\n"),
             info->value.publish.filename);
    suri =
      GNUNET_FS_uri_to_string (info->value.publish.specifics.completed.chk_uri);
    fprintf (stdout, _ ("URI is `%s'.\n"), suri);
    GNUNET_free (suri);
    if (NULL != info->value.publish.specifics.completed.sks_uri)
    {
      suri = GNUNET_FS_uri_to_string (
        info->value.publish.specifics.completed.sks_uri);
      fprintf (stdout, _ ("Namespace URI is `%s'.\n"), suri);
      GNUNET_free (suri);
    }
    if (NULL == info->value.publish.pctx)
    {
      ret = 0;
      GNUNET_SCHEDULER_shutdown ();
    }
    break;

  case GNUNET_FS_STATUS_PUBLISH_STOPPED:
    GNUNET_break (NULL == pc);
    return NULL;

  case GNUNET_FS_STATUS_UNINDEX_START:
    fprintf (stderr, "%s", _ ("Starting cleanup after abort\n"));
    return NULL;

  case GNUNET_FS_STATUS_UNINDEX_PROGRESS:
    return NULL;

  case GNUNET_FS_STATUS_UNINDEX_COMPLETED:
    fprintf (stderr, "%s", _ ("Cleanup after abort completed.\n"));
    GNUNET_FS_unindex_stop (info->value.unindex.uc);
    return NULL;

  case GNUNET_FS_STATUS_UNINDEX_ERROR:
    fprintf (stderr, "%s", _ ("Cleanup after abort failed.\n"));
    GNUNET_FS_unindex_stop (info->value.unindex.uc);
    return NULL;

  case GNUNET_FS_STATUS_UNINDEX_STOPPED:
    return NULL;

  default:
    fprintf (stderr, _ ("Unexpected status: %d\n"), info->status);
    return NULL;
  }
  return ""; /* non-null */
}


static int
meta_printer (void *cls,
              const char *plugin_name,
              enum EXTRACTOR_MetaType type,
              enum EXTRACTOR_MetaFormat format,
              const char *data_mime_type,
              const char *data,
              size_t data_size)
{
  if ((EXTRACTOR_METAFORMAT_UTF8 != format) &&
      (EXTRACTOR_METAFORMAT_C_STRING != format))
    return 0;
  if (EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME == type)
    return 0;
#if HAVE_LIBEXTRACTOR
  fprintf (stdout, "\t%s - %s\n", EXTRACTOR_metatype_to_string (type), data);
#else
  fprintf (stdout, "\t%d - %s\n", type, data);
#endif
  return 0;
}


static int
keyword_printer (void *cls, const char *keyword, int is_mandatory)
{
  fprintf (stdout, "\t%s\n", keyword);
  return GNUNET_OK;
}


static int
publish_inspector (void *cls,
                   struct GNUNET_FS_FileInformation *fi,
                   uint64_t length,
                   struct GNUNET_CONTAINER_MetaData *m,
                   struct GNUNET_FS_Uri **uri,
                   struct GNUNET_FS_BlockOptions *bo,
                   int *do_index,
                   void **client_info)
{
  char *fn;
  char *fs;
  struct GNUNET_FS_Uri *new_uri;

  if (cls == fi)
    return GNUNET_OK;
  if ((disable_extractor) && (NULL != *uri))
  {
    GNUNET_FS_uri_destroy (*uri);
    *uri = NULL;
  }
  if (NULL != topKeywords)
  {
    if (NULL != *uri)
    {
      new_uri = GNUNET_FS_uri_ksk_merge (topKeywords, *uri);
      GNUNET_FS_uri_destroy (*uri);
      *uri = new_uri;
      GNUNET_FS_uri_destroy (topKeywords);
    }
    else
    {
      *uri = topKeywords;
    }
    topKeywords = NULL;
  }
  if (NULL != meta)
  {
    GNUNET_CONTAINER_meta_data_merge (m, meta);
    GNUNET_CONTAINER_meta_data_destroy (meta);
    meta = NULL;
  }
  if (enable_creation_time)
    GNUNET_CONTAINER_meta_data_add_publication_date (m);
  if (extract_only)
  {
    fn = GNUNET_CONTAINER_meta_data_get_by_type (
      m,
      EXTRACTOR_METATYPE_GNUNET_ORIGINAL_FILENAME);
    fs = GNUNET_STRINGS_byte_size_fancy (length);
    fprintf (stdout, _ ("Meta data for file `%s' (%s)\n"), fn, fs);
    GNUNET_CONTAINER_meta_data_iterate (m, &meta_printer, NULL);
    fprintf (stdout, _ ("Keywords for file `%s' (%s)\n"), fn, fs);
    GNUNET_free (fn);
    GNUNET_free (fs);
    if (NULL != *uri)
      GNUNET_FS_uri_ksk_get_keywords (*uri, &keyword_printer, NULL);
    fprintf (stdout, "%s", "\n");
  }
  if (GNUNET_YES == GNUNET_FS_meta_data_test_for_directory (m))
    GNUNET_FS_file_information_inspect (fi, &publish_inspector, fi);
  return GNUNET_OK;
}


static void
uri_sks_continuation (void *cls,
                      const struct GNUNET_FS_Uri *sks_uri,
                      const char *emsg)
{
  if (NULL != emsg)
  {
    fprintf (stderr, "%s\n", emsg);
    ret = 1;
  }
  GNUNET_SCHEDULER_shutdown ();
}


static void
uri_ksk_continuation (void *cls,
                      const struct GNUNET_FS_Uri *ksk_uri,
                      const char *emsg)
{
  const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv;

  if (NULL != emsg)
  {
    fprintf (stderr, "%s\n", emsg);
    ret = 1;
  }
  if (NULL == namespace)
  {
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  priv = GNUNET_IDENTITY_ego_get_private_key (namespace);
  GNUNET_FS_publish_sks (ctx,
                         priv,
                         this_id,
                         next_id,
                         meta,
                         uri,
                         &bo,
                         GNUNET_FS_PUBLISH_OPTION_NONE,
                         &uri_sks_continuation,
                         NULL);
}


static struct GNUNET_FS_FileInformation *
get_file_information (struct GNUNET_FS_ShareTreeItem *item)
{
  struct GNUNET_FS_FileInformation *fi;
  struct GNUNET_FS_FileInformation *fic;
  struct GNUNET_FS_ShareTreeItem *child;

  if (GNUNET_YES == item->is_directory)
  {
    if (NULL == item->meta)
      item->meta = GNUNET_CONTAINER_meta_data_create ();
    GNUNET_CONTAINER_meta_data_delete (item->meta,
                                       EXTRACTOR_METATYPE_MIMETYPE,
                                       NULL,
                                       0);
    GNUNET_FS_meta_data_make_directory (item->meta);
    if (NULL == item->ksk_uri)
    {
      const char *mime = GNUNET_FS_DIRECTORY_MIME;
      item->ksk_uri = GNUNET_FS_uri_ksk_create_from_args (1, &mime);
    }
    else
      GNUNET_FS_uri_ksk_add_keyword (item->ksk_uri,
                                     GNUNET_FS_DIRECTORY_MIME,
                                     GNUNET_NO);
    fi = GNUNET_FS_file_information_create_empty_directory (ctx,
                                                            NULL,
                                                            item->ksk_uri,
                                                            item->meta,
                                                            &bo,
                                                            item->filename);
    for (child = item->children_head; child; child = child->next)
    {
      fic = get_file_information (child);
      GNUNET_break (GNUNET_OK == GNUNET_FS_file_information_add (fi, fic));
    }
  }
  else
  {
    fi = GNUNET_FS_file_information_create_from_file (ctx,
                                                      NULL,
                                                      item->filename,
                                                      item->ksk_uri,
                                                      item->meta,
                                                      ! do_insert,
                                                      &bo);
  }
  return fi;
}


static void
directory_trim_complete (struct GNUNET_FS_ShareTreeItem *directory_scan_result)
{
  struct GNUNET_FS_FileInformation *fi;
  const struct GNUNET_CRYPTO_EcdsaPrivateKey *priv;

  fi = get_file_information (directory_scan_result);
  GNUNET_FS_share_tree_free (directory_scan_result);
  if (NULL == fi)
  {
    fprintf (stderr, "%s", _ ("Could not publish\n"));
    ret = 1;
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  GNUNET_FS_file_information_inspect (fi, &publish_inspector, NULL);
  if (extract_only)
  {
    GNUNET_FS_file_information_destroy (fi, NULL, NULL);
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  if (NULL == namespace)
    priv = NULL;
  else
    priv = GNUNET_IDENTITY_ego_get_private_key (namespace);
  pc = GNUNET_FS_publish_start (ctx,
                                fi,
                                priv,
                                this_id,
                                next_id,
                                (do_simulate)
                                ? GNUNET_FS_PUBLISH_OPTION_SIMULATE_ONLY
                                : GNUNET_FS_PUBLISH_OPTION_NONE);
  if (NULL == pc)
  {
    fprintf (stderr, "%s", _ ("Could not start publishing.\n"));
    ret = 1;
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
}


static void
directory_scan_cb (void *cls,
                   const char *filename,
                   int is_directory,
                   enum GNUNET_FS_DirScannerProgressUpdateReason reason)
{
  struct GNUNET_FS_ShareTreeItem *directory_scan_result;

  switch (reason)
  {
  case GNUNET_FS_DIRSCANNER_FILE_START:
    if (verbose > 1)
    {
      if (is_directory == GNUNET_YES)
        fprintf (stdout, _ ("Scanning directory `%s'.\n"), filename);
      else
        fprintf (stdout, _ ("Scanning file `%s'.\n"), filename);
    }
    break;

  case GNUNET_FS_DIRSCANNER_FILE_IGNORED:
    fprintf (stderr,
             _ ("There was trouble processing file `%s', skipping it.\n"),
             filename);
    break;

  case GNUNET_FS_DIRSCANNER_ALL_COUNTED:
    if (verbose)
      fprintf (stdout, "%s", _ ("Preprocessing complete.\n"));
    break;

  case GNUNET_FS_DIRSCANNER_EXTRACT_FINISHED:
    if (verbose > 2)
      fprintf (stdout,
               _ ("Extracting meta data from file `%s' complete.\n"),
               filename);
    break;

  case GNUNET_FS_DIRSCANNER_FINISHED:
    if (verbose > 1)
      fprintf (stdout, "%s", _ ("Meta data extraction has finished.\n"));
    directory_scan_result = GNUNET_FS_directory_scan_get_result (ds);
    ds = NULL;
    GNUNET_FS_share_tree_trim (directory_scan_result);
    directory_trim_complete (directory_scan_result);
    break;

  case GNUNET_FS_DIRSCANNER_INTERNAL_ERROR:
    fprintf (stdout, "%s", _ ("Error scanning directory.\n"));
    ret = 1;
    GNUNET_SCHEDULER_shutdown ();
    break;

  default:
    GNUNET_assert (0);
    break;
  }
  fflush (stdout);
}


static void
identity_continuation (const char *args0)
{
  char *ex;
  char *emsg;

  if ((NULL != pseudonym) && (NULL == namespace))
  {
    fprintf (stderr, _ ("Selected pseudonym `%s' unknown\n"), pseudonym);
    ret = 1;
    GNUNET_SCHEDULER_shutdown ();
    return;
  }
  if (NULL != uri_string)
  {
    emsg = NULL;
    if (NULL == (uri = GNUNET_FS_uri_parse (uri_string, &emsg)))
    {
      fprintf (stderr, _ ("Failed to parse URI: %s\n"), emsg);
      GNUNET_free (emsg);
      ret = 1;
      GNUNET_SCHEDULER_shutdown ();
      return;
    }
    GNUNET_FS_publish_ksk (ctx,
                           topKeywords,
                           meta,
                           uri,
                           &bo,
                           GNUNET_FS_PUBLISH_OPTION_NONE,
                           &uri_ksk_continuation,
                           NULL);
    return;
  }
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_string (cfg, "FS", "EXTRACTORS", &ex))
    ex = NULL;
  if (0 != access (args0, R_OK))
  {
    fprintf (stderr,
             _ ("Failed to access `%s': %s\n"),
             args0,
             strerror (errno));
    GNUNET_free (ex);
    return;
  }
  ds = GNUNET_FS_directory_scan_start (args0,
                                       disable_extractor,
                                       ex,
                                       &directory_scan_cb,
                                       NULL);
  if (NULL == ds)
  {
    fprintf (
      stderr,
      "%s",
      _ (
        "Failed to start meta directory scanner.  Is gnunet-helper-publish-fs installed?\n"));
    GNUNET_free (ex);
    return;
  }
  GNUNET_free (ex);
}


static void
identity_cb (void *cls,
             struct GNUNET_IDENTITY_Ego *ego,
             void **ctx,
             const char *name)
{
  const char *args0 = cls;

  if (NULL == ego)
  {
    identity_continuation (args0);
    return;
  }
  if (NULL == name)
    return;
  if (0 == strcmp (name, pseudonym))
    namespace = ego;
}


static void
run (void *cls,
     char *const *args,
     const char *cfgfile,
     const struct GNUNET_CONFIGURATION_Handle *c)
{
  /* check arguments */
  if ((NULL != uri_string) && (extract_only))
  {
    printf (_ ("Cannot extract metadata from a URI!\n"));
    ret = -1;
    return;
  }
  if (((NULL == uri_string) || (extract_only)) &&
      ((NULL == args[0]) || (NULL != args[1])))
  {
    printf (_ ("You must specify one and only one filename for insertion.\n"));
    ret = -1;
    return;
  }
  if ((NULL != uri_string) && (NULL != args[0]))
  {
    printf (_ ("You must NOT specify an URI and a filename.\n"));
    ret = -1;
    return;
  }
  if (NULL != pseudonym)
  {
    if (NULL == this_id)
    {
      fprintf (stderr,
               _ ("Option `%s' is required when using option `%s'.\n"),
               "-t",
               "-P");
      ret = -1;
      return;
    }
  }
  else
  {   /* ordinary insertion checks */
    if (NULL != next_id)
    {
      fprintf (stderr,
               _ ("Option `%s' makes no sense without option `%s'.\n"),
               "-N",
               "-P");
      ret = -1;
      return;
    }
    if (NULL != this_id)
    {
      fprintf (stderr,
               _ ("Option `%s' makes no sense without option `%s'.\n"),
               "-t",
               "-P");
      ret = -1;
      return;
    }
  }
  cfg = c;
  ctx = GNUNET_FS_start (cfg,
                         "gnunet-publish",
                         &progress_cb,
                         NULL,
                         GNUNET_FS_FLAGS_NONE,
                         GNUNET_FS_OPTIONS_END);
  if (NULL == ctx)
  {
    fprintf (stderr, _ ("Could not initialize `%s' subsystem.\n"), "FS");
    ret = 1;
    return;
  }
  GNUNET_SCHEDULER_add_shutdown (&do_stop_task, NULL);
  if (NULL != pseudonym)
    identity = GNUNET_IDENTITY_connect (cfg, &identity_cb, args[0]);
  else
    identity_continuation (args[0]);
}


int
main (int argc, char *const *argv)
{
  struct GNUNET_GETOPT_CommandLineOption options[] =
  { GNUNET_GETOPT_option_uint ('a',
                               "anonymity",
                               "LEVEL",
                               gettext_noop (
                                 "set the desired LEVEL of sender-anonymity"),
                               &bo.anonymity_level),
    GNUNET_GETOPT_option_flag (
      'D',
      "disable-extractor",
      gettext_noop ("do not use libextractor to add keywords or metadata"),
      &disable_extractor),
    GNUNET_GETOPT_option_flag ('E',
                               "enable-creation-time",
                               gettext_noop (
                                 "enable adding the creation time to the "
                                 "metadata of the uploaded file"),
                               &enable_creation_time),
    GNUNET_GETOPT_option_flag ('e',
                               "extract",
                               gettext_noop (
                                 "print list of extracted keywords that would "
                                 "be used, but do not perform upload"),
                               &extract_only),
    GNUNET_FS_GETOPT_KEYWORDS (
      'k',
      "key",
      "KEYWORD",
      gettext_noop (
        "add an additional keyword for the top-level "
        "file or directory (this option can be specified multiple times)"),
      &topKeywords),
    GNUNET_FS_GETOPT_METADATA (
      'm',
      "meta",
      "TYPE:VALUE",
      gettext_noop ("set the meta-data for the given TYPE to the given VALUE"),
      &meta),
    GNUNET_GETOPT_option_flag (
      'n',
      "noindex",
      gettext_noop ("do not index, perform full insertion (stores "
                    "entire file in encrypted form in GNUnet database)"),
      &do_insert),
    GNUNET_GETOPT_option_string (
      'N',
      "next",
      "ID",
      gettext_noop ("specify ID of an updated version to be "
                    "published in the future (for namespace insertions only)"),
      &next_id),
    GNUNET_GETOPT_option_uint ('p',
                               "priority",
                               "PRIORITY",
                               gettext_noop (
                                 "specify the priority of the content"),
                               &bo.content_priority),
    GNUNET_GETOPT_option_string ('P',
                                 "pseudonym",
                                 "NAME",
                                 gettext_noop (
                                   "publish the files under the pseudonym "
                                   "NAME (place file into namespace)"),
                                 &pseudonym),
    GNUNET_GETOPT_option_uint ('r',
                               "replication",
                               "LEVEL",
                               gettext_noop (
                                 "set the desired replication LEVEL"),
                               &bo.replication_level),
    GNUNET_GETOPT_option_flag ('s',
                               "simulate-only",
                               gettext_noop (
                                 "only simulate the process but do not do "
                                 "any actual publishing (useful to compute URIs)"),
                               &do_simulate),
    GNUNET_GETOPT_option_string ('t',
                                 "this",
                                 "ID",
                                 gettext_noop (
                                   "set the ID of this version of the publication "
                                   "(for namespace insertions only)"),
                                 &this_id),
    GNUNET_GETOPT_option_string (
      'u',
      "uri",
      "URI",
      gettext_noop (
        "URI to be published (can be used instead of passing a "
        "file to add keywords to the file with the respective URI)"),
      &uri_string),

    GNUNET_GETOPT_option_verbose (&verbose),

    GNUNET_GETOPT_OPTION_END };

  bo.expiration_time =
    GNUNET_TIME_year_to_time (GNUNET_TIME_get_current_year () + 2);

  if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
    return 2;
  ret =
    (GNUNET_OK ==
     GNUNET_PROGRAM_run (argc,
                         argv,
                         "gnunet-publish [OPTIONS] FILENAME",
                         gettext_noop ("Publish a file or directory on GNUnet"),
                         options,
                         &run,
                         NULL))
    ? ret
    : 1;
  GNUNET_free_nz ((void *) argv);
  return ret;
}


/* end of gnunet-publish.c */