hcache.c

Go to the documentation of this file.

#include "config.h"
#include <assert.h>
#include <errno.h>
#include <limits.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <unistd.h>
#include "mutt/lib.h"
#include "config/lib.h"
#include "email/lib.h"
#include "core/lib.h"
#include "lib.h"
#include "compress/lib.h"
#include "store/lib.h"
#include "hcache/hcversion.h"
#include "muttlib.h"
#include "serialize.h"

#if !(defined(HAVE_BDB) || defined(HAVE_GDBM) || defined(HAVE_KC) ||           \
      defined(HAVE_LMDB) || defined(HAVE_QDBM) || defined(HAVE_ROCKSDB) ||     \
      defined(HAVE_TC) || defined(HAVE_TDB))
#error "No hcache backend defined"
#endif

static unsigned int hcachever = 0x0;

static size_t header_size(void)
{
  return sizeof(int) + sizeof(uint32_t);
}

static void *dump(struct HeaderCache *hc, const struct Email *e, int *off, uint32_t uidvalidity)
{
  struct Email e_dump;
  bool convert = !CharsetIsUtf8;

  *off = 0;
  unsigned char *d = mutt_mem_malloc(4096);

  d = serial_dump_uint32_t((uidvalidity != 0) ? uidvalidity : mutt_date_epoch(), d, off);
  d = serial_dump_int(hc->crc, d, off);

  assert((size_t) *off == header_size());

  lazy_realloc(&d, *off + sizeof(struct Email));
  memcpy(&e_dump, e, sizeof(struct Email));

  /* some fields are not safe to cache */
  e_dump.tagged = false;
  e_dump.changed = false;
  e_dump.threaded = false;
  e_dump.recip_valid = false;
  e_dump.searched = false;
  e_dump.matched = false;
  e_dump.collapsed = false;
  e_dump.visible = true;
  e_dump.num_hidden = 0;
  e_dump.recipient = 0;
  e_dump.pair = 0;
  e_dump.attach_valid = false;
  e_dump.path = NULL;
  e_dump.tree = NULL;
  e_dump.thread = NULL;
  e_dump.sequence = 0;
  e_dump.notify = NULL;
  STAILQ_INIT(&e_dump.tags);
#ifdef MIXMASTER
  STAILQ_INIT(&e_dump.chain);
#endif
  e_dump.edata = NULL;

  memcpy(d + *off, &e_dump, sizeof(struct Email));
  *off += sizeof(struct Email);

  d = serial_dump_envelope(e_dump.env, d, off, convert);
  d = serial_dump_body(e_dump.body, d, off, convert);
  d = serial_dump_tags(&e->tags, d, off);

  return d;
}

static struct Email *restore(const unsigned char *d)
{
  int off = 0;
  struct Email *e = email_new();
  bool convert = !CharsetIsUtf8;

  /* skip validate */
  off += sizeof(uint32_t);

  /* skip crc */
  off += sizeof(unsigned int);

  size_t sequence = e->sequence;
  struct Notify *notify = e->notify;
  memcpy(e, d + off, sizeof(struct Email));
  off += sizeof(struct Email);
  e->sequence = sequence;
  e->notify = notify;

  STAILQ_INIT(&e->tags);
#ifdef MIXMASTER
  STAILQ_INIT(&e->chain);
#endif

  e->env = mutt_env_new();
  serial_restore_envelope(e->env, d, &off, convert);

  e->body = mutt_body_new();
  serial_restore_body(e->body, d, &off, convert);
  serial_restore_tags(&e->tags, d, &off);

  return e;
}

struct RealKey
{
  char key[1024];
  size_t len;
};

static struct RealKey *realkey(const char *key, size_t keylen)
{
  static struct RealKey rk;
#ifdef USE_HCACHE_COMPRESSION
  const char *const c_header_cache_compress_method =
      cs_subset_string(NeoMutt->sub, "header_cache_compress_method");
  if (c_header_cache_compress_method)
  {
    const struct ComprOps *cops = compress_get_ops(c_header_cache_compress_method);

    rk.len = snprintf(rk.key, sizeof(rk.key), "%s-%s", key, cops->name);
  }
  else
#endif
  {
    memcpy(rk.key, key, keylen + 1); // Including NUL byte
    rk.len = keylen;
  }
  return &rk;
}

static bool create_hcache_dir(const char *path)
{
  char *dir = mutt_str_dup(path);
  if (!dir)
    return false;

  char *p = strrchr(dir, '/');
  if (!p)
  {
    FREE(&dir);
    return true;
  }

  *p = '\0';

  int rc = mutt_file_mkdir(dir, S_IRWXU | S_IRWXG | S_IRWXO);
  if (rc != 0)
    mutt_error(_("Can't create %s: %s"), dir, strerror(errno));

  FREE(&dir);
  return (rc == 0);
}

static void hcache_per_folder(struct Buffer *hcpath, const char *path,
                              const char *folder, hcache_namer_t namer)
{
  struct stat sb;

  int plen = mutt_str_len(path);
  int rc = stat(path, &sb);
  bool slash = (path[plen - 1] == '/');

  if (((rc == 0) && !S_ISDIR(sb.st_mode)) || ((rc == -1) && !slash))
  {
    /* An existing file or a non-existing path not ending with a slash */
    mutt_encode_path(hcpath, path);
    create_hcache_dir(mutt_buffer_string(hcpath));
    return;
  }

  /* We have a directory - no matter whether it exists, or not */
  struct Buffer *hcfile = mutt_buffer_pool_get();
  if (namer)
  {
    namer(folder, hcfile);
    mutt_buffer_concat_path(hcpath, path, mutt_buffer_string(hcfile));
  }
  else
  {
    unsigned char m[16]; /* binary md5sum */
    struct Buffer *name = mutt_buffer_pool_get();

    const char *const c_header_cache_backend =
        cs_subset_string(NeoMutt->sub, "header_cache_backend");
    const struct StoreOps *ops = store_get_backend_ops(c_header_cache_backend);

#ifdef USE_HCACHE_COMPRESSION
    const char *const c_header_cache_compress_method =
        cs_subset_string(NeoMutt->sub, "header_cache_compress_method");
    const char *cm = c_header_cache_compress_method;
    mutt_buffer_printf(name, "%s|%s%s", ops->name, folder, cm ? cm : "");
#else
    mutt_buffer_printf(name, "%s|%s", ops->name, folder);
#endif
    mutt_md5(mutt_buffer_string(name), m);
    mutt_buffer_reset(name);
    mutt_md5_toascii(m, name->data);
    mutt_buffer_printf(hcpath, "%s%s%s", path, slash ? "" : "/", mutt_buffer_string(name));
    mutt_buffer_pool_release(&name);
  }

  mutt_encode_path(hcpath, mutt_buffer_string(hcpath));
  create_hcache_dir(mutt_buffer_string(hcpath));
  mutt_buffer_pool_release(&hcfile);
}

static char *get_foldername(const char *folder)
{
  /* if the folder is local, canonify the path to avoid
   * to ensure equivalent paths share the hcache */
  char *p = mutt_mem_malloc(PATH_MAX + 1);
  if (!realpath(folder, p))
    mutt_str_replace(&p, folder);

  return p;
}

struct HeaderCache *mutt_hcache_open(const char *path, const char *folder, hcache_namer_t namer)
{
  const char *const c_header_cache_backend =
      cs_subset_string(NeoMutt->sub, "header_cache_backend");
  const struct StoreOps *ops = store_get_backend_ops(c_header_cache_backend);
  if (!ops)
    return NULL;

  struct HeaderCache *hc = mutt_mem_calloc(1, sizeof(struct HeaderCache));

  /* Calculate the current hcache version from dynamic configuration */
  if (hcachever == 0x0)
  {
    union
    {
      unsigned char charval[16];
      unsigned int intval;
    } digest;
    struct Md5Ctx md5ctx;

    hcachever = HCACHEVER;

    mutt_md5_init_ctx(&md5ctx);

    /* Seed with the compiled-in header structure hash */
    mutt_md5_process_bytes(&hcachever, sizeof(hcachever), &md5ctx);

    /* Mix in user's spam list */
    struct Replace *sp = NULL;
    STAILQ_FOREACH(sp, &SpamList, entries)
    {
      mutt_md5_process(sp->regex->pattern, &md5ctx);
      mutt_md5_process(sp->templ, &md5ctx);
    }

    /* Mix in user's nospam list */
    struct RegexNode *np = NULL;
    STAILQ_FOREACH(np, &NoSpamList, entries)
    {
      mutt_md5_process(np->regex->pattern, &md5ctx);
    }

    /* Get a hash and take its bytes as an (unsigned int) hash version */
    mutt_md5_finish_ctx(&md5ctx, digest.charval);
    hcachever = digest.intval;
  }

#ifdef USE_HCACHE_COMPRESSION
  const char *const c_header_cache_compress_method =
      cs_subset_string(NeoMutt->sub, "header_cache_compress_method");
  if (c_header_cache_compress_method)
  {
    const struct ComprOps *cops = compress_get_ops(c_header_cache_compress_method);

    const short c_header_cache_compress_level =
        cs_subset_number(NeoMutt->sub, "header_cache_compress_level");
    hc->cctx = cops->open(c_header_cache_compress_level);
    if (!hc->cctx)
    {
      FREE(&hc);
      return NULL;
    }

    /* remember the buffer of database backend */
    mutt_debug(LL_DEBUG3, "Header cache will use %s compression\n", cops->name);
  }
#endif

  hc->folder = get_foldername(folder);
  hc->crc = hcachever;

  if (!path || (path[0] == '\0'))
  {
    FREE(&hc->folder);
    FREE(&hc);
    return NULL;
  }

  struct Buffer *hcpath = mutt_buffer_pool_get();
  hcache_per_folder(hcpath, path, hc->folder, namer);

  hc->ctx = ops->open(mutt_buffer_string(hcpath));
  if (!hc->ctx)
  {
    /* remove a possibly incompatible version */
    if (unlink(mutt_buffer_string(hcpath)) == 0)
    {
      hc->ctx = ops->open(mutt_buffer_string(hcpath));
      if (!hc->ctx)
      {
        FREE(&hc->folder);
        FREE(&hc);
      }
    }
  }

  mutt_buffer_pool_release(&hcpath);
  return hc;
}

void mutt_hcache_close(struct HeaderCache *hc)
{
  const char *const c_header_cache_backend =
      cs_subset_string(NeoMutt->sub, "header_cache_backend");
  const struct StoreOps *ops = store_get_backend_ops(c_header_cache_backend);
  if (!hc || !ops)
    return;

#ifdef USE_HCACHE_COMPRESSION
  const char *const c_header_cache_compress_method =
      cs_subset_string(NeoMutt->sub, "header_cache_compress_method");
  if (c_header_cache_compress_method)
  {
    const struct ComprOps *cops = compress_get_ops(c_header_cache_compress_method);
    cops->close(&hc->cctx);
  }
#endif

  ops->close(&hc->ctx);
  FREE(&hc->folder);
  FREE(&hc);
}

struct HCacheEntry mutt_hcache_fetch(struct HeaderCache *hc, const char *key,
                                     size_t keylen, uint32_t uidvalidity)
{
  struct RealKey *rk = realkey(key, keylen);
  struct HCacheEntry entry = { 0 };

  size_t dlen;
  void *data = mutt_hcache_fetch_raw(hc, rk->key, rk->len, &dlen);
  void *to_free = data;
  if (!data)
  {
    goto end;
  }

  /* restore uidvalidity and crc */
  size_t hlen = header_size();
  int off = 0;
  serial_restore_uint32_t(&entry.uidvalidity, data, &off);
  serial_restore_int(&entry.crc, data, &off);
  assert((size_t) off == hlen);
  if (entry.crc != hc->crc || ((uidvalidity != 0) && uidvalidity != entry.uidvalidity))
  {
    goto end;
  }

#ifdef USE_HCACHE_COMPRESSION
  const char *const c_header_cache_compress_method =
      cs_subset_string(NeoMutt->sub, "header_cache_compress_method");
  if (c_header_cache_compress_method)
  {
    const struct ComprOps *cops = compress_get_ops(c_header_cache_compress_method);

    void *dblob = cops->decompress(hc->cctx, (char *) data + hlen, dlen - hlen);
    if (!dblob)
    {
      goto end;
    }
    data = (char *) dblob - hlen; /* restore skips uidvalidity and crc */
  }
#endif

  entry.email = restore(data);

end:
  mutt_hcache_free_raw(hc, &to_free);
  return entry;
}

void *mutt_hcache_fetch_raw(struct HeaderCache *hc, const char *key,
                            size_t keylen, size_t *dlen)
{
  const char *const c_header_cache_backend =
      cs_subset_string(NeoMutt->sub, "header_cache_backend");
  const struct StoreOps *ops = store_get_backend_ops(c_header_cache_backend);

  if (!hc || !ops)
    return NULL;

  struct Buffer path = mutt_buffer_make(1024);
  keylen = mutt_buffer_printf(&path, "%s%.*s", hc->folder, (int) keylen, key);
  void *blob = ops->fetch(hc->ctx, mutt_buffer_string(&path), keylen, dlen);
  mutt_buffer_dealloc(&path);
  return blob;
}

void mutt_hcache_free_raw(struct HeaderCache *hc, void **data)
{
  const char *const c_header_cache_backend =
      cs_subset_string(NeoMutt->sub, "header_cache_backend");
  const struct StoreOps *ops = store_get_backend_ops(c_header_cache_backend);

  if (!hc || !ops || !data || !*data)
    return;

  ops->free(hc->ctx, data);
}

int mutt_hcache_store(struct HeaderCache *hc, const char *key, size_t keylen,
                      struct Email *e, uint32_t uidvalidity)
{
  if (!hc)
    return -1;

  int dlen = 0;
  char *data = dump(hc, e, &dlen, uidvalidity);

#ifdef USE_HCACHE_COMPRESSION
  const char *const c_header_cache_compress_method =
      cs_subset_string(NeoMutt->sub, "header_cache_compress_method");
  if (c_header_cache_compress_method)
  {
    /* We don't compress uidvalidity and the crc, so we can check them before
     * decompressing on fetch().  */
    size_t hlen = header_size();

    const struct ComprOps *cops = compress_get_ops(c_header_cache_compress_method);

    /* data / dlen gets ptr to compressed data here */
    size_t clen = dlen;
    void *cdata = cops->compress(hc->cctx, data + hlen, dlen - hlen, &clen);
    if (!cdata)
    {
      FREE(&data);
      return -1;
    }

    char *whole = mutt_mem_malloc(hlen + clen);
    memcpy(whole, data, hlen);
    memcpy(whole + hlen, cdata, clen);

    FREE(&data);

    data = whole;
    dlen = hlen + clen;
  }
#endif

  /* store uncompressed data */
  struct RealKey *rk = realkey(key, keylen);
  int rc = mutt_hcache_store_raw(hc, rk->key, rk->len, data, dlen);

  FREE(&data);

  return rc;
}

int mutt_hcache_store_raw(struct HeaderCache *hc, const char *key,
                          size_t keylen, void *data, size_t dlen)
{
  const char *const c_header_cache_backend =
      cs_subset_string(NeoMutt->sub, "header_cache_backend");
  const struct StoreOps *ops = store_get_backend_ops(c_header_cache_backend);

  if (!hc || !ops)
    return -1;

  struct Buffer path = mutt_buffer_make(1024);

  keylen = mutt_buffer_printf(&path, "%s%.*s", hc->folder, (int) keylen, key);
  int rc = ops->store(hc->ctx, mutt_buffer_string(&path), keylen, data, dlen);
  mutt_buffer_dealloc(&path);

  return rc;
}

int mutt_hcache_delete_record(struct HeaderCache *hc, const char *key, size_t keylen)
{
  if (!hc)
    return -1;

  const char *const c_header_cache_backend =
      cs_subset_string(NeoMutt->sub, "header_cache_backend");
  const struct StoreOps *ops = store_get_backend_ops(c_header_cache_backend);

  struct Buffer path = mutt_buffer_make(1024);

  keylen = mutt_buffer_printf(&path, "%s%s", hc->folder, key);

  int rc = ops->delete_record(hc->ctx, mutt_buffer_string(&path), keylen);
  mutt_buffer_dealloc(&path);
  return rc;
}