ok = false;
goto bail_out; /* fatal error */
}
+ if (dcr->despooling && dev->device->block_encryption!=ET_NONE && dev->crypto_device_ctx!=NULL) {
+// DEVICE *old = block->dev;
+ block->dev = dev;
+// uint64_t checksum = ser_block_header(block, dev->do_checksum());
+// block->dev = old;
+ }
Dmsg1(500, "Write block to dev=%p\n", dcr->dev);
if (!write_block_to_dev()) {
uint32_t wlen; /* length to write */
bool ok = true;
DCR *dcr = this;
- uint32_t checksum;
+ uint64_t checksum;
uint32_t pad; /* padding or zeros written */
boffset_t pos;
char ed1[50];
bmicrosleep(5, 0); /* pause a bit if busy or lots of errors */
dev->clrerror(-1);
}
- stat = dev->write(block->buf, (size_t)wlen);
+ stat = dev->write(block->buf_out, (size_t)wlen);
+
Dmsg4(100, "%s write() BlockAddr=%lld wlen=%d Vol=%s\n",
- block->adata?"Adata":"Ameta",
- block->BlockAddr, wlen, dev->VolHdr.VolumeName);
+ block->adata?"Adata":"Ameta", block->BlockAddr, wlen,
+ dev->VolHdr.VolumeName);
} while (stat == -1 && (errno == EBUSY || errno == EIO) && retry++ < 3);
/* ***FIXME*** remove 2 lines debug */
}
}
- if (stat != (ssize_t)wlen) {
+ if (stat == (ssize_t)wlen) {
+ if (block->first_block)
+ { // It was a volume label, it has been successfully written
+ // next block should not hold a volume_label and should
+ // be encrypted if encryption is enable
+ block->first_block = false;
+ }
+ } else {
/* Some devices simply report EIO when the volume is full.
* With a little more thought we may be able to check
* capacity and distinguish real errors and EOT
/* Block Header definitions. */
#define BLKHDR1_ID "BB01"
#define BLKHDR2_ID "BB02"
+#define BLKHDR3_ID "BB03"
#define BLKHDR_ID_LENGTH 4
#define BLKHDR_CS_LENGTH 4 /* checksum length */
#define BLKHDR1_LENGTH 16 /* Total length */
#define BLKHDR2_LENGTH 24 /* Total length */
+#define BLKHDR3_LENGTH 32 /* Total length including the checksum) */
+#define BLKHDR_CS64_OFFSET 24 /* 64bits checksum offset */
+#define BLKHDR_CS64_LENGTH 8 /* 64bits checksum length */
-#define WRITE_BLKHDR_ID BLKHDR2_ID
-#define WRITE_BLKHDR_LENGTH BLKHDR2_LENGTH
+#define WRITE_BLKHDR_ID BLKHDR3_ID
+#define WRITE_BLKHDR_LENGTH BLKHDR3_LENGTH
#define WRITE_ADATA_BLKHDR_LENGTH (6*sizeof(int32_t)+sizeof(uint64_t))
-#define BLOCK_VER 2
+#define BLOCK_VER 3
+
+enum BLKH_OPTIONS {
+ BLKHOPT_NONE = 0x0,
+ BLKHOPT_CHKSUM = 0x1, // set if the checksum is enable
+ BLKHOPT_ENCRYPT_VOL = 0x2, // set if the volume is encrypted
+ BLKHOPT_ENCRYPT_BLOCK = 0x4, // set if this block is encrypted (volume label are not)
+
+ BLKHOPT_MASK = 0x7, // Update this one after adding some values above
+ BLKHOPT_LAST = 0x8, // Update this one too
+};
/* Record header definitions */
#define RECHDR1_LENGTH 20
* uint32_t data_length
*/
#define RECHDR2_LENGTH (3*sizeof(int32_t))
+#define RECHDR3_LENGTH RECHDR2_LENGTH
#define WRITE_RECHDR_LENGTH RECHDR2_LENGTH
/*
uint32_t VolSessionTime;
uint64_t BlockAddr;
+ * for BB03 block, we have
+ variable header length, minimum is 24 bytes
+
+ uint32_t blkh_options; // bit field describing the extra fields
+ uint32_t block_len;
+ uint32_t BlockNumber;
+ char Id[BLKHDR_ID_LENGTH];
+ uint32_t VolSessionId;
+ uint32_t VolSessionTime;
+ ---- no come extra field ----
+ uint64_t xxhash64; if (blkh_options&BLKHOPT_CHKSUM)
+
*/
class DEVICE; /* for forward reference */
uint32_t VolSessionId; /* */
uint32_t VolSessionTime; /* */
uint32_t read_errors; /* block errors (checksum, header, ...) */
- uint32_t CheckSum; /* Block checksum */
+ uint64_t CheckSum64; /* Block checksum */
uint32_t RecNum; /* Number of records read from the current block */
uint32_t extra_bytes; /* the extra size that must be accounted in VolABytes */
int BlockVer; /* block version 1 or 2 */
bool adata; /* adata block */
bool no_header; /* Set if no block header */
bool new_fi; /* New FI arrived */
+ bool first_block; /* Is the block holding a volume label */
int32_t FirstIndex; /* first index this block */
int32_t LastIndex; /* last index this block */
int32_t rechdr_items; /* number of items in rechdr queue */
char *bufp; /* pointer into buffer */
POOLMEM *rechdr_queue; /* record header queue */
POOLMEM *buf; /* actual data buffer */
+ POOLMEM *buf_enc; /* buffer with the encrypted data */
+ char *buf_out; /* point to the data that must be written to the media */
alist *filemedia; /* Filemedia attached to the current block */
+ uint32_t blkh_options; /* see BLKH_OPTIONS */
};
#define block_is_empty(block) ((block)->read_len == 0)
#include "bacula.h"
#include "stored.h"
+#include "lib/xxhash.h"
static const int dbglvl = 160;
char *p;
char *bufp;
char Id[BLKHDR_ID_LENGTH+1];
- uint32_t CheckSum, BlockCheckSum;
+ uint64_t CheckSum64, BlockCheckSum64;
+ uint32_t CheckSumLo;
uint32_t block_len, reclen;
uint32_t BlockNumber;
uint32_t VolSessionId, VolSessionTime, data_len;
}
}
unser_begin(b->buf, BLKHDR1_LENGTH);
- unser_uint32(CheckSum);
+ unser_uint32(CheckSumLo);
unser_uint32(block_len);
unser_uint32(BlockNumber);
unser_bytes(Id, BLKHDR_ID_LENGTH);
ASSERT(unser_length(b->buf) == BLKHDR1_LENGTH);
Id[BLKHDR_ID_LENGTH] = 0;
- if (Id[3] == '2') {
+
+ if (block_len > 4000000 || block_len < BLKHDR_CS_LENGTH) {
+ Dmsg3(20, "Will not dump blocksize too %s %lu msg: %s\n",
+ (block_len < BLKHDR_CS_LENGTH)?"small":"big",
+ block_len, msg);
+ return;
+ }
+
+ if (Id[3] == '3') {
+ // CheckSumLo is the header option field
+ unser_uint32(VolSessionId);
+ unser_uint32(VolSessionTime);
+ unser_uint64(CheckSum64);
+ bhl = BLKHDR3_LENGTH;
+ rhl = RECHDR3_LENGTH;
+ /* backup the CheckSum64, replace it with 0x0...0 and calculate the CheckSum64 */
+ uint64_t save;
+ memcpy(&save, b->buf+BLKHDR_CS64_OFFSET, BLKHDR_CS64_LENGTH);
+ memset(b->buf+BLKHDR_CS64_OFFSET, '\0', BLKHDR_CS64_LENGTH);
+ BlockCheckSum64 = XXH3_64bits((uint8_t *)b->buf+BLKHDR_CS_LENGTH,
+ block_len-BLKHDR_CS_LENGTH);
+ /* restore the checksum */
+ memcpy(b->buf+BLKHDR_CS64_OFFSET, &save, BLKHDR_CS64_LENGTH);
+ } else if (Id[3] == '2') {
unser_uint32(VolSessionId);
unser_uint32(VolSessionTime);
bhl = BLKHDR2_LENGTH;
rhl = RECHDR2_LENGTH;
+ CheckSum64 = CheckSumLo;
+ BlockCheckSum64 = bcrc32((uint8_t *)b->buf+BLKHDR_CS_LENGTH,
+ block_len-BLKHDR_CS_LENGTH);
} else {
VolSessionId = VolSessionTime = 0;
bhl = BLKHDR1_LENGTH;
rhl = RECHDR1_LENGTH;
+ CheckSum64 = CheckSumLo;
+ BlockCheckSum64 = bcrc32((uint8_t *)b->buf+BLKHDR_CS_LENGTH,
+ block_len-BLKHDR_CS_LENGTH);
}
- if (block_len > 4000000 || block_len < BLKHDR_CS_LENGTH) {
- Dmsg3(20, "Will not dump blocksize too %s %lu msg: %s\n",
- (block_len < BLKHDR_CS_LENGTH)?"small":"big",
- block_len, msg);
- return;
- }
-
- BlockCheckSum = bcrc32((uint8_t *)b->buf+BLKHDR_CS_LENGTH,
- block_len-BLKHDR_CS_LENGTH);
Pmsg7(000, _("Dump block %s %p: adata=%d size=%d BlkNum=%d\n"
-" Hdrcksum=%x cksum=%x\n"),
- msg, b, b->adata, block_len, BlockNumber, CheckSum, BlockCheckSum);
+" Hdrcksum=%llx cksum=%llx\n"),
+ msg, b, b->adata, block_len, BlockNumber, CheckSum64, BlockCheckSum64);
p = b->buf + bhl;
while (p < bufp) {
unser_begin(p, WRITE_RECHDR_LENGTH);
}
block->buf_len = len;
block->buf = get_memory(block->buf_len);
+ block->buf_enc = get_memory(block->buf_len);
block->rechdr_queue = get_memory(block->buf_len);
block->rechdr_items = 0;
Dmsg2(510, "Rechdr len=%d max_items=%d\n", sizeof_pool_memory(block->rechdr_queue),
sizeof_pool_memory(block->rechdr_queue)/WRITE_ADATA_RECHDR_LENGTH);
block->filemedia = New(alist(1, owned_by_alist));
+ block->blkh_options = 0;
+ block->blkh_options |= do_checksum() ? BLKHOPT_CHKSUM : 0;
empty_block(block);
block->BlockVer = BLOCK_VER; /* default write version */
Dmsg3(150, "New block adata=%d len=%d block=%p\n", block->adata, len, block);
memcpy(block, eblock, sizeof(DEV_BLOCK));
block->buf = get_memory(buf_len);
+ block->buf_enc = get_memory(buf_len);
+ block->buf_out = (eblock->buf_out==eblock->buf)?block->buf:eblock->buf_enc;
+
memcpy(block->buf, eblock->buf, buf_len);
+ memcpy(block->buf_enc, eblock->buf_enc, buf_len);
block->rechdr_queue = get_memory(rechdr_len);
memcpy(block->rechdr_queue, eblock->rechdr_queue, rechdr_len);
if (block->buf) {
free_memory(block->buf);
}
+ if (block->buf_enc) {
+ free_memory(block->buf_enc);
+ }
if (block->rechdr_queue) {
free_memory(block->rechdr_queue);
}
Dmsg3(250, "empty_block: adata=%d len=%d set binbuf=%d\n",
block->adata, block->buf_len, block->binbuf);
block->bufp = block->buf + block->binbuf;
+ block->buf_out = block->buf;
block->read_len = 0;
block->write_failed = false;
block->block_read = false;
block->BlockAddr = 0;
block->filemedia->destroy();
block->extra_bytes = 0;
+ block->first_block = false; // by default this block don't hold a volume label
}
/*
* in the buffer should have already been reserved by
* init_block.
*/
-uint32_t ser_block_header(DEV_BLOCK *block, bool do_checksum)
+uint64_t ser_block_header(DEV_BLOCK *block, bool do_checksum)
{
+ DEVICE *dev = block->dev;
ser_declare;
uint32_t block_len = block->binbuf;
-
- block->CheckSum = 0;
+ uint32_t hdr_option = 0x0;
+ bool do_encrypt_vol = dev->device->block_encryption!=ET_NONE && dev->crypto_device_ctx!=NULL;
+ bool do_encrypt_block = do_encrypt_vol && !block->first_block;
+ hdr_option |= (do_checksum?BLKHOPT_CHKSUM:0) |
+ (do_encrypt_vol?BLKHOPT_ENCRYPT_VOL:0) |
+ (do_encrypt_block?BLKHOPT_ENCRYPT_BLOCK:0);
+ block->CheckSum64 = 0;
/* ***BEEF*** */
if (block->adata) {
/* Checksum whole block */
if (do_checksum) {
- block->CheckSum = bcrc32((uint8_t *)block->buf, block_len);
+ block->CheckSum64 = bcrc32((uint8_t *)block->buf, block_len);
}
} else {
+ int chk_off = BLKHDR_CS64_OFFSET;
+ int bhl = BLKHDR3_LENGTH;
Dmsg1(160, "block_header: block_len=%d\n", block_len);
- ser_begin(block->buf, BLKHDR2_LENGTH);
- ser_uint32(block->CheckSum);
+ ser_begin(block->buf, BLKHDR3_LENGTH);
+ ser_uint32(hdr_option); /* Was the crc32, is now an "option" bit field in BB03 */
ser_uint32(block_len);
ser_uint32(block->BlockNumber);
- ser_bytes(WRITE_BLKHDR_ID, BLKHDR_ID_LENGTH);
- if (BLOCK_VER >= 2) {
- ser_uint32(block->VolSessionId);
- ser_uint32(block->VolSessionTime);
- }
-
- /* Checksum whole block except for the checksum */
+ ser_bytes(WRITE_BLKHDR_ID, BLKHDR_ID_LENGTH); // BB03
+ // BLOCK_VER >= 2
+ ser_uint32(block->VolSessionId);
+ ser_uint32(block->VolSessionTime);
+ // BLOCK_VER >= 3;
+// ASSERTD((do_checksum && (block->blkh_options&BLKHOPT_CHKSUM))
+// || (!do_checksum && !(block->blkh_options&BLKHOPT_CHKSUM)), "checksum configuration mismatch");
+ ser_uint64(0x00); // the XXH3_64 checksum
if (do_checksum) {
- block->CheckSum = bcrc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH,
- block_len-BLKHDR_CS_LENGTH);
+ /* Checksum whole block including the checksum with value 0x0 */
+ block->CheckSum64 = XXH3_64bits((uint8_t *)block->buf,
+ block_len);
+ /* update the checksum in the block header */
+ ser_begin(block->buf+chk_off, BLKHDR_CS64_LENGTH);
+ ser_uint64(block->CheckSum64);
+ }
+ Dmsg3(160, "ser_block_header: adata=%d checksum=0x%016llx enc=%d\n",
+ block->adata, block->CheckSum64, do_encrypt_block);
+ block->buf_out = block->buf;
+ if (do_encrypt_block) {
+ /* Does the encryption, checksum is encrypted too */
+ block_cipher_init_iv_header(block->dev->crypto_device_ctx, block->BlockNumber, block->VolSessionId, block->VolSessionTime);
+ block_cipher_encrypt(block->dev->crypto_device_ctx, block_len-bhl, block->buf+bhl, block->buf_enc+bhl);
+ memcpy(block->buf_enc, block->buf, bhl);
+ block->buf_out = block->buf_enc;
}
- Dmsg2(160, "ser_block_header: adata=%d checksum=%x\n", block->adata, block->CheckSum);
- ser_begin(block->buf, BLKHDR2_LENGTH);
- ser_uint32(block->CheckSum); /* now add checksum to block header */
}
- return block->CheckSum;
+ return block->CheckSum64;
}
/*
{
ser_declare;
char Id[BLKHDR_ID_LENGTH+1];
- uint32_t BlockCheckSum;
+ uint64_t BlockCheckSum64;
uint32_t block_len;
uint32_t block_end;
uint32_t BlockNumber;
+ uint32_t CheckSumLo;
JCR *jcr = dcr->jcr;
int bhl;
+ int chk_off = BLKHDR_CS64_OFFSET; /* used only in BB03 */
if (block->adata) {
/* Checksum the whole block */
if (block->block_len <= block->read_len && dev->do_checksum()) {
- BlockCheckSum = dcr->crc32((uint8_t *)block->buf, block->block_len, block->CheckSum);
- if (BlockCheckSum != block->CheckSum) {
+ BlockCheckSum64 = dcr->crc32((uint8_t *)block->buf, block->block_len, block->CheckSum64);
+ if (BlockCheckSum64 != block->CheckSum64) {
dev->dev_errno = EIO;
Mmsg5(dev->errmsg, _("Volume data error at %lld!\n"
- "Adata block checksum mismatch in block=%u len=%d: calc=%x blk=%x\n"),
+ "Adata block checksum mismatch in block=%u len=%d: calc=%llx blk=%llx\n"),
block->BlockAddr, block->BlockNumber,
- block->block_len, BlockCheckSum, block->CheckSum);
+ block->block_len, BlockCheckSum64, block->CheckSum64);
if (block->read_errors == 0 || verbose >= 2) {
Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
dump_block(dev, block, "with checksum error");
return true;
}
unser_begin(block->buf, BLKHDR_LENGTH);
- unser_uint32(block->CheckSum);
+ unser_uint32(CheckSumLo); // can be blkh_options in BB03
unser_uint32(block_len);
unser_uint32(BlockNumber);
unser_bytes(Id, BLKHDR_ID_LENGTH);
ASSERT(unser_length(block->buf) == BLKHDR1_LENGTH);
Id[BLKHDR_ID_LENGTH] = 0;
-
+ block->CheckSum64 = CheckSumLo; // only for BB01 & BB02
if (Id[3] == '1') {
bhl = BLKHDR1_LENGTH;
block->BlockVer = 1;
block->read_errors++;
return false;
}
+ } else if (Id[3] == '3') {
+ bhl = BLKHDR3_LENGTH;
+ uint32_t blkh_options = CheckSumLo; /* in BB03 blkh_options is stored at the checksum location */
+ unser_uint32(block->VolSessionId);
+ unser_uint32(block->VolSessionTime);
+
+ /* decrypt the block before to calculate the checksum */
+ if ((blkh_options & BLKHOPT_ENCRYPT_BLOCK) && block->dev->crypto_device_ctx)
+ {
+ block_cipher_init_iv_header(block->dev->crypto_device_ctx, BlockNumber, block->VolSessionId, block->VolSessionTime);
+ block_cipher_decrypt(block->dev->crypto_device_ctx, block_len-bhl, block->buf+bhl, block->buf_enc);
+ memcpy(block->buf+bhl, block->buf_enc, block_len-bhl);
+ }
+
+ unser_begin(block->buf+chk_off, BLKHDR_CS64_LENGTH);
+ unser_uint64(block->CheckSum64);
+
+ block->BlockVer = 3;
+ block->bufp = block->buf + bhl;
+ //Dmsg5(100, "Read-blkhdr Block=%p adata=%d buf=%p bufp=%p off=%d\n", block, block->adata,
+ // block->buf, block->bufp, block->bufp-block->buf);
+ if (strncmp(Id, BLKHDR3_ID, BLKHDR_ID_LENGTH) != 0) {
+ dev->dev_errno = EIO;
+ Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Wanted ID: \"%s\", got \"%s\". Buffer discarded.\n"),
+ dev->get_hi_addr(block->BlockAddr),
+ dev->get_low_addr(block->BlockAddr),
+ BLKHDR3_ID, Id);
+ if (block->read_errors == 0 || verbose >= 2) {
+ Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
+ }
+ block->read_errors++;
+ return false;
+ }
} else {
dev->dev_errno = EIO;
Mmsg4(dev->errmsg, _("Volume data error at %u:%u! Wanted ID: \"%s\", got \"%s\". Buffer discarded.\n"),
dev->get_hi_addr(block->BlockAddr),
dev->get_low_addr(block->BlockAddr),
- BLKHDR2_ID, Id);
+ BLKHDR3_ID, Id);
Dmsg1(50, "%s", dev->errmsg);
if (block->read_errors == 0 || verbose >= 2) {
Jmsg(jcr, M_FATAL, 0, "%s", dev->errmsg);
Dmsg3(390, "Read binbuf = %d %d block_len=%d\n", block->binbuf,
bhl, block_len);
if (block_len <= block->read_len && dev->do_checksum()) {
- BlockCheckSum = dcr->crc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH,
+ if (Id[3] == '3') {
+ uint64_t save; /* make a copy of the xxh64 */
+ memcpy(&save, block->buf+chk_off, BLKHDR_CS64_LENGTH);
+ memset(block->buf+chk_off, '\0', BLKHDR_CS64_LENGTH);
+ BlockCheckSum64 = XXH3_64bits((uint8_t *)block->buf,
+ block_len);
+ memcpy(block->buf+chk_off, &save, BLKHDR_CS64_LENGTH);
+ } else {
+ BlockCheckSum64 = dcr->crc32((uint8_t *)block->buf+BLKHDR_CS_LENGTH,
block_len-BLKHDR_CS_LENGTH,
- block->CheckSum);
-
- if (BlockCheckSum != block->CheckSum) {
+ block->CheckSum64);
+ }
+ if (BlockCheckSum64 != block->CheckSum64) {
dev->dev_errno = EIO;
Mmsg6(dev->errmsg, _("Volume data error at %u:%u!\n"
- "Block checksum mismatch in block=%u len=%d: calc=%x blk=%x\n"),
+ "Block checksum mismatch in block=%u len=%d: calc=%llx blk=%llx\n"),
dev->file, dev->block_num, (unsigned)BlockNumber,
- block_len, BlockCheckSum, block->CheckSum);
+ block_len, BlockCheckSum64, block->CheckSum64);
if (block->read_errors == 0 || verbose >= 2) {
Jmsg(jcr, M_ERROR, 0, "%s", dev->errmsg);
dump_block(dev, block, "with checksum error");
* and on tape devices, apply min and fixed blocking.
*/
wlen = block->binbuf;
+
if (wlen != block->buf_len) {
Dmsg2(250, "binbuf=%d buf_len=%d\n", block->binbuf, block->buf_len);
}
}
if (block->adata && dev->padding_size > 0) {
- /* Write to next aligned boundry */
+ /* Write to next aligned boundary */
wlen = ((wlen + dev->padding_size - 1) / dev->padding_size) * dev->padding_size;
}
ASSERT(wlen <= block->buf_len);
}
}
+static void display_enctype(HPKT &hpkt)
+{
+ int i;
+ for (i=0; enc_types[i].name; i++) {
+ if (*(int32_t *)(hpkt.ritem->value) == enc_types[i].token) {
+ hpkt.sendit(hpkt, "\n \"%s\": \"%s\"", hpkt.ritem->name,
+ enc_types[i].name);
+ return;
+ }
+ }
+}
+
static void display_label(HPKT &hpkt)
{
int i;
display_int32_pair(hpkt);
} else if (items[item].handler == store_devtype) {
display_devtype(hpkt);
+ } else if (items[item].handler == store_enctype) {
+ display_enctype(hpkt);
} else if (items[item].handler == store_label) {
display_label(hpkt);
} else if (items[item].handler == store_cloud_driver) {
if (device && device->dev == this) {
device->dev = NULL;
}
+ if (crypto_device_ctx) {
+ block_cipher_context_free(crypto_device_ctx);
+ crypto_device_ctx = NULL;
+ }
delete this;
}
{
return false;
}
+
+bool DEVICE::load_encryption_key(DCR *dcr, const char *operation,
+ const char *volume_name,
+ uint32_t *enc_cipher_key_size, unsigned char *enc_cipher_key,
+ uint32_t *master_keyid_size, unsigned char *master_keyid)
+{
+ enum { op_none, op_label, op_read };
+ bool ok = true; // No error
+ if (!device->block_encryption) {
+ return ok;
+ }
+ JCR *jcr = dcr->jcr;
+// char *edit_device_codes(DCR *dcr, char *omsg, const char *imsg, const char *cmd)
+ POOLMEM *encrypt_program = get_pool_memory(PM_FNAME);
+ POOL_MEM results(PM_MESSAGE);
+ POOL_MEM err_msg(PM_MESSAGE);
+ POOL_MEM envv;
+
+ int op = op_none;
+ if (0 == strcmp(operation, "LABEL")) {
+ op = op_label;
+ } else if (0 == strcmp(operation, "READ")) {
+ op = op_read;
+ }
+
+ edit_device_codes(dcr, &encrypt_program, me->encryption_command, "load");
+ char *envp[5];
+ Mmsg(envv, "OPERATION=%s", operation);
+ envp[0] = bstrdup(envv.c_str());
+ Mmsg(envv, "VOLUME_NAME=%s", volume_name);
+ envp[1] = bstrdup(envv.c_str());
+ if (op == op_read && enc_cipher_key != NULL && *enc_cipher_key_size > 0) {
+ char buf[2*MAX_BLOCK_CIPHER_KEY_LEN]; // for the base64 encoded enc_cipherkey
+ bin_to_base64_pad(buf, sizeof(buf), (char *)enc_cipher_key, *enc_cipher_key_size);
+ Mmsg(envv, "ENC_CIPHER_KEY=%s", buf);
+ } else {
+ Mmsg(envv, "ENC_CIPHER_KEY=");
+ }
+ envp[2] = bstrdup(envv.c_str());
+ if (op == op_read && master_keyid != NULL && *master_keyid_size > 0) {
+ char buf[2*MAX_MASTERKEY_ID_LEN]; // for the base64 encoded masterkey_id
+ bin_to_base64_pad(buf, sizeof(buf), (char *)master_keyid, *master_keyid_size);
+ Mmsg(envv, "MASTER_KEYID=%s", buf);
+ } else {
+ Mmsg(envv, "MASTER_KEYID=");
+ }
+ envp[3] = bstrdup(envv.c_str());
+ envp[4] = NULL;
+
+ Dmsg3(60, "Run keymanager op=%s volume=%s %s\n", operation, volume_name, encrypt_program);
+ for (char **p=envp; *p!=NULL; p++) Dmsg1(200, "keymanager query %s\n", *p);
+
+ uint32_t timeout = 60;
+ int status = run_program_full_output(encrypt_program, timeout, results.addr(), envp);
+ free_pool_memory(encrypt_program);
+ for (unsigned i = 0; i < sizeof(envp)/sizeof(char *)-1; i++)
+ {
+ bfree(envp[i]);
+ }
+
+ block_cipher_type cipher = BLOCK_CIPHER_NONE;
+ const char *cipher_name = "undefined";
+ int cipher_key_size = 0; /* from the cipher */
+ char *in_cipher_key = NULL;
+ int in_cipher_key_size = 0;
+ char *in_enc_cipher_key = NULL;
+ int in_enc_cipher_key_size = 0;
+ int in_master_keyid_size = 0;
+ char *in_master_keyid = NULL;
+ char *comment = NULL;
+ char keybuf[4096], enckeybuf[4096], masterkeyidbuf[4096];
+ if (status == 0) {
+ if (chk_dbglvl(200)) {
+ /* display the response of the key manager */
+ char *s = results.c_str();
+ while (*s != '\0') {
+ char *e = strchr(s, '\n');
+ if (e != NULL) {
+ char c = *e;
+ *e = '\0';
+ Dmsg1(200, "keymanager response %s\n", s);
+ *e = c;
+ s = e + 1;
+ }
+ }
+ }
+ /* iterate line to retrieve field */
+ char *p = results.c_str();
+ char *fieldname = NULL;
+ char *value = NULL;
+ char *end;
+ while (*p != '\0') {
+ while (isspace(*p)) p++;
+ fieldname = p; /* start of the fieldname */
+ while (isalnum(*p) || *p == '_') p++;
+ if (fieldname == p) {
+ /* fieldname is empty, EOF */
+ break;
+ }
+ if (isblank(*p) || *p=='=' || *p==':') {
+ end = p;
+ } else {
+ Dmsg1(10, "keymanager response format mismatch at %d\n", p-results.c_str());
+ Mmsg(err_msg, "line format mismatch");
+ break;
+ }
+ while (isblank(*p)) p++; /* skip blank just before the separator */
+ if (*p != '=' && *p != ':') {
+ Dmsg2(10, "keymanager response wrong separator %d %d\n", p-results.c_str(), *p);
+ Mmsg(err_msg, "wrong separator");
+ break;
+ }
+ p++; /* skip the separator */
+ while (isblank(*p)) p++; /* skip blank just after the separator */
+ *end = '\0'; /* end the fieldname */
+ value = p; /* start of the value */
+ while (*p && *p != '\n') p++;
+ if (*p != '\0') {
+ *p = '\0';
+ p++;
+ }
+ Dmsg3(200, "keymanager response fieldname=%s value=\"%s\" pos=%d\n",
+ fieldname, value, p-results.c_str());
+ if (0==strcmp("error", fieldname)) {
+ Mmsg(err_msg, "got error message: \"%s\"", value);
+ break;
+ } else if (0==strcmp("volume_name", fieldname)) {
+ /* ignore */
+ } else if (0==strcmp("comment", fieldname)) {
+ comment=value;
+ /* ignore */
+ } else if (0==strcmp("cipher", fieldname)) {
+ cipher_name = value;
+ if (0==strcasecmp("AES_128_XTS", value)) {
+ cipher = BLOCK_CIPHER_AES_128_XTS;
+ cipher_key_size = 32;
+ } else if (0==strcasecmp("AES_256_XTS", value)) {
+ cipher = BLOCK_CIPHER_AES_256_XTS;
+ cipher_key_size = 64;
+ } else if (0==strcasecmp("NULL", value)) {
+ cipher = BLOCK_CIPHER_NULL;
+ cipher_key_size = 16;
+ } else {
+ cipher_key_size = 0;
+ Mmsg(err_msg, "unknown cipher: \"%s\"", value);
+ break;
+ }
+ } else if (0==strcmp("cipher_key", fieldname)) {
+ in_cipher_key = value;
+ } else if (0==strcmp("enc_cipher_key", fieldname)) {
+ in_enc_cipher_key = value;
+ } else if (0==strcmp("master_keyid", fieldname)) {
+ in_master_keyid = value;
+ }
+ }
+ if (err_msg.c_str()[0] == '\0') {
+ /* no error, check that we have all the parameter we need */
+ if (cipher == BLOCK_CIPHER_NONE) {
+ Mmsg(err_msg, "cipher is missing");
+ } else if (in_cipher_key == NULL) {
+ Mmsg(err_msg, "key is missing");
+ }
+ }
+ if (err_msg.c_str()[0] == '\0' && in_cipher_key != NULL) {
+ /* no error, check that we can decode the key */
+ in_cipher_key_size = base64_to_bin(keybuf, sizeof(keybuf), in_cipher_key, strlen(in_cipher_key));
+ if (cipher_key_size != in_cipher_key_size) {
+ Mmsg(err_msg, "Wrong cipher key size for \"%s\" expect %d, got %d", cipher_name, cipher_key_size, in_cipher_key_size);
+ }
+ }
+ if (err_msg.c_str()[0] == '\0' && in_enc_cipher_key != NULL) {
+ /* no error, check that we can decode the key */
+ in_enc_cipher_key_size = base64_to_bin(enckeybuf, sizeof(enckeybuf), in_enc_cipher_key, strlen(in_enc_cipher_key));
+ if (cipher_key_size != in_enc_cipher_key_size) {
+ Mmsg(err_msg, "Wrong cipher key size for \"%s\" expect %d, got %d", cipher_name, cipher_key_size, in_cipher_key_size);
+ }
+ }
+ if (err_msg.c_str()[0] == '\0' && in_master_keyid != NULL) {
+ /* no error, check that we can decode the master_keyid*/
+ in_master_keyid_size = base64_to_bin(masterkeyidbuf, sizeof(masterkeyidbuf), in_master_keyid, strlen(in_master_keyid));
+ }
+ } else {
+ /* status != 0 the script returned an error code */
+ berrno be;
+ be.set_errno(status);
+ Mmsg(err_msg, "encryption script returned an error, code=%d ERR=%s", status, be.bstrerror());
+ }
+ if (crypto_device_ctx != NULL) {
+ block_cipher_context_free(crypto_device_ctx);
+ crypto_device_ctx = NULL;
+ }
+ if (err_msg.c_str()[0] == '\0' && in_enc_cipher_key_size > 0) {
+ if (in_enc_cipher_key_size > MAX_BLOCK_CIPHER_KEY_LEN) {
+ Mmsg(err_msg, "encrypted key is too large");
+ } else {
+ *enc_cipher_key_size = in_enc_cipher_key_size;
+ memcpy(enc_cipher_key, enckeybuf, *enc_cipher_key_size);
+ }
+ }
+ if (err_msg.c_str()[0] == '\0' && in_master_keyid_size > 0) {
+ if (in_master_keyid_size > MAX_MASTERKEY_ID_LEN) {
+ Mmsg(err_msg, "masterkey id is too large");
+ } else {
+ *master_keyid_size = in_master_keyid_size;
+ memcpy(master_keyid, masterkeyidbuf, *master_keyid_size);
+ }
+ }
+
+ if (err_msg.c_str()[0] == '\0') {
+ /* initialize the crypto context */
+ crypto_device_ctx = block_cipher_context_new(cipher);
+ block_cipher_init_key(crypto_device_ctx, (unsigned char*)keybuf);
+ Jmsg(jcr, M_INFO, 0, _("3305 LoadEncryptionKey \"for Volume %s\", status is OK.\n"),
+ dcr->VolumeName);
+ if (comment!=NULL) {
+ /* ignored for now */
+ }
+ Dmsg1(20, "load encryption key for volume %s OK\n", dcr->VolumeName);
+ }
+ if (err_msg.c_str()[0] != '\0') {
+ Dmsg2(10, "load encryption key for volume %s Err=%s\n", dcr->VolumeName, err_msg.c_str());
+ Jmsg(jcr, M_FATAL, 0, _("3992 Bad LoadEncryptionKey \"load Volume %s\": "
+ "ERR=%s\n"), dcr->VolumeName, err_msg.c_str());
+ if (jcr != NULL) {
+ Mmsg(jcr->errmsg, _("3992 Bad LoadEncryptionKey \"load Volume %s\": "
+ "ERR=%s\n"), dcr->VolumeName, err_msg.c_str());
+ }
+ ok = false;
+ } else {
+ /* crypto key successfully loaded */
+ if (op == op_label) {
+ }
+ }
+ return ok;
+}
bool mount_tape(int mount, int dotimeout); /* in dev.c */
protected:
void set_mode(int omode); /* in dev.c */
+ bool load_encryption_key(DCR *dcr, const char *operation,
+ const char *volume_name, /* in dev.c */
+ uint32_t *enc_cipher_key_size, unsigned char *enc_cipher_key,
+ uint32_t *master_keyid_size, unsigned char *master_keyid);
+
};
inline const char *DEVICE::strerror() const { return errmsg; }
inline const char *DEVICE::archive_name() const { return dev_name; }
pm_strcat(archive_name, getVolCatName());
}
}
-
mount(1); /* do mount if required */
set_mode(omode);
dev->read_only = device->read_only;
dev->dev_type = device->dev_type;
dev->device = device;
+ dev->crypto_device_ctx = NULL;
if (dev->is_tape()) { /* No parts on tapes */
dev->max_part_size = 0;
} else {
goto bail_out;
}
+ if (!load_encryption_key(dcr, "READ", VolHdr.VolumeName, &VolHdr.EncCypherKeySize, VolHdr.EncCypherKey, &VolHdr.MasterKeyIdSize, VolHdr.MasterKeyId)) {
+ stat = VOL_LABEL_ERROR;
+ goto bail_out; /* a message is already loaded into jcr->errmsg */
+ }
+
if (dcr->is_writing()) {
empty_block(dcr->block);
}
}
Dmsg1(150, "Label type=%d\n", dev->label_type);
+ if (!load_encryption_key(dcr, "LABEL", VolName, &VolHdr.EncCypherKeySize, VolHdr.EncCypherKey, &VolHdr.MasterKeyIdSize, VolHdr.MasterKeyId)) {
+ goto bail_out;
+ }
if (!write_volume_label_to_dev(dcr, VolName, PoolName, relabel, no_prelabel)) {
goto bail_out;
}
ser_string(dev->VolHdr.PoolType);
ser_string(dev->VolHdr.MediaType);
- ser_string(dev->VolHdr.HostName);
+ if (dev->device->block_encryption == ET_STRONG) {
+ ser_string("OBFUSCATED");
+ } else {
+ ser_string(dev->VolHdr.HostName);
+ }
ser_string(dev->VolHdr.LabelProg);
ser_string(dev->VolHdr.ProgVersion);
ser_string(dev->VolHdr.ProgDate);
/* adata and dedup volumes */
ser_uint32(dev->VolHdr.BlockSize);
+ /* new in for BB03 */
+ ser_uint32(dev->VolHdr.EncCypherKeySize);
+ ser_bytes(dev->VolHdr.EncCypherKey, dev->VolHdr.EncCypherKeySize);
+ ser_uint32(dev->VolHdr.MasterKeyIdSize);
+ ser_bytes(dev->VolHdr.MasterKeyId, dev->VolHdr.MasterKeyIdSize);
+ ser_uint32(0); /* This for a Signature Key identifier a upcoming feature */
+
ser_end(rec->data, SER_LENGTH_Volume_Label);
if (!adata) {
bstrncpy(dcr->VolumeName, dev->VolHdr.VolumeName, sizeof(dcr->VolumeName));
}
+ dcr->block->first_block = true;
ASSERT2(dcr->VolumeName[0], "Empty Volume name");
rec->data_len = ser_length(rec->data);
rec->FileIndex = dev->VolHdr.LabelType;
bstrncpy(dev->VolHdr.LabelProg, my_name, sizeof(dev->VolHdr.LabelProg));
sprintf(dev->VolHdr.ProgVersion, "Ver. %s %s ", VERSION, BDATE);
sprintf(dev->VolHdr.ProgDate, "Build %s %s ", __DATE__, __TIME__);
+
+ /* Warning in BB03 EncCypherKey* & MasterKeyId* have already been set, don't reset */
+
dev->set_labeled(); /* set has Bacula label */
if (chk_dbglvl(100)) {
dev->dump_volume_label();
unser_string(dev->VolHdr.ProgVersion);
unser_string(dev->VolHdr.ProgDate);
-// unser_string(dev->VolHdr.AlignedVolumeName);
+ unser_string(dev->VolHdr.AlignedVolumeName);
dev->VolHdr.AlignedVolumeName[0] = 0;
unser_uint64(dev->VolHdr.FirstData);
unser_uint32(dev->VolHdr.FileAlignment);
unser_uint32(dev->VolHdr.PaddingSize);
unser_uint32(dev->VolHdr.BlockSize);
+ if (rec->BlockVer == 3) {
+ unser_uint32(dev->VolHdr.EncCypherKeySize);
+ unser_bytes(dev->VolHdr.EncCypherKey, dev->VolHdr.EncCypherKeySize);
+ unser_uint32(dev->VolHdr.MasterKeyIdSize);
+ unser_bytes(dev->VolHdr.MasterKeyId, dev->VolHdr.MasterKeyIdSize);
+ uint32_t SignatureKeyIdSize;
+ unser_uint32(SignatureKeyIdSize);
+ (void)SignatureKeyIdSize; /* don't complain about unused variable */
+ } else {
+ dev->VolHdr.EncCypherKeySize = 0;
+ dev->VolHdr.MasterKeyIdSize = 0;
+ }
+
ser_end(rec->data, SER_LENGTH_Volume_Label);
Dmsg0(190, "unser_vol_label\n");
if (chk_dbglvl(100)) {
break;
}
- Pmsg12(-1, _("\nVolume Label:\n"
+ Pmsg14(-1, _("\nVolume Label:\n"
"Adata : %d\n"
"Id : %s"
"VerNo : %d\n"
"MediaType : %s\n"
"PoolType : %s\n"
"HostName : %s\n"
+"EncCypherKeySize : %ld\n"
+"MasterKeyIdSize : %ld\n"
""),
adata,
VolHdr.Id, VolHdr.VerNum,
VolHdr.VolumeName, VolHdr.PrevVolumeName,
File, LabelType, VolHdr.LabelSize,
VolHdr.PoolName, VolHdr.MediaType,
- VolHdr.PoolType, VolHdr.HostName);
+ VolHdr.PoolType, VolHdr.HostName,
+ VolHdr.EncCypherKeySize, VolHdr.MasterKeyIdSize);
if (VolHdr.VerNum >= 11) {
char dt[50];
bool find_truncate_option(const char* truncate, uint32_t& truncate_option);
void store_upload(LEX *lc, RES_ITEM *item, int index, int pass);
void store_devtype(LEX *lc, RES_ITEM *item, int index, int pass);
+void store_enctype(LEX *lc, RES_ITEM *item, int index, int pass);
void store_cloud_driver(LEX *lc, RES_ITEM *item, int index, int pass);
void store_maxblocksize(LEX *lc, RES_ITEM *item, int index, int pass);
void store_transfer_priority(LEX *lc, RES_ITEM *item, int index, int pass);
uint32_t extra_bytes; /* the extra size that must be accounted in VolABytes */
uint32_t state_bits; /* state bits */
uint32_t RecNum; /* Record number in the block */
+ int BlockVer; /* 1, 2 or .. from BB01, BB02, ... */
uint32_t BlockNumber; /* Block number for this record (used in read_records()) */
bool invalid; /* The record may be invalid if it was merged with a previous record */
rec_state wstate; /* state of write_record_to_block */
/* For Cloud */
uint64_t MaxPartSize; /* Maximum Part Size */
+ /* For Volume encryption */
+ uint32_t EncCypherKeySize;
+ uint32_t MasterKeyIdSize;
+ unsigned char EncCypherKey[MAX_BLOCK_CIPHER_KEY_LEN]; /* The encryption key
+ encrypted with the MasterKey */
+ unsigned char MasterKeyId[MAX_MASTERKEY_ID_LEN]; /* the ID of the MasterKey */
+ /* For Volume Signature (not yet implemented) */
+ uint32_t SignatureKeyIdSize;
+ unsigned char SignatureKeyId[MAX_MASTERKEY_ID_LEN];
};
-#define SER_LENGTH_Volume_Label 1024 /* max serialised length of volume label */
+#define SER_LENGTH_Volume_Label 2048 /* max serialised length of volume label */
#define SER_LENGTH_Session_Label 1024 /* max serialised length of session label */
typedef struct Volume_Label VOLUME_LABEL;
block->adata, block->BlockNumber, block->BlockVer, block->block_len);
if (block->BlockVer == 1) {
rhl = RECHDR1_LENGTH;
- } else {
+ } if (block->BlockVer == 2) {
rhl = RECHDR2_LENGTH;
+ } else {
+ rhl = RECHDR3_LENGTH;
}
if (rec->remlen >= rhl) {
Dmsg0(dbgep, "=== rpath 2 begin unserial header\n");
rec->VolumeName = dcr->CurrentVol->VolumeName; /* From JCR::VolList, freed at the end */
rec->Addr = rec->StartAddr = dcr->block->BlockAddr;
}
-
+ rec->BlockVer = dcr->block->BlockVer; /* needed for unser_volume_label() */
/* We read the next record */
dcr->block->RecNum++;
Dmsg0(dbgep, "=== wpath 13 write_header_to_block\n");
ser_uint32(rec->VolSessionId);
ser_uint32(rec->VolSessionTime);
- } else {
+ } else { // BLOCK_VER is 2 or 3
Dmsg0(dbgep, "=== wpath 14 write_header_to_block\n");
block->VolSessionId = rec->VolSessionId;
block->VolSessionTime = rec->VolSessionTime;
rdev->device = dcr->dev->device;
rdcr = new_dcr(jcr, NULL, rdev, SD_READ);
rdcr->spool_fd = dcr->spool_fd;
- block = dcr->block; /* save block */
- dcr->block = rdcr->block; /* make read and write block the same */
+ block = rdcr->block; /* save block */
+ rdcr->block = dcr->block; /* make read and write block the same */
Dmsg1(800, "read/write block size = %d\n", block->buf_len);
lseek(rdcr->spool_fd, 0, SEEK_SET); /* rewind */
despool_elapsed / 3600, despool_elapsed % 3600 / 60, despool_elapsed % 60,
edit_uint64_with_suffix(jcr->dcr->job_spool_size / despool_elapsed, ec1));
- dcr->block = block; /* reset block */
+ rdcr->block = block; /* reset block */
#if defined(HAVE_POSIX_FADVISE) && defined(POSIX_FADV_DONTNEED)
posix_fadvise(rdcr->spool_fd, 0, 0, POSIX_FADV_DONTNEED);
if (block->binbuf <= WRITE_BLKHDR_LENGTH) { /* Does block have data in it? */
return true;
}
-
hlen = sizeof(spool_hdr);
wlen = block->binbuf;
P(dcr->dev->spool_mutex);
--- /dev/null
+
+
+New Storage Daemon encryption and XXH64 checksum
+++++++++++++++++++++++++++++++++++++++++++++++++
+
+This new version switch from the ``BB02`` volume format to the new ``BB03``.
+This new format replaces the old 32bits CRC32 checksum with
+the new and faster 64bits *XXH64*.
+The new format also add support for volume encryption by the Storage Daemon.
+Finally this patch add 3 new file Digest Algorithm *XXH64*,
+*XXH3_64* and *XXH3_128* to the ones that already exists *MD5*, *SHA1*, *SHA256*,
+*SHA512*. These 3 new *XX...* algorithms are not Cryptographic algorithms,
+but are very good regarding the dispersion and randomness qualities.
+There are also the fastest ones available.
+
+The New XXH64 checksum
+======================
+
+The new XXH64 checksum, is optional you can disable it using the same
+directive than for the old CRC32 in ``BB02`` ``Block Checksum = no``
+
+Storage Daemon encryption
+=========================
+
+Blocks in the volumes can be encrypted using the
+*BLOCK_CIPHER_AES_128_XTS* or *BLOCK_CIPHER_AES_256_XTS* ciphers algorithms.
+These symmetrical ciphers are fast and used by most applications
+that need to do symmetrical encryption of blocks.
+
+Every blocks are encrypted using a *key* that is unique for the
+volume and an *IV* (Initialization vector) that is the block number
+that is saved in the block header.
+The *XTS's* ciphers are specifically designed to support an *IV* with a low
+entropy
+
+The first *block* of the volume that hold the *Volume Label*
+is not encrypted because some fields as the *volume name*
+are required to manage the volume and the encryption.
+The user has the option to obfuscate some fields
+that are not required and could hold critical
+information like for example the *hostname*.
+These field are replaced by the string *"OBFUSCATED"*
+
+The header of the block are not encrypted. This 24 bytes header
+don't hold any user information. Here is the content of the header:
+
+- the 32bits header option bit field
+- the 32bits block length
+- the 32bits block number
+- the ``BB03`` string
+- the 32bits volume session id
+- the 32bits volume session time
+
+The *volume session time* is the time of the Storage Daemon
+at startup and not the time of the backup.
+The *volume session id* is reset at zero when the daemon starts
+and is incremented at every backup by the Storage Daemon
+
+The 64bits *XXH64* checksum is encrypted with the data.
+The *block* must be be decrypted and then the checksum can be verified.
+If the checksum matches, you can be confident that you have the right
+encryption key and that the block has not been modified.
+The drawback is that it is not possible to verify the integrity of
+the block without the encryption key.
+
+The data in the *data spool* are not encrypted, don't store you
+data spool on an unsafe storage!
+
+The new Storage Daemon Directives
+=================================
+
+Here are the new directives in the Storage resource of the Storage Daemon
+
+.. _Storage:Device:BlockEncryption:
+
+**Block Encryption = <none|enable|strong>**
+ This directive allows you to enable the encryption for the given device.
+ The encryption can be of 3 different types:
+
+ **none** This is the default, the device don't do any encryption.
+
+ **enable** The device encrypts the data but let all information in
+ the *volume label* in clear text.
+
+ **strong** The device encrypts the data, and obfuscate any
+ information in the *volume label* except the one that are needed
+ for the management of the volume. The fields that are obfuscate are:
+ *volume name*.
+
+.. _Storage:Storage:EncryptionCommand:
+
+**Encryption Command = <command>** The **command** specifies an external
+ program that must provide the key related to a volume.
+
+
+The Encryption Command
+======================
+
+The *Encryption Command* is called by the Storage Daemon every time it
+initialize a new volume or mount an existing one using a device
+with encryption enable.
+We are providing a very simple script that can handle keys
+Here is an example about how to call the command.
+
+::
+
+ EncryptionCommand = "/opt/bacula/script/sd_encryption_command.py getkey --key-dir /opt/bacula/working/keys"
+
+Be careful the command is limited to 127 characters.
+The same variable substitution than for the *AutoChanger command* are provided.
+
+The program can be an interface with your existing key management system or
+do the key management on its own.
+*Bacula* provides a sample script *sd_encryption_command.py* that do the work.
+
+Lets illustrate our protocol with some usage of our script::
+
+ $ OPERATION=LABEL VOLUME_NAME=Volume0001 ./sd_encryption_command.py getkey --cipher AES_128_XTS --key-dir tmp/keys
+ cipher: AES_128_XTS
+ cipher_key: G6HksAYDnNGr67AAx2Lb/vecTVjZoYAqSLZ7lGMyDVE=
+ volume_name: Volume0001
+
+ $ OPERATION=READ VOLUME_NAME=Volume0001 ./sd_encryption_command.py getkey --cipher AES_128_XTS --key-dir tmp/keys
+ cipher: AES_128_XTS
+ cipher_key: G6HksAYDnNGr67AAx2Lb/vecTVjZoYAqSLZ7lGMyDVE=
+ volume_name: Volume0001
+
+ $ cat tmp/keys/Volume0001
+ cipher: AES_128_XTS
+ cipher_key: G6HksAYDnNGr67AAx2Lb/vecTVjZoYAqSLZ7lGMyDVE=
+ volume_name: Volume0001
+
+ $ OPERATION=READ VOLUME_NAME=DontExist ./sd_encryption_command.py getkey --cipher AES_128_XTS --key-dir tmp/keys 2>/dev/null
+ error: no key information for volume "DontExist"
+ $ echo $?
+ 0
+
+ $ OPERATION=BAD_CMD VOLUME_NAME=Volume0002 ./sd_encryption_command.py getkey --cipher AES_128_XTS --key-dir tmp/keys 2>/dev/null
+ error: environment variable OPERATION invalid "BAD_CMD" for volume "Volume0002"
+ $ echo $?
+ 0
+
+You can see in this command above that we keep the keys into one
+directory *tmp/keys*, we pass the arguments using the *environment variables*.
+
+*Bacula* pass the following variables via the *environment*:
+
+ **OPERATION** This is can *LABEL* when the volume is labeled, in this case
+ the script should generate a new key or this can be *READ* when
+ the volume has already a label and the Storage Daemon need the already
+ existing key to read or append data to the volume
+
+ **VOLUME_NAME** This is the name of the volume
+
+Some variables are already there to support a *Master Key* in the future.
+This feature is not yet supported, but will come later:
+
+ **ENC_CIPHER_KEY** This is a base64 encoded version of the key encrypted by
+ the *master key*
+
+ **MASTER_KEYID** This is a base64 encoded version of the key Id of
+ the *master key* that was used to encrypt the *ENC_CIPHER_KEY* above.
+
+*Bacula* expects some values in return:
+
+ **volumename** This is a repetition of the name of the volume that is
+ given to the script. This field is optional and ignored by Bacula.
+
+ **cipher** This is the cipher that Bacula must use.
+ Bacula knows the following ciphers: *AES_128_XTS* and *AES_256_XTS*.
+ Of course the key length vary with the cipher.
+
+ **cipher_key** This is the symmetric *key* in *base 64* format.
+
+ **comment** This is a single line of comment that is optional and ignored by Bacula.
+
+ **error** This is a single line error message.
+ This is optional, but when provided, Bacula consider that the script
+ returned an error and display this error in the job log.
+
+Bacula expect an *exit code" of 0, if the script exits with a different
+error code, any output are ignored and Bacula display a generic message
+with the exit code in the job log.
+To return an error to bacula, the script must use the *error* field
+and return an error code of 0.
+
+What is encrypted and what is not
+=================================
+
+The main goal of encryption is to prevent anybody that don't own the key to
+read the data. And *Bacula* does it well. But encryption alone don't protect
+again some modification.
+
+The first block of the volume is the *volume label* and it is not encrypted.
+Some information are required for the management of the *volume* itself.
+The only data in the *volume label* coming from the user are: the *hostname*,
+the *volumename* *poolname*. The *hostname* can be obfuscated using
+the *STRONG* mode of the encryption, the *poolname* and the *volumename*
+could be made useless to a attacker by using generic name lie ``PoolAlpha`` or
+``Volume12345``.
+
+Also be aware that data in the catalog: the directories, filenames, and
+the *JobLog* are not encrypted.
+
+An attacker could make some undetected modification to the volume.
+The easiest way is to remove one block inside the volume.
+Other verification inside *Bacula* could detect such modification and
+an attacker must be meticulous, but it is possible.
+
+The *XXH64* checksum inside each volume are encrypted using the encryption key.
+This is not as good as using a certified signature but this provides
+substantial confidence that the block will not be modified easily.
+
+To resume, you can be confident that:
+
+- An attacker cannot read any of your data: **Very Strong**
+- An attacker cannot substitute the volume by another one: **Strong**
+- An attacker cannot modify the content of the volume: **Good**
+
+
+Notes for the developers and support and personal
++++++++++++++++++++++++++++++++++++++++++++++++++
+
+The new Volume Format
+=====================
+In the new ``BB03`` volume format, the space used by the CRC32 is now
+used by a 32bits bit field that for now only knows about the
+two *option* bits:
+
+::
+
+ BLKHOPT_CHKSUM = 1 << 0, // If the XXH64 checksum is enable
+ BLKHOPT_ENCRYPT_VOL = 1 << 1, // If the volume is encrypted
+ BLKHOPT_ENCRYPT_BLOCK = 1 << 2, // If this block is encrypted
+
+Notice that the *block number* is reset to zero when the volume is happened
+and then is not unique! This could be a little security concern,
+as multiple block could be encrypted using the same key and *IV*
+
+The New XXH64 checksum
+======================
+
+The new XXH64 checksum, is saved just after *VolSessionTime* and is set to
+zero if the checksum is disabled. The checksum is calculated on the all
+block with the new 64bits checksum field set to zero.
+To verify the checksum
+you must first verify that *the BLKHOPT_CHKSUM* bit is enable, then un-serialize
+the value of the checksum in the block and save it, then replace it with zeroes,
+calculate the checksum of the block and compare it with the value that
+you have saved.
+
+
+Volume Signature
+================
+
+The purpose is to be able to verify:
+ - the authenticity of the volume (that we have the volume we are looking for)
+ - the integrity of the volume (that the data on the volume did not change)
+
+This is independent of the volume encryption.
+
+Some assertion to validate:
+ - The Volume Signature is configured on the SD side only.
+ - Nothing is configured on the DIR side
+ - The SD should be able to tell the DIR that this feature is enable on one volume,
+ and the DIR **could** (something to decide) store this information in the *volume media* in the catalog
+ - The DIR must be able to initiate a volume signature check and display the result.
+ We need to define an interfece for that.
+ - The signature are more commonly done using public/private keys pairs.
+ - The customer could appreciate to use different keys pairs for different groups of volumes
+ (for example regarding the poolname)
+ - The KeyManager can handle these keys the same way it is handling the encryption keys.
+
+Something in the volume must tell the user which key was used to sign the volume and which one can be used to verify:
+ - this information is not mandatory as the user could keep this information somewhere else
+ - we could store the keyid, the KeyManager can then provide the public key
+ - we can store the public key. A 4096bits public key is about 717bytes width (this include the *modulus*).
+ But this is not valid has me must validate the content of the volume from information from outer the volume.
+ Then the keyid is better solution, we are sure that the public key will come from outside.
+
+At the block level.
+
+The signature digest for a SHA256 and a 2048bits key is 122bytes.
+I don't know how the size fluctuate, but the size could vary if we want to use another
+hash function or maybe another key size.
+Then the size in the block must be variable.
+The best place at the end of the header.
+How to combine the signature and the XXH64 checksum ?
+
+First set the signature and the XXH64 checksum spaces to 0x00.
+Calculate and fill in the signature area.
+Second calculate and fill in the XXH64 checksum area.
+For the verification, the XXH64 checksum will always be verified while
+the signature will only be verified if we can provide or verify
+the authenticity of the public key.
+
+
+
+Weakness of the signature in bacula
+===================================
+
+The signature sign individual block.
+If one block is removed from the volume we don't know it.
+We could ensure that the BlockNumber is well updated
+and verify that there is no missing BlockNumber.
+Today the BlockNumber is reset to zero every time the volume is unmounted.
+In fact when it is mounted again we don't read the value of the last BlockNumber
+and restart at zero.
+Even with *valid* BlockNumber we are not safe.
+The last blocks of the volume could be removed and we will not know.
+For that we should refresh the volume label with this information.
+This is impossible with some device like a tape.
+
+Worst it is possible to swap some block inside a file.
+An attacker could recompose a file by switching and dropping some blocks of the same file.
+
+This make our signature very weak. We only secure the front door.
+Do we still need to implement a signature?
+We don't need to decide today. What I'm doing today will have a little
+impact, not more than 4 extra bytes in the volume label that could
+be recycled for the next feature.
+
+The signature, checksum64 and encryptions together
+==================================================
+
+The question is how to order the 3 operation at backup and restore time::
+
+ uint32_t header_option; // was checksum in BB02
+ uint32_t block_len
+ uint32_t BlockNumber
+ char ID[4] // BB03
+ uint32_t VolSessionId
+ uint32_t VolSessionTime
+ uint64_t CheckSum64
+ if (header_option | BLKHOPT_SIGN_BLOCK)
+ {
+ uint32_t signature_size
+ char signature[signature_size]
+ }
+ char PAYLOAD[]
+
+notice that the checksum64 and the signature are in the middle of the block
+the only way for them to protect the entire block it to reset these
+area to zero before to calculate them.
+
+(choice 1)
+CheckSum64 should be like a hardware checksum.
+It should protect all the block, be computed last (after encryption).
+At read time it should be checked first. (before decryption)
+if we use the wrong decryption key, we will have a block
+totally messed up and we will not know
+and maybe some crash while the SD try to decode and use the *random* data
+
+(choice 2)
+But It can be done in a different way!
+Having the CheckSum64 calculated BEFORE the encryption.
+would validate that we have the right key to decrypt the block.
+In this case it should be calculated before the encryption
+and at read time it should be used after decryption.
+In this situation the key is required to verify that there
+is no "hardware" error, but reading the volume without the right
+key is useless.
+This is the choice I made at first
+I we are using the wrong key at restore time, the CheckSum64
+will detect it and bacula will not try to decode a corrupted block
+
+About the signature.
+The signature is two time optional:
+- we don't need to use a signature
+- if we have it in a volume, we don't need to verify it and we can still read the volume.
+
+The signature must be independent of encryption
+we must be able to verify the signature without having the key to decrypt the data.
+Then the signature must be calculated AFTER the encryption
+At read time it must be verified before decryption.
+
+In choice1::
+
+ do encryption
+ set checksum64 to zero
+ set signature to zero
+ calculate and fill in signature
+ calculate and fill in checksum64
+
+At restore time (choice1)::
+
+ read and copy checksum64
+ set checksum64 to zero
+ calculate & verify checksum64
+ read and copy signature
+ set signature to zero
+ calculate & verify signature <== we can stop here if we just want to check the signature
+ do decryption
+
+In choice2::
+
+ set signature to zero
+ set checksum64 to zero
+ calculate and fill in checksum64
+ do encryption
+ set signature to zero one more time
+ calculate and fill in signature
+
+At restore time (choice2)::
+
+ read and copy signature
+ set signature to zero
+ calculate & verify signature <== we can stop here if we just want to check the signature
+ do decryption
+ set signature to zero one more time
+ read and copy checksum64
+ set checksum64 to zero
+ calculate & verify checksum64
+
+Despite the *"set signature to zero one more time"*, and the fact that the CheckSum64
+don't work like a hardware checksum (that anybody can check), I prefer (choice2)
+
+Signatures
+==========
+Because we don't know in advance on which volume a block will be written,
+and because the space for the signature must be reserved before to
+start writing the records inside the volume, all the blocks written by a
+device must share the same signature size.
+Then some signature parameters must be defined at the device level.
+The Key manager can still provide the key for the signature, but
+they must match the device parameters!
+
+The signature should not prevent the volume to be read on any other
+device or machine. The information inside the volume, that describe
+the structure of the volume must be self sufficient to read the volume.
+
+If we want to change the configuration of the device, for example
+to increase the "security" by increasing the hash or the key size,
+the old volumes should be able to be read on this device.
+
+
+
+https://medium.com/@bn121rajesh/rsa-sign-and-verify-using-openssl-behind-the-scene-bf3cac0aade2
+https://github.com/bn121rajesh/ipython-notebooks/blob/master/BehindTheScene/OpensslSignVerify/openssl_sign_verify.ipynb
+
*/
static RES_ITEM store_items[] = {
{"Name", store_name, ITEM(res_store.hdr.name), 0, ITEM_REQUIRED, 0},
- {"Description", store_str, ITEM(res_dir.hdr.desc), 0, 0, 0},
+ {"Description", store_str, ITEM(res_store.hdr.desc), 0, 0, 0},
{"SdAddress", store_addresses_address, ITEM(res_store.sdaddrs), 0, ITEM_DEFAULT, 9103},
{"SdAddresses", store_addresses, ITEM(res_store.sdaddrs), 0, ITEM_DEFAULT, 9103},
{"Messages", store_res, ITEM(res_store.messages), R_MSGS, 0, 0},
{"DedupScrubMaximumBandwidth", store_speed, ITEM(res_store.dedup_scrub_max_bandwidth), 0, ITEM_DEFAULT, 15*1024*1024},
{"MaximumContainerSize", store_size64, ITEM(res_store.max_container_size), 0, 0, 0},
#endif
+ {"EncryptionCommand", store_strname,ITEM(res_store.encryption_command), 0, 0, 0},
{NULL, NULL, {0}, 0, 0, 0}
};
{"Dedupengine", store_res, ITEM(res_dev.dedup), R_DEDUP, 0, 0},
#endif
{"SyncOnClose", store_bit, ITEM(res_dev.cap_bits), CAP_SYNCONCLOSE, ITEM_DEFAULT, 0},
+ {"BlockEncryption", store_enctype,ITEM(res_dev.block_encryption), 0, ITEM_DEFAULT, 0},
+
{NULL, NULL, {0}, 0, 0, 0}
};
{NULL, 0}
};
+/*
+ * Device Encryption types
+ *
+ * encryption type encryption code = token
+ */
+s_kw enc_types[] = {
+ {"None", ET_NONE},
+ {"Enable", ET_ENABLE},
+ {"Strong", ET_STRONG},
+ {NULL, 0}
+};
+
+/*
+ * Store Encryption Type (None, Enable, Strong)
+ *
+ */
+void store_enctype(LEX *lc, RES_ITEM *item, int index, int pass)
+{
+ bool found = false;
+
+ lex_get_token(lc, T_NAME);
+ /* Store the label pass 2 so that type is defined */
+ for (int i=0; enc_types[i].name; i++) {
+ if (strcasecmp(lc->str, enc_types[i].name) == 0) {
+ *(uint32_t *)(item->value) = enc_types[i].token;
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ scan_err1(lc, _("Expected a Device Encryption Type keyword, got: %s"), lc->str);
+ }
+ scan_to_eol(lc);
+ set_bit(index, res_all.hdr.item_present);
+}
/*
* Store Device Type (File, FIFO, Tape, Cloud, ...)
if (res->res_store.dedup_index_dir) {
free(res->res_store.dedup_index_dir);
}
+ if (res->res_store.encryption_command) {
+ free(res->res_store.encryption_command);
+ }
break;
case R_CLOUD:
if (res->res_cloud.host_name) {
int64_t max_container_size; /* Maximum container size then split */
bool dedup_check_hash; /* Check Hash of each chunk after rehydration */
int64_t dedup_scrub_max_bandwidth; /* Maximum disk bandwidth usable for scrub */
+
+ char *encryption_command; /* encryption key command -- external program */
};
typedef class s_res_store STORES;
bool set_vol_append_only; /* Set 'Append Only' filesystem flag for volumes */
bool set_vol_immutable; /* Set 'Immutable' filesystem flag for volumes */
bool set_vol_read_only; /* Set permission of volumes when marking them as Full/Used */
+ uint32_t block_encryption; /* call the key-manager command to get the cipher and the key to use */
utime_t min_volume_protection_time; /* Minimum Volume Protection Time */
uint32_t drive_index; /* Autochanger drive index */
uint32_t cap_bits; /* Capabilities of this device */
/* Get the size of a give resource */
int get_resource_size(int type);
+
+extern s_kw enc_types[];
+
+/* Encryption type */
+enum {
+ ET_NONE,
+ ET_ENABLE,
+ ET_STRONG,
+};
projects / thirdparty / bacula.git / commitdiff
