* @tfm: crypto API transform object
* @blk_key: key for blk-crypto
*
- * Normally only one of the fields will be non-NULL.
+ * Only one of the fields is non-NULL.
*/
struct fscrypt_prepared_key {
struct crypto_sync_skcipher *tfm;
#endif
};
+/* An entry in the linked list ->mk_mode_keys */
+struct fscrypt_mode_key {
+ struct fscrypt_prepared_key key;
+ struct list_head link;
+ u8 hkdf_context;
+ u8 mode_num;
+ u8 data_unit_bits;
+};
+
/*
* fscrypt_inode_info - the "encryption key" for an inode
*
* @prep_key, depending on which encryption implementation the file will use.
*/
static inline bool
-fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
+fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key,
const struct fscrypt_inode_info *ci)
{
- /*
- * The two smp_load_acquire()'s here pair with the smp_store_release()'s
- * in fscrypt_prepare_inline_crypt_key() and fscrypt_prepare_key().
- * I.e., in some cases (namely, if this prep_key is a per-mode
- * encryption key) another task can publish blk_key or tfm concurrently,
- * executing a RELEASE barrier. We need to use smp_load_acquire() here
- * to safely ACQUIRE the memory the other task published.
- */
if (fscrypt_using_inline_encryption(ci))
- return smp_load_acquire(&prep_key->blk_key) != NULL;
- return smp_load_acquire(&prep_key->tfm) != NULL;
+ return prep_key->blk_key != NULL;
+ return prep_key->tfm != NULL;
}
#else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
}
static inline bool
-fscrypt_is_key_prepared(struct fscrypt_prepared_key *prep_key,
+fscrypt_is_key_prepared(const struct fscrypt_prepared_key *prep_key,
const struct fscrypt_inode_info *ci)
{
- return smp_load_acquire(&prep_key->tfm) != NULL;
+ return prep_key->tfm != NULL;
}
#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */
/*
* Active and structural reference counts. An active ref guarantees
* that the struct continues to exist, continues to be in the keyring
- * ->s_master_keys, and that any embedded subkeys (e.g.
- * ->mk_direct_keys) that have been prepared continue to exist.
+ * ->s_master_keys, and that any non-file-scoped subkeys (e.g.
+ * ->mk_mode_keys) that have been prepared continue to exist.
* A structural ref only guarantees that the struct continues to exist.
*
* There is one active ref associated with ->mk_present being true, and
spinlock_t mk_decrypted_inodes_lock;
/*
- * Per-mode encryption keys for the various types of encryption policies
- * that use them. Allocated and derived on-demand.
+ * A list of 'struct fscrypt_mode_key' for the (hkdf_context, mode_num,
+ * data_unit_bits, inlinecrypt) combinations that are in use for this
+ * master key, for hkdf_context in [HKDF_CONTEXT_DIRECT_KEY,
+ * HKDF_CONTEXT_IV_INO_LBLK_32_KEY, HKDF_CONTEXT_IV_INO_LBLK_64_KEY].
+ *
+ * This is a linked list and not a hash table because in practice
+ * there's just a single encryption policy per master key, using
+ * _at most_ 2 nodes in this list. Per-file keys don't use this at all.
+ *
+ * This list is append-only until the master key is fully removed, at
+ * which time the list is cleared. Before then,
+ * fscrypt_mode_key_setup_mutex synchronizes appends, and searches use
+ * the RCU read lock together with ->mk_sem held for read.
*/
- struct fscrypt_prepared_key mk_direct_keys[FSCRYPT_MODE_MAX + 1];
- struct fscrypt_prepared_key mk_iv_ino_lblk_64_keys[FSCRYPT_MODE_MAX + 1];
- struct fscrypt_prepared_key mk_iv_ino_lblk_32_keys[FSCRYPT_MODE_MAX + 1];
+ struct list_head mk_mode_keys;
/* Hash key for inode numbers. Initialized only when needed. */
siphash_key_t mk_ino_hash_key;
void fscrypt_put_master_key_activeref(struct super_block *sb,
struct fscrypt_master_key *mk)
{
- size_t i;
+ struct fscrypt_mode_key *node, *tmp;
if (!refcount_dec_and_test(&mk->mk_active_refs))
return;
/*
* No active references left, so complete the full removal of this
* fscrypt_master_key struct by removing it from the keyring and
- * destroying any subkeys embedded in it.
+ * destroying any non-file-scoped subkeys.
*/
if (WARN_ON_ONCE(!sb->s_master_keys))
WARN_ON_ONCE(mk->mk_present);
WARN_ON_ONCE(!list_empty(&mk->mk_decrypted_inodes));
- for (i = 0; i <= FSCRYPT_MODE_MAX; i++) {
- fscrypt_destroy_prepared_key(
- sb, &mk->mk_direct_keys[i]);
- fscrypt_destroy_prepared_key(
- sb, &mk->mk_iv_ino_lblk_64_keys[i]);
- fscrypt_destroy_prepared_key(
- sb, &mk->mk_iv_ino_lblk_32_keys[i]);
+ /*
+ * Destroy any non-file-scoped subkeys. Since ->mk_active_refs == 0,
+ * they're no longer referenced by any inodes. Nor can key setup run
+ * and use them again. So they're no longer needed. (This implies no
+ * concurrent readers, so we don't need list_del_rcu() for example.)
+ */
+ list_for_each_entry_safe(node, tmp, &mk->mk_mode_keys, link) {
+ fscrypt_destroy_prepared_key(sb, &node->key);
+ list_del(&node->link);
+ kfree(node);
}
memzero_explicit(&mk->mk_ino_hash_key,
sizeof(mk->mk_ino_hash_key));
INIT_LIST_HEAD(&mk->mk_decrypted_inodes);
spin_lock_init(&mk->mk_decrypted_inodes_lock);
+ INIT_LIST_HEAD(&mk->mk_mode_keys);
+
if (mk_spec->type == FSCRYPT_KEY_SPEC_TYPE_IDENTIFIER) {
err = allocate_master_key_users_keyring(mk);
if (err)
tfm = fscrypt_allocate_skcipher(ci->ci_mode, raw_key, ci->ci_inode);
if (IS_ERR(tfm))
return PTR_ERR(tfm);
- /*
- * Pairs with the smp_load_acquire() in fscrypt_is_key_prepared().
- * I.e., here we publish ->tfm with a RELEASE barrier so that
- * concurrent tasks can ACQUIRE it. Note that this concurrency is only
- * possible for per-mode keys, not for per-file keys.
- */
- smp_store_release(&prep_key->tfm, tfm);
+ prep_key->tfm = tfm;
return 0;
}
return fscrypt_prepare_key(&ci->ci_enc_key, raw_key, ci);
}
+/*
+ * Find the fscrypt_prepared_key (if any) for a particular (mk, hkdf_context,
+ * mode_num, data_unit_bits, inlinecrypt) combination.
+ *
+ * The caller must hold ->mk_sem for reading and ->mk_present must be true,
+ * ensuring that ->mk_mode_keys is still append-only.
+ */
+static struct fscrypt_prepared_key *
+fscrypt_find_mode_key(struct fscrypt_master_key *mk, u8 hkdf_context,
+ u8 mode_num, const struct fscrypt_inode_info *ci)
+{
+ struct fscrypt_mode_key *node;
+
+ /*
+ * The RCU read lock here is used only to synchronize with concurrent
+ * list_add_tail_rcu(). Concurrent deletions are impossible here, so
+ * returning a pointer to a node without taking any refcount is safe.
+ */
+ guard(rcu)();
+ list_for_each_entry_rcu(node, &mk->mk_mode_keys, link) {
+ if (node->hkdf_context == hkdf_context &&
+ node->mode_num == mode_num &&
+ node->data_unit_bits == ci->ci_data_unit_bits &&
+ fscrypt_is_key_prepared(&node->key, ci))
+ return &node->key;
+ }
+ return NULL;
+}
+
static int setup_per_mode_enc_key(struct fscrypt_inode_info *ci,
struct fscrypt_master_key *mk,
- struct fscrypt_prepared_key *keys,
u8 hkdf_context, bool include_fs_uuid)
{
const struct inode *inode = ci->ci_inode;
struct fscrypt_mode *mode = ci->ci_mode;
const u8 mode_num = mode - fscrypt_modes;
struct fscrypt_prepared_key *prep_key;
- u8 mode_key[FSCRYPT_MAX_RAW_KEY_SIZE];
+ struct fscrypt_mode_key *new_node;
+ u8 raw_mode_key[FSCRYPT_MAX_RAW_KEY_SIZE];
u8 hkdf_info[sizeof(mode_num) + sizeof(sb->s_uuid)];
unsigned int hkdf_infolen = 0;
bool use_hw_wrapped_key = false;
use_hw_wrapped_key = true;
}
- prep_key = &keys[mode_num];
- if (fscrypt_is_key_prepared(prep_key, ci)) {
+ prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci);
+ if (prep_key) {
ci->ci_enc_key = *prep_key;
return 0;
}
- mutex_lock(&fscrypt_mode_key_setup_mutex);
+ guard(mutex)(&fscrypt_mode_key_setup_mutex);
- if (fscrypt_is_key_prepared(prep_key, ci))
- goto done_unlock;
+ prep_key = fscrypt_find_mode_key(mk, hkdf_context, mode_num, ci);
+ if (prep_key) {
+ ci->ci_enc_key = *prep_key;
+ return 0;
+ }
+
+ new_node = kzalloc_obj(*new_node);
+ if (!new_node)
+ return -ENOMEM;
+ new_node->hkdf_context = hkdf_context;
+ new_node->mode_num = mode_num;
+ new_node->data_unit_bits = ci->ci_data_unit_bits;
+ prep_key = &new_node->key;
if (use_hw_wrapped_key) {
err = fscrypt_prepare_inline_crypt_key(prep_key,
mk->mk_secret.bytes,
mk->mk_secret.size, true,
ci);
- if (err)
- goto out_unlock;
- goto done_unlock;
+ } else {
+ static_assert(sizeof(mode_num) == 1);
+ static_assert(sizeof(sb->s_uuid) == 16);
+ static_assert(sizeof(hkdf_info) == 17);
+ hkdf_info[hkdf_infolen++] = mode_num;
+ if (include_fs_uuid) {
+ memcpy(&hkdf_info[hkdf_infolen], &sb->s_uuid,
+ sizeof(sb->s_uuid));
+ hkdf_infolen += sizeof(sb->s_uuid);
+ }
+ fscrypt_hkdf_expand(&mk->mk_secret.hkdf, hkdf_context,
+ hkdf_info, hkdf_infolen, raw_mode_key,
+ mode->keysize);
+ err = fscrypt_prepare_key(prep_key, raw_mode_key, ci);
+ memzero_explicit(raw_mode_key, mode->keysize);
}
-
- BUILD_BUG_ON(sizeof(mode_num) != 1);
- BUILD_BUG_ON(sizeof(sb->s_uuid) != 16);
- BUILD_BUG_ON(sizeof(hkdf_info) != 17);
- hkdf_info[hkdf_infolen++] = mode_num;
- if (include_fs_uuid) {
- memcpy(&hkdf_info[hkdf_infolen], &sb->s_uuid,
- sizeof(sb->s_uuid));
- hkdf_infolen += sizeof(sb->s_uuid);
+ if (err) {
+ kfree(new_node);
+ return err;
}
- fscrypt_hkdf_expand(&mk->mk_secret.hkdf, hkdf_context, hkdf_info,
- hkdf_infolen, mode_key, mode->keysize);
- err = fscrypt_prepare_key(prep_key, mode_key, ci);
- memzero_explicit(mode_key, mode->keysize);
- if (err)
- goto out_unlock;
-done_unlock:
+ list_add_tail_rcu(&new_node->link, &mk->mk_mode_keys);
ci->ci_enc_key = *prep_key;
- err = 0;
-out_unlock:
- mutex_unlock(&fscrypt_mode_key_setup_mutex);
- return err;
+ return 0;
}
/*
{
int err;
- err = setup_per_mode_enc_key(ci, mk, mk->mk_iv_ino_lblk_32_keys,
- HKDF_CONTEXT_IV_INO_LBLK_32_KEY, true);
+ err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_IV_INO_LBLK_32_KEY,
+ true);
if (err)
return err;
* encryption key. This ensures that the master key is
* consistently used only for HKDF, avoiding key reuse issues.
*/
- err = setup_per_mode_enc_key(ci, mk, mk->mk_direct_keys,
- HKDF_CONTEXT_DIRECT_KEY, false);
+ err = setup_per_mode_enc_key(ci, mk, HKDF_CONTEXT_DIRECT_KEY,
+ false);
} else if (ci->ci_policy.v2.flags &
FSCRYPT_POLICY_FLAG_IV_INO_LBLK_64) {
/*
* the IVs. This format is optimized for use with inline
* encryption hardware compliant with the UFS standard.
*/
- err = setup_per_mode_enc_key(ci, mk, mk->mk_iv_ino_lblk_64_keys,
- HKDF_CONTEXT_IV_INO_LBLK_64_KEY,
- true);
+ err = setup_per_mode_enc_key(
+ ci, mk, HKDF_CONTEXT_IV_INO_LBLK_64_KEY, true);
} else if (ci->ci_policy.v2.flags &
FSCRYPT_POLICY_FLAG_IV_INO_LBLK_32) {
err = fscrypt_setup_iv_ino_lblk_32_key(ci, mk);
projects / thirdparty / kernel / stable.git / commitdiff
