drivers/net/wireguard/send.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4  */
   5
   6 #include "queueing.h"
   7 #include "timers.h"
   8 #include "device.h"
   9 #include "peer.h"
  10 #include "socket.h"
  11 #include "messages.h"
  12 #include "cookie.h"
  13
  14 #include <linux/uio.h>
  15 #include <linux/inetdevice.h>
  16 #include <linux/socket.h>
  17 #include <net/ip_tunnels.h>
  18 #include <net/udp.h>
  19 #include <net/sock.h>
  20
  21 static void wg_packet_send_handshake_initiation(struct wg_peer *peer)
  22 {
  23         struct message_handshake_initiation packet;
  24
  25         if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
  26                                       REKEY_TIMEOUT))
  27                 return; /* This function is rate limited. */
  28
  29         atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
  30         net_dbg_ratelimited("%s: Sending handshake initiation to peer %llu (%pISpfsc)\n",
  31                             peer->device->dev->name, peer->internal_id,
  32                             &peer->endpoint.addr);
  33
  34         if (wg_noise_handshake_create_initiation(&packet, &peer->handshake)) {
  35                 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
  36                 wg_timers_any_authenticated_packet_traversal(peer);
  37                 wg_timers_any_authenticated_packet_sent(peer);
  38                 atomic64_set(&peer->last_sent_handshake,
  39                              ktime_get_coarse_boottime_ns());
  40                 wg_socket_send_buffer_to_peer(peer, &packet, sizeof(packet),
  41                                               HANDSHAKE_DSCP);
  42                 wg_timers_handshake_initiated(peer);
  43         }
  44 }
  45
  46 void wg_packet_handshake_send_worker(struct work_struct *work)
  47 {
  48         struct wg_peer *peer = container_of(work, struct wg_peer,
  49                                             transmit_handshake_work);
  50
  51         wg_packet_send_handshake_initiation(peer);
  52         wg_peer_put(peer);
  53 }
  54
  55 void wg_packet_send_queued_handshake_initiation(struct wg_peer *peer,
  56                                                 bool is_retry)
  57 {
  58         if (!is_retry)
  59                 peer->timer_handshake_attempts = 0;
  60
  61         rcu_read_lock_bh();
  62         /* We check last_sent_handshake here in addition to the actual function
  63          * we're queueing up, so that we don't queue things if not strictly
  64          * necessary:
  65          */
  66         if (!wg_birthdate_has_expired(atomic64_read(&peer->last_sent_handshake),
  67                                       REKEY_TIMEOUT) ||
  68                         unlikely(READ_ONCE(peer->is_dead)))
  69                 goto out;
  70
  71         wg_peer_get(peer);
  72         /* Queues up calling packet_send_queued_handshakes(peer), where we do a
  73          * peer_put(peer) after:
  74          */
  75         if (!queue_work(peer->device->handshake_send_wq,
  76                         &peer->transmit_handshake_work))
  77                 /* If the work was already queued, we want to drop the
  78                  * extra reference:
  79                  */
  80                 wg_peer_put(peer);
  81 out:
  82         rcu_read_unlock_bh();
  83 }
  84
  85 void wg_packet_send_handshake_response(struct wg_peer *peer)
  86 {
  87         struct message_handshake_response packet;
  88
  89         atomic64_set(&peer->last_sent_handshake, ktime_get_coarse_boottime_ns());
  90         net_dbg_ratelimited("%s: Sending handshake response to peer %llu (%pISpfsc)\n",
  91                             peer->device->dev->name, peer->internal_id,
  92                             &peer->endpoint.addr);
  93
  94         if (wg_noise_handshake_create_response(&packet, &peer->handshake)) {
  95                 wg_cookie_add_mac_to_packet(&packet, sizeof(packet), peer);
  96                 if (wg_noise_handshake_begin_session(&peer->handshake,
  97                                                      &peer->keypairs)) {
  98                         wg_timers_session_derived(peer);
  99                         wg_timers_any_authenticated_packet_traversal(peer);
 100                         wg_timers_any_authenticated_packet_sent(peer);
 101                         atomic64_set(&peer->last_sent_handshake,
 102                                      ktime_get_coarse_boottime_ns());
 103                         wg_socket_send_buffer_to_peer(peer, &packet,
 104                                                       sizeof(packet),
 105                                                       HANDSHAKE_DSCP);
 106                 }
 107         }
 108 }
 109
 110 void wg_packet_send_handshake_cookie(struct wg_device *wg,
 111                                      struct sk_buff *initiating_skb,
 112                                      __le32 sender_index)
 113 {
 114         struct message_handshake_cookie packet;
 115
 116         net_dbg_skb_ratelimited("%s: Sending cookie response for denied handshake message for %pISpfsc\n",
 117                                 wg->dev->name, initiating_skb);
 118         wg_cookie_message_create(&packet, initiating_skb, sender_index,
 119                                  &wg->cookie_checker);
 120         wg_socket_send_buffer_as_reply_to_skb(wg, initiating_skb, &packet,
 121                                               sizeof(packet));
 122 }
 123
 124 static void keep_key_fresh(struct wg_peer *peer)
 125 {
 126         struct noise_keypair *keypair;
 127         bool send = false;
 128
 129         rcu_read_lock_bh();
 130         keypair = rcu_dereference_bh(peer->keypairs.current_keypair);
 131         if (likely(keypair && READ_ONCE(keypair->sending.is_valid)) &&
 132             (unlikely(atomic64_read(&keypair->sending.counter.counter) >
 133                       REKEY_AFTER_MESSAGES) ||
 134              (keypair->i_am_the_initiator &&
 135               unlikely(wg_birthdate_has_expired(keypair->sending.birthdate,
 136                                                 REKEY_AFTER_TIME)))))
 137                 send = true;
 138         rcu_read_unlock_bh();
 139
 140         if (send)
 141                 wg_packet_send_queued_handshake_initiation(peer, false);
 142 }
 143
 144 static unsigned int calculate_skb_padding(struct sk_buff *skb)
 145 {
 146         unsigned int padded_size, last_unit = skb->len;
 147
 148         if (unlikely(!PACKET_CB(skb)->mtu))
 149                 return ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE) - last_unit;
 150
 151         /* We do this modulo business with the MTU, just in case the networking
 152          * layer gives us a packet that's bigger than the MTU. In that case, we
 153          * wouldn't want the final subtraction to overflow in the case of the
 154          * padded_size being clamped. Fortunately, that's very rarely the case,
 155          * so we optimize for that not happening.
 156          */
 157         if (unlikely(last_unit > PACKET_CB(skb)->mtu))
 158                 last_unit %= PACKET_CB(skb)->mtu;
 159
 160         padded_size = min(PACKET_CB(skb)->mtu,
 161                           ALIGN(last_unit, MESSAGE_PADDING_MULTIPLE));
 162         return padded_size - last_unit;
 163 }
 164
 165 static bool encrypt_packet(struct sk_buff *skb, struct noise_keypair *keypair)
 166 {
 167         unsigned int padding_len, plaintext_len, trailer_len;
 168         struct scatterlist sg[MAX_SKB_FRAGS + 8];
 169         struct message_data *header;
 170         struct sk_buff *trailer;
 171         int num_frags;
 172
 173         /* Calculate lengths. */
 174         padding_len = calculate_skb_padding(skb);
 175         trailer_len = padding_len + noise_encrypted_len(0);
 176         plaintext_len = skb->len + padding_len;
 177
 178         /* Expand data section to have room for padding and auth tag. */
 179         num_frags = skb_cow_data(skb, trailer_len, &trailer);
 180         if (unlikely(num_frags < 0 || num_frags > ARRAY_SIZE(sg)))
 181                 return false;
 182
 183         /* Set the padding to zeros, and make sure it and the auth tag are part
 184          * of the skb.
 185          */
 186         memset(skb_tail_pointer(trailer), 0, padding_len);
 187
 188         /* Expand head section to have room for our header and the network
 189          * stack's headers.
 190          */
 191         if (unlikely(skb_cow_head(skb, DATA_PACKET_HEAD_ROOM) < 0))
 192                 return false;
 193
 194         /* Finalize checksum calculation for the inner packet, if required. */
 195         if (unlikely(skb->ip_summed == CHECKSUM_PARTIAL &&
 196                      skb_checksum_help(skb)))
 197                 return false;
 198
 199         /* Only after checksumming can we safely add on the padding at the end
 200          * and the header.
 201          */
 202         skb_set_inner_network_header(skb, 0);
 203         header = (struct message_data *)skb_push(skb, sizeof(*header));
 204         header->header.type = cpu_to_le32(MESSAGE_DATA);
 205         header->key_idx = keypair->remote_index;
 206         header->counter = cpu_to_le64(PACKET_CB(skb)->nonce);
 207         pskb_put(skb, trailer, trailer_len);
 208
 209         /* Now we can encrypt the scattergather segments */
 210         sg_init_table(sg, num_frags);
 211         if (skb_to_sgvec(skb, sg, sizeof(struct message_data),
 212                          noise_encrypted_len(plaintext_len)) <= 0)
 213                 return false;
 214         return chacha20poly1305_encrypt_sg_inplace(sg, plaintext_len, NULL, 0,
 215                                                    PACKET_CB(skb)->nonce,
 216                                                    keypair->sending.key);
 217 }
 218
 219 void wg_packet_send_keepalive(struct wg_peer *peer)
 220 {
 221         struct sk_buff *skb;
 222
 223         if (skb_queue_empty(&peer->staged_packet_queue)) {
 224                 skb = alloc_skb(DATA_PACKET_HEAD_ROOM + MESSAGE_MINIMUM_LENGTH,
 225                                 GFP_ATOMIC);
 226                 if (unlikely(!skb))
 227                         return;
 228                 skb_reserve(skb, DATA_PACKET_HEAD_ROOM);
 229                 skb->dev = peer->device->dev;
 230                 PACKET_CB(skb)->mtu = skb->dev->mtu;
 231                 skb_queue_tail(&peer->staged_packet_queue, skb);
 232                 net_dbg_ratelimited("%s: Sending keepalive packet to peer %llu (%pISpfsc)\n",
 233                                     peer->device->dev->name, peer->internal_id,
 234                                     &peer->endpoint.addr);
 235         }
 236
 237         wg_packet_send_staged_packets(peer);
 238 }
 239
 240 static void wg_packet_create_data_done(struct sk_buff *first,
 241                                        struct wg_peer *peer)
 242 {
 243         struct sk_buff *skb, *next;
 244         bool is_keepalive, data_sent = false;
 245
 246         wg_timers_any_authenticated_packet_traversal(peer);
 247         wg_timers_any_authenticated_packet_sent(peer);
 248         skb_list_walk_safe(first, skb, next) {
 249                 is_keepalive = skb->len == message_data_len(0);
 250                 if (likely(!wg_socket_send_skb_to_peer(peer, skb,
 251                                 PACKET_CB(skb)->ds) && !is_keepalive))
 252                         data_sent = true;
 253         }
 254
 255         if (likely(data_sent))
 256                 wg_timers_data_sent(peer);
 257
 258         keep_key_fresh(peer);
 259 }
 260
 261 void wg_packet_tx_worker(struct work_struct *work)
 262 {
 263         struct crypt_queue *queue = container_of(work, struct crypt_queue,
 264                                                  work);
 265         struct noise_keypair *keypair;
 266         enum packet_state state;
 267         struct sk_buff *first;
 268         struct wg_peer *peer;
 269
 270         while ((first = __ptr_ring_peek(&queue->ring)) != NULL &&
 271                (state = atomic_read_acquire(&PACKET_CB(first)->state)) !=
 272                        PACKET_STATE_UNCRYPTED) {
 273                 __ptr_ring_discard_one(&queue->ring);
 274                 peer = PACKET_PEER(first);
 275                 keypair = PACKET_CB(first)->keypair;
 276
 277                 if (likely(state == PACKET_STATE_CRYPTED))
 278                         wg_packet_create_data_done(first, peer);
 279                 else
 280                         kfree_skb_list(first);
 281
 282                 wg_noise_keypair_put(keypair, false);
 283                 wg_peer_put(peer);
 284         }
 285 }
 286
 287 void wg_packet_encrypt_worker(struct work_struct *work)
 288 {
 289         struct crypt_queue *queue = container_of(work, struct multicore_worker,
 290                                                  work)->ptr;
 291         struct sk_buff *first, *skb, *next;
 292
 293         while ((first = ptr_ring_consume_bh(&queue->ring)) != NULL) {
 294                 enum packet_state state = PACKET_STATE_CRYPTED;
 295
 296                 skb_list_walk_safe(first, skb, next) {
 297                         if (likely(encrypt_packet(skb,
 298                                         PACKET_CB(first)->keypair))) {
 299                                 wg_reset_packet(skb);
 300                         } else {
 301                                 state = PACKET_STATE_DEAD;
 302                                 break;
 303                         }
 304                 }
 305                 wg_queue_enqueue_per_peer(&PACKET_PEER(first)->tx_queue, first,
 306                                           state);
 307
 308         }
 309 }
 310
 311 static void wg_packet_create_data(struct sk_buff *first)
 312 {
 313         struct wg_peer *peer = PACKET_PEER(first);
 314         struct wg_device *wg = peer->device;
 315         int ret = -EINVAL;
 316
 317         rcu_read_lock_bh();
 318         if (unlikely(READ_ONCE(peer->is_dead)))
 319                 goto err;
 320
 321         ret = wg_queue_enqueue_per_device_and_peer(&wg->encrypt_queue,
 322                                                    &peer->tx_queue, first,
 323                                                    wg->packet_crypt_wq,
 324                                                    &wg->encrypt_queue.last_cpu);
 325         if (unlikely(ret == -EPIPE))
 326                 wg_queue_enqueue_per_peer(&peer->tx_queue, first,
 327                                           PACKET_STATE_DEAD);
 328 err:
 329         rcu_read_unlock_bh();
 330         if (likely(!ret || ret == -EPIPE))
 331                 return;
 332         wg_noise_keypair_put(PACKET_CB(first)->keypair, false);
 333         wg_peer_put(peer);
 334         kfree_skb_list(first);
 335 }
 336
 337 void wg_packet_purge_staged_packets(struct wg_peer *peer)
 338 {
 339         spin_lock_bh(&peer->staged_packet_queue.lock);
 340         peer->device->dev->stats.tx_dropped += peer->staged_packet_queue.qlen;
 341         __skb_queue_purge(&peer->staged_packet_queue);
 342         spin_unlock_bh(&peer->staged_packet_queue.lock);
 343 }
 344
 345 void wg_packet_send_staged_packets(struct wg_peer *peer)
 346 {
 347         struct noise_symmetric_key *key;
 348         struct noise_keypair *keypair;
 349         struct sk_buff_head packets;
 350         struct sk_buff *skb;
 351
 352         /* Steal the current queue into our local one. */
 353         __skb_queue_head_init(&packets);
 354         spin_lock_bh(&peer->staged_packet_queue.lock);
 355         skb_queue_splice_init(&peer->staged_packet_queue, &packets);
 356         spin_unlock_bh(&peer->staged_packet_queue.lock);
 357         if (unlikely(skb_queue_empty(&packets)))
 358                 return;
 359
 360         /* First we make sure we have a valid reference to a valid key. */
 361         rcu_read_lock_bh();
 362         keypair = wg_noise_keypair_get(
 363                 rcu_dereference_bh(peer->keypairs.current_keypair));
 364         rcu_read_unlock_bh();
 365         if (unlikely(!keypair))
 366                 goto out_nokey;
 367         key = &keypair->sending;
 368         if (unlikely(!READ_ONCE(key->is_valid)))
 369                 goto out_nokey;
 370         if (unlikely(wg_birthdate_has_expired(key->birthdate,
 371                                               REJECT_AFTER_TIME)))
 372                 goto out_invalid;
 373
 374         /* After we know we have a somewhat valid key, we now try to assign
 375          * nonces to all of the packets in the queue. If we can't assign nonces
 376          * for all of them, we just consider it a failure and wait for the next
 377          * handshake.
 378          */
 379         skb_queue_walk(&packets, skb) {
 380                 /* 0 for no outer TOS: no leak. TODO: at some later point, we
 381                  * might consider using flowi->tos as outer instead.
 382                  */
 383                 PACKET_CB(skb)->ds = ip_tunnel_ecn_encap(0, ip_hdr(skb), skb);
 384                 PACKET_CB(skb)->nonce =
 385                                 atomic64_inc_return(&key->counter.counter) - 1;
 386                 if (unlikely(PACKET_CB(skb)->nonce >= REJECT_AFTER_MESSAGES))
 387                         goto out_invalid;
 388         }
 389
 390         packets.prev->next = NULL;
 391         wg_peer_get(keypair->entry.peer);
 392         PACKET_CB(packets.next)->keypair = keypair;
 393         wg_packet_create_data(packets.next);
 394         return;
 395
 396 out_invalid:
 397         WRITE_ONCE(key->is_valid, false);
 398 out_nokey:
 399         wg_noise_keypair_put(keypair, false);
 400
 401         /* We orphan the packets if we're waiting on a handshake, so that they
 402          * don't block a socket's pool.
 403          */
 404         skb_queue_walk(&packets, skb)
 405                 skb_orphan(skb);
 406         /* Then we put them back on the top of the queue. We're not too
 407          * concerned about accidentally getting things a little out of order if
 408          * packets are being added really fast, because this queue is for before
 409          * packets can even be sent and it's small anyway.
 410          */
 411         spin_lock_bh(&peer->staged_packet_queue.lock);
 412         skb_queue_splice(&packets, &peer->staged_packet_queue);
 413         spin_unlock_bh(&peer->staged_packet_queue.lock);
 414
 415         /* If we're exiting because there's something wrong with the key, it
 416          * means we should initiate a new handshake.
 417          */
 418         wg_packet_send_queued_handshake_initiation(peer, false);
 419 }