releases/4.19.33/powerpc-bpf-fix-generation-of-load-store-dw-instructions.patch

   1 From 86be36f6502c52ddb4b85938145324fd07332da1 Mon Sep 17 00:00:00 2001
   2 From: "Naveen N. Rao" <naveen.n.rao@linux.vnet.ibm.com>
   3 Date: Fri, 15 Mar 2019 20:21:19 +0530
   4 Subject: powerpc: bpf: Fix generation of load/store DW instructions
   5
   6 From: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
   7
   8 commit 86be36f6502c52ddb4b85938145324fd07332da1 upstream.
   9
  10 Yauheni Kaliuta pointed out that PTR_TO_STACK store/load verifier test
  11 was failing on powerpc64 BE, and rightfully indicated that the PPC_LD()
  12 macro is not masking away the last two bits of the offset per the ISA,
  13 resulting in the generation of 'lwa' instruction instead of the intended
  14 'ld' instruction.
  15
  16 Segher also pointed out that we can't simply mask away the last two bits
  17 as that will result in loading/storing from/to a memory location that
  18 was not intended.
  19
  20 This patch addresses this by using ldx/stdx if the offset is not
  21 word-aligned. We load the offset into a temporary register (TMP_REG_2)
  22 and use that as the index register in a subsequent ldx/stdx. We fix
  23 PPC_LD() macro to mask off the last two bits, but enhance PPC_BPF_LL()
  24 and PPC_BPF_STL() to factor in the offset value and generate the proper
  25 instruction sequence. We also convert all existing users of PPC_LD() and
  26 PPC_STD() to use these macros. All existing uses of these macros have
  27 been audited to ensure that TMP_REG_2 can be clobbered.
  28
  29 Fixes: 156d0e290e96 ("powerpc/ebpf/jit: Implement JIT compiler for extended BPF")
  30 Cc: stable@vger.kernel.org # v4.9+
  31
  32 Reported-by: Yauheni Kaliuta <yauheni.kaliuta@redhat.com>
  33 Signed-off-by: Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com>
  34 Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
  35 Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
  36
  37 ---
  38  arch/powerpc/include/asm/ppc-opcode.h |    2 ++
  39  arch/powerpc/net/bpf_jit.h            |   17 +++++------------
  40  arch/powerpc/net/bpf_jit32.h          |    4 ++++
  41  arch/powerpc/net/bpf_jit64.h          |   20 ++++++++++++++++++++
  42  arch/powerpc/net/bpf_jit_comp64.c     |   12 ++++++------
  43  5 files changed, 37 insertions(+), 18 deletions(-)
  44
  45 --- a/arch/powerpc/include/asm/ppc-opcode.h
  46 +++ b/arch/powerpc/include/asm/ppc-opcode.h
  47 @@ -300,6 +300,7 @@
  48  /* Misc instructions for BPF compiler */
  49  #define PPC_INST_LBZ                   0x88000000
  50  #define PPC_INST_LD                    0xe8000000
  51 +#define PPC_INST_LDX                   0x7c00002a
  52  #define PPC_INST_LHZ                   0xa0000000
  53  #define PPC_INST_LWZ                   0x80000000
  54  #define PPC_INST_LHBRX                 0x7c00062c
  55 @@ -307,6 +308,7 @@
  56  #define PPC_INST_STB                   0x98000000
  57  #define PPC_INST_STH                   0xb0000000
  58  #define PPC_INST_STD                   0xf8000000
  59 +#define PPC_INST_STDX                  0x7c00012a
  60  #define PPC_INST_STDU                  0xf8000001
  61  #define PPC_INST_STW                   0x90000000
  62  #define PPC_INST_STWU                  0x94000000
  63 --- a/arch/powerpc/net/bpf_jit.h
  64 +++ b/arch/powerpc/net/bpf_jit.h
  65 @@ -51,6 +51,8 @@
  66  #define PPC_LIS(r, i)          PPC_ADDIS(r, 0, i)
  67  #define PPC_STD(r, base, i)    EMIT(PPC_INST_STD | ___PPC_RS(r) |            \
  68                                      ___PPC_RA(base) | ((i) & 0xfffc))
  69 +#define PPC_STDX(r, base, b)   EMIT(PPC_INST_STDX | ___PPC_RS(r) |           \
  70 +                                    ___PPC_RA(base) | ___PPC_RB(b))
  71  #define PPC_STDU(r, base, i)   EMIT(PPC_INST_STDU | ___PPC_RS(r) |           \
  72                                      ___PPC_RA(base) | ((i) & 0xfffc))
  73  #define PPC_STW(r, base, i)    EMIT(PPC_INST_STW | ___PPC_RS(r) |            \
  74 @@ -65,7 +67,9 @@
  75  #define PPC_LBZ(r, base, i)    EMIT(PPC_INST_LBZ | ___PPC_RT(r) |            \
  76                                      ___PPC_RA(base) | IMM_L(i))
  77  #define PPC_LD(r, base, i)     EMIT(PPC_INST_LD | ___PPC_RT(r) |             \
  78 -                                    ___PPC_RA(base) | IMM_L(i))
  79 +                                    ___PPC_RA(base) | ((i) & 0xfffc))
  80 +#define PPC_LDX(r, base, b)    EMIT(PPC_INST_LDX | ___PPC_RT(r) |            \
  81 +                                    ___PPC_RA(base) | ___PPC_RB(b))
  82  #define PPC_LWZ(r, base, i)    EMIT(PPC_INST_LWZ | ___PPC_RT(r) |            \
  83                                      ___PPC_RA(base) | IMM_L(i))
  84  #define PPC_LHZ(r, base, i)    EMIT(PPC_INST_LHZ | ___PPC_RT(r) |            \
  85 @@ -85,17 +89,6 @@
  86                                         ___PPC_RA(a) | ___PPC_RB(b))
  87  #define PPC_BPF_STDCX(s, a, b) EMIT(PPC_INST_STDCX | ___PPC_RS(s) |          \
  88                                         ___PPC_RA(a) | ___PPC_RB(b))
  89 -
  90 -#ifdef CONFIG_PPC64
  91 -#define PPC_BPF_LL(r, base, i) do { PPC_LD(r, base, i); } while(0)
  92 -#define PPC_BPF_STL(r, base, i) do { PPC_STD(r, base, i); } while(0)
  93 -#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0)
  94 -#else
  95 -#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0)
  96 -#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0)
  97 -#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0)
  98 -#endif
  99 -
 100  #define PPC_CMPWI(a, i)                EMIT(PPC_INST_CMPWI | ___PPC_RA(a) | IMM_L(i))
 101  #define PPC_CMPDI(a, i)                EMIT(PPC_INST_CMPDI | ___PPC_RA(a) | IMM_L(i))
 102  #define PPC_CMPW(a, b)         EMIT(PPC_INST_CMPW | ___PPC_RA(a) |           \
 103 --- a/arch/powerpc/net/bpf_jit32.h
 104 +++ b/arch/powerpc/net/bpf_jit32.h
 105 @@ -123,6 +123,10 @@ DECLARE_LOAD_FUNC(sk_load_byte_msh);
 106  #define PPC_NTOHS_OFFS(r, base, i)     PPC_LHZ_OFFS(r, base, i)
 107  #endif
 108
 109 +#define PPC_BPF_LL(r, base, i) do { PPC_LWZ(r, base, i); } while(0)
 110 +#define PPC_BPF_STL(r, base, i) do { PPC_STW(r, base, i); } while(0)
 111 +#define PPC_BPF_STLU(r, base, i) do { PPC_STWU(r, base, i); } while(0)
 112 +
 113  #define SEEN_DATAREF 0x10000 /* might call external helpers */
 114  #define SEEN_XREG    0x20000 /* X reg is used */
 115  #define SEEN_MEM     0x40000 /* SEEN_MEM+(1<<n) = use mem[n] for temporary
 116 --- a/arch/powerpc/net/bpf_jit64.h
 117 +++ b/arch/powerpc/net/bpf_jit64.h
 118 @@ -68,6 +68,26 @@ static const int b2p[] = {
 119  /* PPC NVR range -- update this if we ever use NVRs below r27 */
 120  #define BPF_PPC_NVR_MIN                27
 121
 122 +/*
 123 + * WARNING: These can use TMP_REG_2 if the offset is not at word boundary,
 124 + * so ensure that it isn't in use already.
 125 + */
 126 +#define PPC_BPF_LL(r, base, i) do {                                          \
 127 +                               if ((i) % 4) {                                \
 128 +                                       PPC_LI(b2p[TMP_REG_2], (i));          \
 129 +                                       PPC_LDX(r, base, b2p[TMP_REG_2]);     \
 130 +                               } else                                        \
 131 +                                       PPC_LD(r, base, i);                   \
 132 +                               } while(0)
 133 +#define PPC_BPF_STL(r, base, i) do {                                         \
 134 +                               if ((i) % 4) {                                \
 135 +                                       PPC_LI(b2p[TMP_REG_2], (i));          \
 136 +                                       PPC_STDX(r, base, b2p[TMP_REG_2]);    \
 137 +                               } else                                        \
 138 +                                       PPC_STD(r, base, i);                  \
 139 +                               } while(0)
 140 +#define PPC_BPF_STLU(r, base, i) do { PPC_STDU(r, base, i); } while(0)
 141 +
 142  #define SEEN_FUNC      0x1000 /* might call external helpers */
 143  #define SEEN_STACK     0x2000 /* uses BPF stack */
 144  #define SEEN_TAILCALL  0x4000 /* uses tail calls */
 145 --- a/arch/powerpc/net/bpf_jit_comp64.c
 146 +++ b/arch/powerpc/net/bpf_jit_comp64.c
 147 @@ -226,7 +226,7 @@ static void bpf_jit_emit_tail_call(u32 *
 148          * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
 149          *   goto out;
 150          */
 151 -       PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
 152 +       PPC_BPF_LL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx));
 153         PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT);
 154         PPC_BCC(COND_GT, out);
 155
 156 @@ -239,7 +239,7 @@ static void bpf_jit_emit_tail_call(u32 *
 157         /* prog = array->ptrs[index]; */
 158         PPC_MULI(b2p[TMP_REG_1], b2p_index, 8);
 159         PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array);
 160 -       PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
 161 +       PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs));
 162
 163         /*
 164          * if (prog == NULL)
 165 @@ -249,7 +249,7 @@ static void bpf_jit_emit_tail_call(u32 *
 166         PPC_BCC(COND_EQ, out);
 167
 168         /* goto *(prog->bpf_func + prologue_size); */
 169 -       PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
 170 +       PPC_BPF_LL(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func));
 171  #ifdef PPC64_ELF_ABI_v1
 172         /* skip past the function descriptor */
 173         PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1],
 174 @@ -573,7 +573,7 @@ bpf_alu32_trunc:
 175                                  * the instructions generated will remain the
 176                                  * same across all passes
 177                                  */
 178 -                               PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx));
 179 +                               PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
 180                                 PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
 181                                 PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
 182                                 break;
 183 @@ -629,7 +629,7 @@ emit_clear:
 184                                 PPC_LI32(b2p[TMP_REG_1], imm);
 185                                 src_reg = b2p[TMP_REG_1];
 186                         }
 187 -                       PPC_STD(src_reg, dst_reg, off);
 188 +                       PPC_BPF_STL(src_reg, dst_reg, off);
 189                         break;
 190
 191                 /*
 192 @@ -676,7 +676,7 @@ emit_clear:
 193                         break;
 194                 /* dst = *(u64 *)(ul) (src + off) */
 195                 case BPF_LDX | BPF_MEM | BPF_DW:
 196 -                       PPC_LD(dst_reg, src_reg, off);
 197 +                       PPC_BPF_LL(dst_reg, src_reg, off);
 198                         break;
 199
 200                 /*