arch/arm64/kernel/insn.c

   1 /*
   2  * Copyright (C) 2013 Huawei Ltd.
   3  * Author: Jiang Liu <liuj97@gmail.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17 #include <linux/bitops.h>
  18 #include <linux/compiler.h>
  19 #include <linux/kernel.h>
  20 #include <linux/smp.h>
  21 #include <linux/stop_machine.h>
  22 #include <linux/uaccess.h>
  23 #include <asm/cacheflush.h>
  24 #include <asm/insn.h>
  25
  26 static int aarch64_insn_encoding_class[] = {
  27         AARCH64_INSN_CLS_UNKNOWN,
  28         AARCH64_INSN_CLS_UNKNOWN,
  29         AARCH64_INSN_CLS_UNKNOWN,
  30         AARCH64_INSN_CLS_UNKNOWN,
  31         AARCH64_INSN_CLS_LDST,
  32         AARCH64_INSN_CLS_DP_REG,
  33         AARCH64_INSN_CLS_LDST,
  34         AARCH64_INSN_CLS_DP_FPSIMD,
  35         AARCH64_INSN_CLS_DP_IMM,
  36         AARCH64_INSN_CLS_DP_IMM,
  37         AARCH64_INSN_CLS_BR_SYS,
  38         AARCH64_INSN_CLS_BR_SYS,
  39         AARCH64_INSN_CLS_LDST,
  40         AARCH64_INSN_CLS_DP_REG,
  41         AARCH64_INSN_CLS_LDST,
  42         AARCH64_INSN_CLS_DP_FPSIMD,
  43 };
  44
  45 enum aarch64_insn_encoding_class __kprobes aarch64_get_insn_class(u32 insn)
  46 {
  47         return aarch64_insn_encoding_class[(insn >> 25) & 0xf];
  48 }
  49
  50 /* NOP is an alias of HINT */
  51 bool __kprobes aarch64_insn_is_nop(u32 insn)
  52 {
  53         if (!aarch64_insn_is_hint(insn))
  54                 return false;
  55
  56         switch (insn & 0xFE0) {
  57         case AARCH64_INSN_HINT_YIELD:
  58         case AARCH64_INSN_HINT_WFE:
  59         case AARCH64_INSN_HINT_WFI:
  60         case AARCH64_INSN_HINT_SEV:
  61         case AARCH64_INSN_HINT_SEVL:
  62                 return false;
  63         default:
  64                 return true;
  65         }
  66 }
  67
  68 /*
  69  * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
  70  * little-endian.
  71  */
  72 int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
  73 {
  74         int ret;
  75         u32 val;
  76
  77         ret = probe_kernel_read(&val, addr, AARCH64_INSN_SIZE);
  78         if (!ret)
  79                 *insnp = le32_to_cpu(val);
  80
  81         return ret;
  82 }
  83
  84 int __kprobes aarch64_insn_write(void *addr, u32 insn)
  85 {
  86         insn = cpu_to_le32(insn);
  87         return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE);
  88 }
  89
  90 static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
  91 {
  92         if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS)
  93                 return false;
  94
  95         return  aarch64_insn_is_b(insn) ||
  96                 aarch64_insn_is_bl(insn) ||
  97                 aarch64_insn_is_svc(insn) ||
  98                 aarch64_insn_is_hvc(insn) ||
  99                 aarch64_insn_is_smc(insn) ||
 100                 aarch64_insn_is_brk(insn) ||
 101                 aarch64_insn_is_nop(insn);
 102 }
 103
 104 /*
 105  * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
 106  * Section B2.6.5 "Concurrent modification and execution of instructions":
 107  * Concurrent modification and execution of instructions can lead to the
 108  * resulting instruction performing any behavior that can be achieved by
 109  * executing any sequence of instructions that can be executed from the
 110  * same Exception level, except where the instruction before modification
 111  * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC,
 112  * or SMC instruction.
 113  */
 114 bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
 115 {
 116         return __aarch64_insn_hotpatch_safe(old_insn) &&
 117                __aarch64_insn_hotpatch_safe(new_insn);
 118 }
 119
 120 int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
 121 {
 122         u32 *tp = addr;
 123         int ret;
 124
 125         /* A64 instructions must be word aligned */
 126         if ((uintptr_t)tp & 0x3)
 127                 return -EINVAL;
 128
 129         ret = aarch64_insn_write(tp, insn);
 130         if (ret == 0)
 131                 flush_icache_range((uintptr_t)tp,
 132                                    (uintptr_t)tp + AARCH64_INSN_SIZE);
 133
 134         return ret;
 135 }
 136
 137 struct aarch64_insn_patch {
 138         void            **text_addrs;
 139         u32             *new_insns;
 140         int             insn_cnt;
 141         atomic_t        cpu_count;
 142 };
 143
 144 static int __kprobes aarch64_insn_patch_text_cb(void *arg)
 145 {
 146         int i, ret = 0;
 147         struct aarch64_insn_patch *pp = arg;
 148
 149         /* The first CPU becomes master */
 150         if (atomic_inc_return(&pp->cpu_count) == 1) {
 151                 for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
 152                         ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
 153                                                              pp->new_insns[i]);
 154                 /*
 155                  * aarch64_insn_patch_text_nosync() calls flush_icache_range(),
 156                  * which ends with "dsb; isb" pair guaranteeing global
 157                  * visibility.
 158                  */
 159                 /* Notify other processors with an additional increment. */
 160                 atomic_inc(&pp->cpu_count);
 161         } else {
 162                 while (atomic_read(&pp->cpu_count) <= num_online_cpus())
 163                         cpu_relax();
 164                 isb();
 165         }
 166
 167         return ret;
 168 }
 169
 170 int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
 171 {
 172         struct aarch64_insn_patch patch = {
 173                 .text_addrs = addrs,
 174                 .new_insns = insns,
 175                 .insn_cnt = cnt,
 176                 .cpu_count = ATOMIC_INIT(0),
 177         };
 178
 179         if (cnt <= 0)
 180                 return -EINVAL;
 181
 182         return stop_machine(aarch64_insn_patch_text_cb, &patch,
 183                             cpu_online_mask);
 184 }
 185
 186 int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
 187 {
 188         int ret;
 189         u32 insn;
 190
 191         /* Unsafe to patch multiple instructions without synchronizaiton */
 192         if (cnt == 1) {
 193                 ret = aarch64_insn_read(addrs[0], &insn);
 194                 if (ret)
 195                         return ret;
 196
 197                 if (aarch64_insn_hotpatch_safe(insn, insns[0])) {
 198                         /*
 199                          * ARMv8 architecture doesn't guarantee all CPUs see
 200                          * the new instruction after returning from function
 201                          * aarch64_insn_patch_text_nosync(). So send IPIs to
 202                          * all other CPUs to achieve instruction
 203                          * synchronization.
 204                          */
 205                         ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]);
 206                         kick_all_cpus_sync();
 207                         return ret;
 208                 }
 209         }
 210
 211         return aarch64_insn_patch_text_sync(addrs, insns, cnt);
 212 }
 213
 214 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
 215                                   u32 insn, u64 imm)
 216 {
 217         u32 immlo, immhi, lomask, himask, mask;
 218         int shift;
 219
 220         switch (type) {
 221         case AARCH64_INSN_IMM_ADR:
 222                 lomask = 0x3;
 223                 himask = 0x7ffff;
 224                 immlo = imm & lomask;
 225                 imm >>= 2;
 226                 immhi = imm & himask;
 227                 imm = (immlo << 24) | (immhi);
 228                 mask = (lomask << 24) | (himask);
 229                 shift = 5;
 230                 break;
 231         case AARCH64_INSN_IMM_26:
 232                 mask = BIT(26) - 1;
 233                 shift = 0;
 234                 break;
 235         case AARCH64_INSN_IMM_19:
 236                 mask = BIT(19) - 1;
 237                 shift = 5;
 238                 break;
 239         case AARCH64_INSN_IMM_16:
 240                 mask = BIT(16) - 1;
 241                 shift = 5;
 242                 break;
 243         case AARCH64_INSN_IMM_14:
 244                 mask = BIT(14) - 1;
 245                 shift = 5;
 246                 break;
 247         case AARCH64_INSN_IMM_12:
 248                 mask = BIT(12) - 1;
 249                 shift = 10;
 250                 break;
 251         case AARCH64_INSN_IMM_9:
 252                 mask = BIT(9) - 1;
 253                 shift = 12;
 254                 break;
 255         default:
 256                 pr_err("aarch64_insn_encode_immediate: unknown immediate encoding %d\n",
 257                         type);
 258                 return 0;
 259         }
 260
 261         /* Update the immediate field. */
 262         insn &= ~(mask << shift);
 263         insn |= (imm & mask) << shift;
 264
 265         return insn;
 266 }
 267
 268 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
 269                                           enum aarch64_insn_branch_type type)
 270 {
 271         u32 insn;
 272         long offset;
 273
 274         /*
 275          * PC: A 64-bit Program Counter holding the address of the current
 276          * instruction. A64 instructions must be word-aligned.
 277          */
 278         BUG_ON((pc & 0x3) || (addr & 0x3));
 279
 280         /*
 281          * B/BL support [-128M, 128M) offset
 282          * ARM64 virtual address arrangement guarantees all kernel and module
 283          * texts are within +/-128M.
 284          */
 285         offset = ((long)addr - (long)pc);
 286         BUG_ON(offset < -SZ_128M || offset >= SZ_128M);
 287
 288         if (type == AARCH64_INSN_BRANCH_LINK)
 289                 insn = aarch64_insn_get_bl_value();
 290         else
 291                 insn = aarch64_insn_get_b_value();
 292
 293         return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
 294                                              offset >> 2);
 295 }
 296
 297 u32 __kprobes aarch64_insn_gen_hint(enum aarch64_insn_hint_op op)
 298 {
 299         return aarch64_insn_get_hint_value() | op;
 300 }
 301
 302 u32 __kprobes aarch64_insn_gen_nop(void)
 303 {
 304         return aarch64_insn_gen_hint(AARCH64_INSN_HINT_NOP);
 305 }