arch/arc/lib/cache.c

   1 /*
   2  * Copyright (C) 2013-2014 Synopsys, Inc. All rights reserved.
   3  *
   4  * SPDX-License-Identifier:     GPL-2.0+
   5  */
   6
   7 #include <config.h>
   8 #include <common.h>
   9 #include <linux/compiler.h>
  10 #include <linux/kernel.h>
  11 #include <asm/arcregs.h>
  12 #include <asm/cache.h>
  13
  14 /* Bit values in IC_CTRL */
  15 #define IC_CTRL_CACHE_DISABLE   (1 << 0)
  16
  17 /* Bit values in DC_CTRL */
  18 #define DC_CTRL_CACHE_DISABLE   (1 << 0)
  19 #define DC_CTRL_INV_MODE_FLUSH  (1 << 6)
  20 #define DC_CTRL_FLUSH_STATUS    (1 << 8)
  21 #define CACHE_VER_NUM_MASK      0xF
  22 #define SLC_CTRL_SB             (1 << 2)
  23
  24 #define OP_INV          0x1
  25 #define OP_FLUSH        0x2
  26 #define OP_INV_IC       0x3
  27
  28 /*
  29  * By default that variable will fall into .bss section.
  30  * But .bss section is not relocated and so it will be initilized before
  31  * relocation but will be used after being zeroed.
  32  */
  33 int l1_line_sz __section(".data");
  34 int dcache_exists __section(".data");
  35 int icache_exists __section(".data");
  36
  37 #define CACHE_LINE_MASK         (~(l1_line_sz - 1))
  38
  39 #ifdef CONFIG_ISA_ARCV2
  40 int slc_line_sz __section(".data");
  41 int slc_exists __section(".data");
  42
  43 static unsigned int __before_slc_op(const int op)
  44 {
  45         unsigned int reg = reg;
  46
  47         if (op == OP_INV) {
  48                 /*
  49                  * IM is set by default and implies Flush-n-inv
  50                  * Clear it here for vanilla inv
  51                  */
  52                 reg = read_aux_reg(ARC_AUX_SLC_CTRL);
  53                 write_aux_reg(ARC_AUX_SLC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
  54         }
  55
  56         return reg;
  57 }
  58
  59 static void __after_slc_op(const int op, unsigned int reg)
  60 {
  61         if (op & OP_FLUSH)      /* flush / flush-n-inv both wait */
  62                 while (read_aux_reg(ARC_AUX_SLC_CTRL) &
  63                        DC_CTRL_FLUSH_STATUS)
  64                         ;
  65
  66         /* Switch back to default Invalidate mode */
  67         if (op == OP_INV)
  68                 write_aux_reg(ARC_AUX_SLC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH);
  69 }
  70
  71 static inline void __slc_line_loop(unsigned long paddr, unsigned long sz,
  72                                    const int op)
  73 {
  74         unsigned int aux_cmd;
  75         int num_lines;
  76
  77 #define SLC_LINE_MASK   (~(slc_line_sz - 1))
  78
  79         aux_cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL;
  80
  81         sz += paddr & ~SLC_LINE_MASK;
  82         paddr &= SLC_LINE_MASK;
  83
  84         num_lines = DIV_ROUND_UP(sz, slc_line_sz);
  85
  86         while (num_lines-- > 0) {
  87                 write_aux_reg(aux_cmd, paddr);
  88                 paddr += slc_line_sz;
  89         }
  90 }
  91
  92 static inline void __slc_entire_op(const int cacheop)
  93 {
  94         int aux;
  95         unsigned int ctrl_reg = __before_slc_op(cacheop);
  96
  97         if (cacheop & OP_INV)   /* Inv or flush-n-inv use same cmd reg */
  98                 aux = ARC_AUX_SLC_INVALIDATE;
  99         else
 100                 aux = ARC_AUX_SLC_FLUSH;
 101
 102         write_aux_reg(aux, 0x1);
 103
 104         __after_slc_op(cacheop, ctrl_reg);
 105 }
 106
 107 static inline void __slc_line_op(unsigned long paddr, unsigned long sz,
 108                                  const int cacheop)
 109 {
 110         unsigned int ctrl_reg = __before_slc_op(cacheop);
 111         __slc_line_loop(paddr, sz, cacheop);
 112         __after_slc_op(cacheop, ctrl_reg);
 113 }
 114 #else
 115 #define __slc_entire_op(cacheop)
 116 #define __slc_line_op(paddr, sz, cacheop)
 117 #endif
 118
 119 #ifdef CONFIG_ISA_ARCV2
 120 static void read_decode_cache_bcr_arcv2(void)
 121 {
 122         union {
 123                 struct {
 124 #ifdef CONFIG_CPU_BIG_ENDIAN
 125                         unsigned int pad:24, way:2, lsz:2, sz:4;
 126 #else
 127                         unsigned int sz:4, lsz:2, way:2, pad:24;
 128 #endif
 129                 } fields;
 130                 unsigned int word;
 131         } slc_cfg;
 132
 133         union {
 134                 struct {
 135 #ifdef CONFIG_CPU_BIG_ENDIAN
 136                         unsigned int pad:24, ver:8;
 137 #else
 138                         unsigned int ver:8, pad:24;
 139 #endif
 140                 } fields;
 141                 unsigned int word;
 142         } sbcr;
 143
 144         sbcr.word = read_aux_reg(ARC_BCR_SLC);
 145         if (sbcr.fields.ver) {
 146                 slc_cfg.word = read_aux_reg(ARC_AUX_SLC_CONFIG);
 147                 slc_exists = 1;
 148                 slc_line_sz = (slc_cfg.fields.lsz == 0) ? 128 : 64;
 149         }
 150 }
 151 #endif
 152
 153 void read_decode_cache_bcr(void)
 154 {
 155         int dc_line_sz = 0, ic_line_sz = 0;
 156
 157         union {
 158                 struct {
 159 #ifdef CONFIG_CPU_BIG_ENDIAN
 160                         unsigned int pad:12, line_len:4, sz:4, config:4, ver:8;
 161 #else
 162                         unsigned int ver:8, config:4, sz:4, line_len:4, pad:12;
 163 #endif
 164                 } fields;
 165                 unsigned int word;
 166         } ibcr, dbcr;
 167
 168         ibcr.word = read_aux_reg(ARC_BCR_IC_BUILD);
 169         if (ibcr.fields.ver) {
 170                 icache_exists = 1;
 171                 l1_line_sz = ic_line_sz = 8 << ibcr.fields.line_len;
 172                 if (!ic_line_sz)
 173                         panic("Instruction exists but line length is 0\n");
 174         }
 175
 176         dbcr.word = read_aux_reg(ARC_BCR_DC_BUILD);
 177         if (dbcr.fields.ver){
 178                 dcache_exists = 1;
 179                 l1_line_sz = dc_line_sz = 16 << dbcr.fields.line_len;
 180                 if (!dc_line_sz)
 181                         panic("Data cache exists but line length is 0\n");
 182         }
 183
 184         if (ic_line_sz && dc_line_sz && (ic_line_sz != dc_line_sz))
 185                 panic("Instruction and data cache line lengths differ\n");
 186 }
 187
 188 void cache_init(void)
 189 {
 190         read_decode_cache_bcr();
 191
 192 #ifdef CONFIG_ISA_ARCV2
 193         read_decode_cache_bcr_arcv2();
 194 #endif
 195 }
 196
 197 int icache_status(void)
 198 {
 199         if (!icache_exists)
 200                 return 0;
 201
 202         if (read_aux_reg(ARC_AUX_IC_CTRL) & IC_CTRL_CACHE_DISABLE)
 203                 return 0;
 204         else
 205                 return 1;
 206 }
 207
 208 void icache_enable(void)
 209 {
 210         if (icache_exists)
 211                 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) &
 212                               ~IC_CTRL_CACHE_DISABLE);
 213 }
 214
 215 void icache_disable(void)
 216 {
 217         if (icache_exists)
 218                 write_aux_reg(ARC_AUX_IC_CTRL, read_aux_reg(ARC_AUX_IC_CTRL) |
 219                               IC_CTRL_CACHE_DISABLE);
 220 }
 221
 222 #ifndef CONFIG_SYS_DCACHE_OFF
 223 void invalidate_icache_all(void)
 224 {
 225         /* Any write to IC_IVIC register triggers invalidation of entire I$ */
 226         if (icache_status()) {
 227                 write_aux_reg(ARC_AUX_IC_IVIC, 1);
 228                 read_aux_reg(ARC_AUX_IC_CTRL);  /* blocks */
 229         }
 230 }
 231 #else
 232 void invalidate_icache_all(void)
 233 {
 234 }
 235 #endif
 236
 237 int dcache_status(void)
 238 {
 239         if (!dcache_exists)
 240                 return 0;
 241
 242         if (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_CACHE_DISABLE)
 243                 return 0;
 244         else
 245                 return 1;
 246 }
 247
 248 void dcache_enable(void)
 249 {
 250         if (!dcache_exists)
 251                 return;
 252
 253         write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) &
 254                       ~(DC_CTRL_INV_MODE_FLUSH | DC_CTRL_CACHE_DISABLE));
 255 }
 256
 257 void dcache_disable(void)
 258 {
 259         if (!dcache_exists)
 260                 return;
 261
 262         write_aux_reg(ARC_AUX_DC_CTRL, read_aux_reg(ARC_AUX_DC_CTRL) |
 263                       DC_CTRL_CACHE_DISABLE);
 264 }
 265
 266 #ifndef CONFIG_SYS_DCACHE_OFF
 267 /*
 268  * Common Helper for Line Operations on {I,D}-Cache
 269  */
 270 static inline void __cache_line_loop(unsigned long paddr, unsigned long sz,
 271                                      const int cacheop)
 272 {
 273         unsigned int aux_cmd;
 274 #if (CONFIG_ARC_MMU_VER == 3)
 275         unsigned int aux_tag;
 276 #endif
 277         int num_lines;
 278
 279         if (cacheop == OP_INV_IC) {
 280                 aux_cmd = ARC_AUX_IC_IVIL;
 281 #if (CONFIG_ARC_MMU_VER == 3)
 282                 aux_tag = ARC_AUX_IC_PTAG;
 283 #endif
 284         } else {
 285                 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */
 286                 aux_cmd = cacheop & OP_INV ? ARC_AUX_DC_IVDL : ARC_AUX_DC_FLDL;
 287 #if (CONFIG_ARC_MMU_VER == 3)
 288                 aux_tag = ARC_AUX_DC_PTAG;
 289 #endif
 290         }
 291
 292         sz += paddr & ~CACHE_LINE_MASK;
 293         paddr &= CACHE_LINE_MASK;
 294
 295         num_lines = DIV_ROUND_UP(sz, l1_line_sz);
 296
 297         while (num_lines-- > 0) {
 298 #if (CONFIG_ARC_MMU_VER == 3)
 299                 write_aux_reg(aux_tag, paddr);
 300 #endif
 301                 write_aux_reg(aux_cmd, paddr);
 302                 paddr += l1_line_sz;
 303         }
 304 }
 305
 306 static unsigned int __before_dc_op(const int op)
 307 {
 308         unsigned int reg;
 309
 310         if (op == OP_INV) {
 311                 /*
 312                  * IM is set by default and implies Flush-n-inv
 313                  * Clear it here for vanilla inv
 314                  */
 315                 reg = read_aux_reg(ARC_AUX_DC_CTRL);
 316                 write_aux_reg(ARC_AUX_DC_CTRL, reg & ~DC_CTRL_INV_MODE_FLUSH);
 317         }
 318
 319         return reg;
 320 }
 321
 322 static void __after_dc_op(const int op, unsigned int reg)
 323 {
 324         if (op & OP_FLUSH)      /* flush / flush-n-inv both wait */
 325                 while (read_aux_reg(ARC_AUX_DC_CTRL) & DC_CTRL_FLUSH_STATUS)
 326                         ;
 327
 328         /* Switch back to default Invalidate mode */
 329         if (op == OP_INV)
 330                 write_aux_reg(ARC_AUX_DC_CTRL, reg | DC_CTRL_INV_MODE_FLUSH);
 331 }
 332
 333 static inline void __dc_entire_op(const int cacheop)
 334 {
 335         int aux;
 336         unsigned int ctrl_reg = __before_dc_op(cacheop);
 337
 338         if (cacheop & OP_INV)   /* Inv or flush-n-inv use same cmd reg */
 339                 aux = ARC_AUX_DC_IVDC;
 340         else
 341                 aux = ARC_AUX_DC_FLSH;
 342
 343         write_aux_reg(aux, 0x1);
 344
 345         __after_dc_op(cacheop, ctrl_reg);
 346 }
 347
 348 static inline void __dc_line_op(unsigned long paddr, unsigned long sz,
 349                                 const int cacheop)
 350 {
 351         unsigned int ctrl_reg = __before_dc_op(cacheop);
 352         __cache_line_loop(paddr, sz, cacheop);
 353         __after_dc_op(cacheop, ctrl_reg);
 354 }
 355 #else
 356 #define __dc_entire_op(cacheop)
 357 #define __dc_line_op(paddr, sz, cacheop)
 358 #endif /* !CONFIG_SYS_DCACHE_OFF */
 359
 360 void invalidate_dcache_range(unsigned long start, unsigned long end)
 361 {
 362         __dc_line_op(start, end - start, OP_INV);
 363 #ifdef CONFIG_ISA_ARCV2
 364         if (slc_exists)
 365                 __slc_line_op(start, end - start, OP_INV);
 366 #endif
 367 }
 368
 369 void flush_dcache_range(unsigned long start, unsigned long end)
 370 {
 371         __dc_line_op(start, end - start, OP_FLUSH);
 372 #ifdef CONFIG_ISA_ARCV2
 373         if (slc_exists)
 374                 __slc_line_op(start, end - start, OP_FLUSH);
 375 #endif
 376 }
 377
 378 void flush_cache(unsigned long start, unsigned long size)
 379 {
 380         flush_dcache_range(start, start + size);
 381 }
 382
 383 void invalidate_dcache_all(void)
 384 {
 385         __dc_entire_op(OP_INV);
 386 #ifdef CONFIG_ISA_ARCV2
 387         if (slc_exists)
 388                 __slc_entire_op(OP_INV);
 389 #endif
 390 }
 391
 392 void flush_dcache_all(void)
 393 {
 394         __dc_entire_op(OP_FLUSH);
 395 #ifdef CONFIG_ISA_ARCV2
 396         if (slc_exists)
 397                 __slc_entire_op(OP_FLUSH);
 398 #endif
 399 }