arch/powerpc/kernel/head_40x.S

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 /*
   3  *    Copyright (c) 1995-1996 Gary Thomas <gdt@linuxppc.org>
   4  *      Initial PowerPC version.
   5  *    Copyright (c) 1996 Cort Dougan <cort@cs.nmt.edu>
   6  *      Rewritten for PReP
   7  *    Copyright (c) 1996 Paul Mackerras <paulus@cs.anu.edu.au>
   8  *      Low-level exception handers, MMU support, and rewrite.
   9  *    Copyright (c) 1997 Dan Malek <dmalek@jlc.net>
  10  *      PowerPC 8xx modifications.
  11  *    Copyright (c) 1998-1999 TiVo, Inc.
  12  *      PowerPC 403GCX modifications.
  13  *    Copyright (c) 1999 Grant Erickson <grant@lcse.umn.edu>
  14  *      PowerPC 403GCX/405GP modifications.
  15  *    Copyright 2000 MontaVista Software Inc.
  16  *      PPC405 modifications
  17  *      PowerPC 403GCX/405GP modifications.
  18  *      Author: MontaVista Software, Inc.
  19  *              frank_rowand@mvista.com or source@mvista.com
  20  *              debbie_chu@mvista.com
  21  *
  22  *    Module name: head_4xx.S
  23  *
  24  *    Description:
  25  *      Kernel execution entry point code.
  26  */
  27
  28 #include <linux/init.h>
  29 #include <asm/processor.h>
  30 #include <asm/page.h>
  31 #include <asm/mmu.h>
  32 #include <asm/pgtable.h>
  33 #include <asm/cputable.h>
  34 #include <asm/thread_info.h>
  35 #include <asm/ppc_asm.h>
  36 #include <asm/asm-offsets.h>
  37 #include <asm/ptrace.h>
  38 #include <asm/export.h>
  39 #include <asm/asm-405.h>
  40
  41 #include "head_32.h"
  42
  43 /* As with the other PowerPC ports, it is expected that when code
  44  * execution begins here, the following registers contain valid, yet
  45  * optional, information:
  46  *
  47  *   r3 - Board info structure pointer (DRAM, frequency, MAC address, etc.)
  48  *   r4 - Starting address of the init RAM disk
  49  *   r5 - Ending address of the init RAM disk
  50  *   r6 - Start of kernel command line string (e.g. "mem=96m")
  51  *   r7 - End of kernel command line string
  52  *
  53  * This is all going to change RSN when we add bi_recs.......  -- Dan
  54  */
  55         __HEAD
  56 _ENTRY(_stext);
  57 _ENTRY(_start);
  58
  59         mr      r31,r3                  /* save device tree ptr */
  60
  61         /* We have to turn on the MMU right away so we get cache modes
  62          * set correctly.
  63          */
  64         bl      initial_mmu
  65
  66 /* We now have the lower 16 Meg mapped into TLB entries, and the caches
  67  * ready to work.
  68  */
  69 turn_on_mmu:
  70         lis     r0,MSR_KERNEL@h
  71         ori     r0,r0,MSR_KERNEL@l
  72         mtspr   SPRN_SRR1,r0
  73         lis     r0,start_here@h
  74         ori     r0,r0,start_here@l
  75         mtspr   SPRN_SRR0,r0
  76         SYNC
  77         rfi                             /* enables MMU */
  78         b       .                       /* prevent prefetch past rfi */
  79
  80 /*
  81  * This area is used for temporarily saving registers during the
  82  * critical exception prolog.
  83  */
  84         . = 0xc0
  85 crit_save:
  86 _ENTRY(crit_r10)
  87         .space  4
  88 _ENTRY(crit_r11)
  89         .space  4
  90 _ENTRY(crit_srr0)
  91         .space  4
  92 _ENTRY(crit_srr1)
  93         .space  4
  94 _ENTRY(saved_ksp_limit)
  95         .space  4
  96
  97 /*
  98  * Exception prolog for critical exceptions.  This is a little different
  99  * from the normal exception prolog above since a critical exception
 100  * can potentially occur at any point during normal exception processing.
 101  * Thus we cannot use the same SPRG registers as the normal prolog above.
 102  * Instead we use a couple of words of memory at low physical addresses.
 103  * This is OK since we don't support SMP on these processors.
 104  */
 105 #define CRITICAL_EXCEPTION_PROLOG                                            \
 106         stw     r10,crit_r10@l(0);      /* save two registers to work with */\
 107         stw     r11,crit_r11@l(0);                                           \
 108         mfcr    r10;                    /* save CR in r10 for now          */\
 109         mfspr   r11,SPRN_SRR3;          /* check whether user or kernel    */\
 110         andi.   r11,r11,MSR_PR;                                              \
 111         lis     r11,critirq_ctx@ha;                                          \
 112         tophys(r11,r11);                                                     \
 113         lwz     r11,critirq_ctx@l(r11);                                      \
 114         beq     1f;                                                          \
 115         /* COMING FROM USER MODE */                                          \
 116         mfspr   r11,SPRN_SPRG_THREAD;   /* if from user, start at top of   */\
 117         lwz     r11,TASK_STACK-THREAD(r11); /* this thread's kernel stack */\
 118 1:      addi    r11,r11,THREAD_SIZE-INT_FRAME_SIZE; /* Alloc an excpt frm  */\
 119         tophys(r11,r11);                                                     \
 120         stw     r10,_CCR(r11);          /* save various registers          */\
 121         stw     r12,GPR12(r11);                                              \
 122         stw     r9,GPR9(r11);                                                \
 123         mflr    r10;                                                         \
 124         stw     r10,_LINK(r11);                                              \
 125         mfspr   r12,SPRN_DEAR;          /* save DEAR and ESR in the frame  */\
 126         stw     r12,_DEAR(r11);         /* since they may have had stuff   */\
 127         mfspr   r9,SPRN_ESR;            /* in them at the point where the  */\
 128         stw     r9,_ESR(r11);           /* exception was taken             */\
 129         mfspr   r12,SPRN_SRR2;                                               \
 130         stw     r1,GPR1(r11);                                                \
 131         mfspr   r9,SPRN_SRR3;                                                \
 132         stw     r1,0(r11);                                                   \
 133         tovirt(r1,r11);                                                      \
 134         rlwinm  r9,r9,0,14,12;          /* clear MSR_WE (necessary?)       */\
 135         stw     r0,GPR0(r11);                                                \
 136         lis     r10, STACK_FRAME_REGS_MARKER@ha; /* exception frame marker */\
 137         addi    r10, r10, STACK_FRAME_REGS_MARKER@l;                         \
 138         stw     r10, 8(r11);                                                 \
 139         SAVE_4GPRS(3, r11);                                                  \
 140         SAVE_2GPRS(7, r11)
 141
 142         /*
 143          * State at this point:
 144          * r9 saved in stack frame, now saved SRR3 & ~MSR_WE
 145          * r10 saved in crit_r10 and in stack frame, trashed
 146          * r11 saved in crit_r11 and in stack frame,
 147          *      now phys stack/exception frame pointer
 148          * r12 saved in stack frame, now saved SRR2
 149          * CR saved in stack frame, CR0.EQ = !SRR3.PR
 150          * LR, DEAR, ESR in stack frame
 151          * r1 saved in stack frame, now virt stack/excframe pointer
 152          * r0, r3-r8 saved in stack frame
 153          */
 154
 155 /*
 156  * Exception vectors.
 157  */
 158 #define CRITICAL_EXCEPTION(n, label, hdlr)                      \
 159         START_EXCEPTION(n, label);                              \
 160         CRITICAL_EXCEPTION_PROLOG;                              \
 161         addi    r3,r1,STACK_FRAME_OVERHEAD;                     \
 162         EXC_XFER_TEMPLATE(hdlr, n+2, (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
 163                           crit_transfer_to_handler, ret_from_crit_exc)
 164
 165 /*
 166  * 0x0100 - Critical Interrupt Exception
 167  */
 168         CRITICAL_EXCEPTION(0x0100, CriticalInterrupt, unknown_exception)
 169
 170 /*
 171  * 0x0200 - Machine Check Exception
 172  */
 173         CRITICAL_EXCEPTION(0x0200, MachineCheck, machine_check_exception)
 174
 175 /*
 176  * 0x0300 - Data Storage Exception
 177  * This happens for just a few reasons.  U0 set (but we don't do that),
 178  * or zone protection fault (user violation, write to protected page).
 179  * If this is just an update of modified status, we do that quickly
 180  * and exit.  Otherwise, we call heavywight functions to do the work.
 181  */
 182         START_EXCEPTION(0x0300, DataStorage)
 183         mtspr   SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */
 184         mtspr   SPRN_SPRG_SCRATCH1, r11
 185 #ifdef CONFIG_403GCX
 186         stw     r12, 0(r0)
 187         stw     r9, 4(r0)
 188         mfcr    r11
 189         mfspr   r12, SPRN_PID
 190         stw     r11, 8(r0)
 191         stw     r12, 12(r0)
 192 #else
 193         mtspr   SPRN_SPRG_SCRATCH3, r12
 194         mtspr   SPRN_SPRG_SCRATCH4, r9
 195         mfcr    r11
 196         mfspr   r12, SPRN_PID
 197         mtspr   SPRN_SPRG_SCRATCH6, r11
 198         mtspr   SPRN_SPRG_SCRATCH5, r12
 199 #endif
 200
 201         /* First, check if it was a zone fault (which means a user
 202         * tried to access a kernel or read-protected page - always
 203         * a SEGV).  All other faults here must be stores, so no
 204         * need to check ESR_DST as well. */
 205         mfspr   r10, SPRN_ESR
 206         andis.  r10, r10, ESR_DIZ@h
 207         bne     2f
 208
 209         mfspr   r10, SPRN_DEAR          /* Get faulting address */
 210
 211         /* If we are faulting a kernel address, we have to use the
 212          * kernel page tables.
 213          */
 214         lis     r11, PAGE_OFFSET@h
 215         cmplw   r10, r11
 216         blt+    3f
 217         lis     r11, swapper_pg_dir@h
 218         ori     r11, r11, swapper_pg_dir@l
 219         li      r9, 0
 220         mtspr   SPRN_PID, r9            /* TLB will have 0 TID */
 221         b       4f
 222
 223         /* Get the PGD for the current thread.
 224          */
 225 3:
 226         mfspr   r11,SPRN_SPRG_THREAD
 227         lwz     r11,PGDIR(r11)
 228 4:
 229         tophys(r11, r11)
 230         rlwimi  r11, r10, 12, 20, 29    /* Create L1 (pgdir/pmd) address */
 231         lwz     r11, 0(r11)             /* Get L1 entry */
 232         rlwinm. r12, r11, 0, 0, 19      /* Extract L2 (pte) base address */
 233         beq     2f                      /* Bail if no table */
 234
 235         rlwimi  r12, r10, 22, 20, 29    /* Compute PTE address */
 236         lwz     r11, 0(r12)             /* Get Linux PTE */
 237
 238         andi.   r9, r11, _PAGE_RW       /* Is it writeable? */
 239         beq     2f                      /* Bail if not */
 240
 241         /* Update 'changed'.
 242         */
 243         ori     r11, r11, _PAGE_DIRTY|_PAGE_ACCESSED|_PAGE_HWWRITE
 244         stw     r11, 0(r12)             /* Update Linux page table */
 245
 246         /* Most of the Linux PTE is ready to load into the TLB LO.
 247          * We set ZSEL, where only the LS-bit determines user access.
 248          * We set execute, because we don't have the granularity to
 249          * properly set this at the page level (Linux problem).
 250          * If shared is set, we cause a zero PID->TID load.
 251          * Many of these bits are software only.  Bits we don't set
 252          * here we (properly should) assume have the appropriate value.
 253          */
 254         li      r12, 0x0ce2
 255         andc    r11, r11, r12           /* Make sure 20, 21 are zero */
 256
 257         /* find the TLB index that caused the fault.  It has to be here.
 258         */
 259         tlbsx   r9, 0, r10
 260
 261         tlbwe   r11, r9, TLB_DATA               /* Load TLB LO */
 262
 263         /* Done...restore registers and get out of here.
 264         */
 265 #ifdef CONFIG_403GCX
 266         lwz     r12, 12(r0)
 267         lwz     r11, 8(r0)
 268         mtspr   SPRN_PID, r12
 269         mtcr    r11
 270         lwz     r9, 4(r0)
 271         lwz     r12, 0(r0)
 272 #else
 273         mfspr   r12, SPRN_SPRG_SCRATCH5
 274         mfspr   r11, SPRN_SPRG_SCRATCH6
 275         mtspr   SPRN_PID, r12
 276         mtcr    r11
 277         mfspr   r9, SPRN_SPRG_SCRATCH4
 278         mfspr   r12, SPRN_SPRG_SCRATCH3
 279 #endif
 280         mfspr   r11, SPRN_SPRG_SCRATCH1
 281         mfspr   r10, SPRN_SPRG_SCRATCH0
 282         PPC405_ERR77_SYNC
 283         rfi                     /* Should sync shadow TLBs */
 284         b       .               /* prevent prefetch past rfi */
 285
 286 2:
 287         /* The bailout.  Restore registers to pre-exception conditions
 288          * and call the heavyweights to help us out.
 289          */
 290 #ifdef CONFIG_403GCX
 291         lwz     r12, 12(r0)
 292         lwz     r11, 8(r0)
 293         mtspr   SPRN_PID, r12
 294         mtcr    r11
 295         lwz     r9, 4(r0)
 296         lwz     r12, 0(r0)
 297 #else
 298         mfspr   r12, SPRN_SPRG_SCRATCH5
 299         mfspr   r11, SPRN_SPRG_SCRATCH6
 300         mtspr   SPRN_PID, r12
 301         mtcr    r11
 302         mfspr   r9, SPRN_SPRG_SCRATCH4
 303         mfspr   r12, SPRN_SPRG_SCRATCH3
 304 #endif
 305         mfspr   r11, SPRN_SPRG_SCRATCH1
 306         mfspr   r10, SPRN_SPRG_SCRATCH0
 307         b       DataAccess
 308
 309 /*
 310  * 0x0400 - Instruction Storage Exception
 311  * This is caused by a fetch from non-execute or guarded pages.
 312  */
 313         START_EXCEPTION(0x0400, InstructionAccess)
 314         EXCEPTION_PROLOG
 315         mr      r4,r12                  /* Pass SRR0 as arg2 */
 316         stw     r4, _DEAR(r11)
 317         li      r5,0                    /* Pass zero as arg3 */
 318         EXC_XFER_LITE(0x400, handle_page_fault)
 319
 320 /* 0x0500 - External Interrupt Exception */
 321         EXCEPTION(0x0500, HardwareInterrupt, do_IRQ, EXC_XFER_LITE)
 322
 323 /* 0x0600 - Alignment Exception */
 324         START_EXCEPTION(0x0600, Alignment)
 325         EXCEPTION_PROLOG
 326         mfspr   r4,SPRN_DEAR            /* Grab the DEAR and save it */
 327         stw     r4,_DEAR(r11)
 328         addi    r3,r1,STACK_FRAME_OVERHEAD
 329         EXC_XFER_STD(0x600, alignment_exception)
 330
 331 /* 0x0700 - Program Exception */
 332         START_EXCEPTION(0x0700, ProgramCheck)
 333         EXCEPTION_PROLOG
 334         mfspr   r4,SPRN_ESR             /* Grab the ESR and save it */
 335         stw     r4,_ESR(r11)
 336         addi    r3,r1,STACK_FRAME_OVERHEAD
 337         EXC_XFER_STD(0x700, program_check_exception)
 338
 339         EXCEPTION(0x0800, Trap_08, unknown_exception, EXC_XFER_STD)
 340         EXCEPTION(0x0900, Trap_09, unknown_exception, EXC_XFER_STD)
 341         EXCEPTION(0x0A00, Trap_0A, unknown_exception, EXC_XFER_STD)
 342         EXCEPTION(0x0B00, Trap_0B, unknown_exception, EXC_XFER_STD)
 343
 344 /* 0x0C00 - System Call Exception */
 345         START_EXCEPTION(0x0C00, SystemCall)
 346         SYSCALL_ENTRY   0xc00
 347
 348         EXCEPTION(0x0D00, Trap_0D, unknown_exception, EXC_XFER_STD)
 349         EXCEPTION(0x0E00, Trap_0E, unknown_exception, EXC_XFER_STD)
 350         EXCEPTION(0x0F00, Trap_0F, unknown_exception, EXC_XFER_STD)
 351
 352 /* 0x1000 - Programmable Interval Timer (PIT) Exception */
 353         . = 0x1000
 354         b Decrementer
 355
 356 /* 0x1010 - Fixed Interval Timer (FIT) Exception
 357 */
 358         . = 0x1010
 359         b FITException
 360
 361 /* 0x1020 - Watchdog Timer (WDT) Exception
 362 */
 363         . = 0x1020
 364         b WDTException
 365
 366 /* 0x1100 - Data TLB Miss Exception
 367  * As the name implies, translation is not in the MMU, so search the
 368  * page tables and fix it.  The only purpose of this function is to
 369  * load TLB entries from the page table if they exist.
 370  */
 371         START_EXCEPTION(0x1100, DTLBMiss)
 372         mtspr   SPRN_SPRG_SCRATCH0, r10 /* Save some working registers */
 373         mtspr   SPRN_SPRG_SCRATCH1, r11
 374 #ifdef CONFIG_403GCX
 375         stw     r12, 0(r0)
 376         stw     r9, 4(r0)
 377         mfcr    r11
 378         mfspr   r12, SPRN_PID
 379         stw     r11, 8(r0)
 380         stw     r12, 12(r0)
 381 #else
 382         mtspr   SPRN_SPRG_SCRATCH3, r12
 383         mtspr   SPRN_SPRG_SCRATCH4, r9
 384         mfcr    r11
 385         mfspr   r12, SPRN_PID
 386         mtspr   SPRN_SPRG_SCRATCH6, r11
 387         mtspr   SPRN_SPRG_SCRATCH5, r12
 388 #endif
 389         mfspr   r10, SPRN_DEAR          /* Get faulting address */
 390
 391         /* If we are faulting a kernel address, we have to use the
 392          * kernel page tables.
 393          */
 394         lis     r11, PAGE_OFFSET@h
 395         cmplw   r10, r11
 396         blt+    3f
 397         lis     r11, swapper_pg_dir@h
 398         ori     r11, r11, swapper_pg_dir@l
 399         li      r9, 0
 400         mtspr   SPRN_PID, r9            /* TLB will have 0 TID */
 401         b       4f
 402
 403         /* Get the PGD for the current thread.
 404          */
 405 3:
 406         mfspr   r11,SPRN_SPRG_THREAD
 407         lwz     r11,PGDIR(r11)
 408 4:
 409         tophys(r11, r11)
 410         rlwimi  r11, r10, 12, 20, 29    /* Create L1 (pgdir/pmd) address */
 411         lwz     r12, 0(r11)             /* Get L1 entry */
 412         andi.   r9, r12, _PMD_PRESENT   /* Check if it points to a PTE page */
 413         beq     2f                      /* Bail if no table */
 414
 415         rlwimi  r12, r10, 22, 20, 29    /* Compute PTE address */
 416         lwz     r11, 0(r12)             /* Get Linux PTE */
 417         andi.   r9, r11, _PAGE_PRESENT
 418         beq     5f
 419
 420         ori     r11, r11, _PAGE_ACCESSED
 421         stw     r11, 0(r12)
 422
 423         /* Create TLB tag.  This is the faulting address plus a static
 424          * set of bits.  These are size, valid, E, U0.
 425         */
 426         li      r12, 0x00c0
 427         rlwimi  r10, r12, 0, 20, 31
 428
 429         b       finish_tlb_load
 430
 431 2:      /* Check for possible large-page pmd entry */
 432         rlwinm. r9, r12, 2, 22, 24
 433         beq     5f
 434
 435         /* Create TLB tag.  This is the faulting address, plus a static
 436          * set of bits (valid, E, U0) plus the size from the PMD.
 437          */
 438         ori     r9, r9, 0x40
 439         rlwimi  r10, r9, 0, 20, 31
 440         mr      r11, r12
 441
 442         b       finish_tlb_load
 443
 444 5:
 445         /* The bailout.  Restore registers to pre-exception conditions
 446          * and call the heavyweights to help us out.
 447          */
 448 #ifdef CONFIG_403GCX
 449         lwz     r12, 12(r0)
 450         lwz     r11, 8(r0)
 451         mtspr   SPRN_PID, r12
 452         mtcr    r11
 453         lwz     r9, 4(r0)
 454         lwz     r12, 0(r0)
 455 #else
 456         mfspr   r12, SPRN_SPRG_SCRATCH5
 457         mfspr   r11, SPRN_SPRG_SCRATCH6
 458         mtspr   SPRN_PID, r12
 459         mtcr    r11
 460         mfspr   r9, SPRN_SPRG_SCRATCH4
 461         mfspr   r12, SPRN_SPRG_SCRATCH3
 462 #endif
 463         mfspr   r11, SPRN_SPRG_SCRATCH1
 464         mfspr   r10, SPRN_SPRG_SCRATCH0
 465         b       DataAccess
 466
 467 /* 0x1200 - Instruction TLB Miss Exception
 468  * Nearly the same as above, except we get our information from different
 469  * registers and bailout to a different point.
 470  */
 471         START_EXCEPTION(0x1200, ITLBMiss)
 472         mtspr   SPRN_SPRG_SCRATCH0, r10  /* Save some working registers */
 473         mtspr   SPRN_SPRG_SCRATCH1, r11
 474 #ifdef CONFIG_403GCX
 475         stw     r12, 0(r0)
 476         stw     r9, 4(r0)
 477         mfcr    r11
 478         mfspr   r12, SPRN_PID
 479         stw     r11, 8(r0)
 480         stw     r12, 12(r0)
 481 #else
 482         mtspr   SPRN_SPRG_SCRATCH3, r12
 483         mtspr   SPRN_SPRG_SCRATCH4, r9
 484         mfcr    r11
 485         mfspr   r12, SPRN_PID
 486         mtspr   SPRN_SPRG_SCRATCH6, r11
 487         mtspr   SPRN_SPRG_SCRATCH5, r12
 488 #endif
 489         mfspr   r10, SPRN_SRR0          /* Get faulting address */
 490
 491         /* If we are faulting a kernel address, we have to use the
 492          * kernel page tables.
 493          */
 494         lis     r11, PAGE_OFFSET@h
 495         cmplw   r10, r11
 496         blt+    3f
 497         lis     r11, swapper_pg_dir@h
 498         ori     r11, r11, swapper_pg_dir@l
 499         li      r9, 0
 500         mtspr   SPRN_PID, r9            /* TLB will have 0 TID */
 501         b       4f
 502
 503         /* Get the PGD for the current thread.
 504          */
 505 3:
 506         mfspr   r11,SPRN_SPRG_THREAD
 507         lwz     r11,PGDIR(r11)
 508 4:
 509         tophys(r11, r11)
 510         rlwimi  r11, r10, 12, 20, 29    /* Create L1 (pgdir/pmd) address */
 511         lwz     r12, 0(r11)             /* Get L1 entry */
 512         andi.   r9, r12, _PMD_PRESENT   /* Check if it points to a PTE page */
 513         beq     2f                      /* Bail if no table */
 514
 515         rlwimi  r12, r10, 22, 20, 29    /* Compute PTE address */
 516         lwz     r11, 0(r12)             /* Get Linux PTE */
 517         andi.   r9, r11, _PAGE_PRESENT
 518         beq     5f
 519
 520         ori     r11, r11, _PAGE_ACCESSED
 521         stw     r11, 0(r12)
 522
 523         /* Create TLB tag.  This is the faulting address plus a static
 524          * set of bits.  These are size, valid, E, U0.
 525         */
 526         li      r12, 0x00c0
 527         rlwimi  r10, r12, 0, 20, 31
 528
 529         b       finish_tlb_load
 530
 531 2:      /* Check for possible large-page pmd entry */
 532         rlwinm. r9, r12, 2, 22, 24
 533         beq     5f
 534
 535         /* Create TLB tag.  This is the faulting address, plus a static
 536          * set of bits (valid, E, U0) plus the size from the PMD.
 537          */
 538         ori     r9, r9, 0x40
 539         rlwimi  r10, r9, 0, 20, 31
 540         mr      r11, r12
 541
 542         b       finish_tlb_load
 543
 544 5:
 545         /* The bailout.  Restore registers to pre-exception conditions
 546          * and call the heavyweights to help us out.
 547          */
 548 #ifdef CONFIG_403GCX
 549         lwz     r12, 12(r0)
 550         lwz     r11, 8(r0)
 551         mtspr   SPRN_PID, r12
 552         mtcr    r11
 553         lwz     r9, 4(r0)
 554         lwz     r12, 0(r0)
 555 #else
 556         mfspr   r12, SPRN_SPRG_SCRATCH5
 557         mfspr   r11, SPRN_SPRG_SCRATCH6
 558         mtspr   SPRN_PID, r12
 559         mtcr    r11
 560         mfspr   r9, SPRN_SPRG_SCRATCH4
 561         mfspr   r12, SPRN_SPRG_SCRATCH3
 562 #endif
 563         mfspr   r11, SPRN_SPRG_SCRATCH1
 564         mfspr   r10, SPRN_SPRG_SCRATCH0
 565         b       InstructionAccess
 566
 567         EXCEPTION(0x1300, Trap_13, unknown_exception, EXC_XFER_STD)
 568         EXCEPTION(0x1400, Trap_14, unknown_exception, EXC_XFER_STD)
 569         EXCEPTION(0x1500, Trap_15, unknown_exception, EXC_XFER_STD)
 570         EXCEPTION(0x1600, Trap_16, unknown_exception, EXC_XFER_STD)
 571 #ifdef CONFIG_IBM405_ERR51
 572         /* 405GP errata 51 */
 573         START_EXCEPTION(0x1700, Trap_17)
 574         b DTLBMiss
 575 #else
 576         EXCEPTION(0x1700, Trap_17, unknown_exception, EXC_XFER_STD)
 577 #endif
 578         EXCEPTION(0x1800, Trap_18, unknown_exception, EXC_XFER_STD)
 579         EXCEPTION(0x1900, Trap_19, unknown_exception, EXC_XFER_STD)
 580         EXCEPTION(0x1A00, Trap_1A, unknown_exception, EXC_XFER_STD)
 581         EXCEPTION(0x1B00, Trap_1B, unknown_exception, EXC_XFER_STD)
 582         EXCEPTION(0x1C00, Trap_1C, unknown_exception, EXC_XFER_STD)
 583         EXCEPTION(0x1D00, Trap_1D, unknown_exception, EXC_XFER_STD)
 584         EXCEPTION(0x1E00, Trap_1E, unknown_exception, EXC_XFER_STD)
 585         EXCEPTION(0x1F00, Trap_1F, unknown_exception, EXC_XFER_STD)
 586
 587 /* Check for a single step debug exception while in an exception
 588  * handler before state has been saved.  This is to catch the case
 589  * where an instruction that we are trying to single step causes
 590  * an exception (eg ITLB/DTLB miss) and thus the first instruction of
 591  * the exception handler generates a single step debug exception.
 592  *
 593  * If we get a debug trap on the first instruction of an exception handler,
 594  * we reset the MSR_DE in the _exception handler's_ MSR (the debug trap is
 595  * a critical exception, so we are using SPRN_CSRR1 to manipulate the MSR).
 596  * The exception handler was handling a non-critical interrupt, so it will
 597  * save (and later restore) the MSR via SPRN_SRR1, which will still have
 598  * the MSR_DE bit set.
 599  */
 600         /* 0x2000 - Debug Exception */
 601         START_EXCEPTION(0x2000, DebugTrap)
 602         CRITICAL_EXCEPTION_PROLOG
 603
 604         /*
 605          * If this is a single step or branch-taken exception in an
 606          * exception entry sequence, it was probably meant to apply to
 607          * the code where the exception occurred (since exception entry
 608          * doesn't turn off DE automatically).  We simulate the effect
 609          * of turning off DE on entry to an exception handler by turning
 610          * off DE in the SRR3 value and clearing the debug status.
 611          */
 612         mfspr   r10,SPRN_DBSR           /* check single-step/branch taken */
 613         andis.  r10,r10,DBSR_IC@h
 614         beq+    2f
 615
 616         andi.   r10,r9,MSR_IR|MSR_PR    /* check supervisor + MMU off */
 617         beq     1f                      /* branch and fix it up */
 618
 619         mfspr   r10,SPRN_SRR2           /* Faulting instruction address */
 620         cmplwi  r10,0x2100
 621         bgt+    2f                      /* address above exception vectors */
 622
 623         /* here it looks like we got an inappropriate debug exception. */
 624 1:      rlwinm  r9,r9,0,~MSR_DE         /* clear DE in the SRR3 value */
 625         lis     r10,DBSR_IC@h           /* clear the IC event */
 626         mtspr   SPRN_DBSR,r10
 627         /* restore state and get out */
 628         lwz     r10,_CCR(r11)
 629         lwz     r0,GPR0(r11)
 630         lwz     r1,GPR1(r11)
 631         mtcrf   0x80,r10
 632         mtspr   SPRN_SRR2,r12
 633         mtspr   SPRN_SRR3,r9
 634         lwz     r9,GPR9(r11)
 635         lwz     r12,GPR12(r11)
 636         lwz     r10,crit_r10@l(0)
 637         lwz     r11,crit_r11@l(0)
 638         PPC405_ERR77_SYNC
 639         rfci
 640         b       .
 641
 642         /* continue normal handling for a critical exception... */
 643 2:      mfspr   r4,SPRN_DBSR
 644         addi    r3,r1,STACK_FRAME_OVERHEAD
 645         EXC_XFER_TEMPLATE(DebugException, 0x2002, \
 646                 (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)), \
 647                 crit_transfer_to_handler, ret_from_crit_exc)
 648
 649         /* Programmable Interval Timer (PIT) Exception. (from 0x1000) */
 650 Decrementer:
 651         EXCEPTION_PROLOG
 652         lis     r0,TSR_PIS@h
 653         mtspr   SPRN_TSR,r0             /* Clear the PIT exception */
 654         addi    r3,r1,STACK_FRAME_OVERHEAD
 655         EXC_XFER_LITE(0x1000, timer_interrupt)
 656
 657         /* Fixed Interval Timer (FIT) Exception. (from 0x1010) */
 658 FITException:
 659         EXCEPTION_PROLOG
 660         addi    r3,r1,STACK_FRAME_OVERHEAD;
 661         EXC_XFER_STD(0x1010, unknown_exception)
 662
 663         /* Watchdog Timer (WDT) Exception. (from 0x1020) */
 664 WDTException:
 665         CRITICAL_EXCEPTION_PROLOG;
 666         addi    r3,r1,STACK_FRAME_OVERHEAD;
 667         EXC_XFER_TEMPLATE(WatchdogException, 0x1020+2,
 668                           (MSR_KERNEL & ~(MSR_ME|MSR_DE|MSR_CE)),
 669                           crit_transfer_to_handler, ret_from_crit_exc)
 670
 671 /*
 672  * The other Data TLB exceptions bail out to this point
 673  * if they can't resolve the lightweight TLB fault.
 674  */
 675 DataAccess:
 676         EXCEPTION_PROLOG
 677         mfspr   r5,SPRN_ESR             /* Grab the ESR, save it, pass arg3 */
 678         stw     r5,_ESR(r11)
 679         mfspr   r4,SPRN_DEAR            /* Grab the DEAR, save it, pass arg2 */
 680         stw     r4, _DEAR(r11)
 681         EXC_XFER_LITE(0x300, handle_page_fault)
 682
 683 /* Other PowerPC processors, namely those derived from the 6xx-series
 684  * have vectors from 0x2100 through 0x2F00 defined, but marked as reserved.
 685  * However, for the 4xx-series processors these are neither defined nor
 686  * reserved.
 687  */
 688
 689         /* Damn, I came up one instruction too many to fit into the
 690          * exception space :-).  Both the instruction and data TLB
 691          * miss get to this point to load the TLB.
 692          *      r10 - TLB_TAG value
 693          *      r11 - Linux PTE
 694          *      r12, r9 - available to use
 695          *      PID - loaded with proper value when we get here
 696          *      Upon exit, we reload everything and RFI.
 697          * Actually, it will fit now, but oh well.....a common place
 698          * to load the TLB.
 699          */
 700 tlb_4xx_index:
 701         .long   0
 702 finish_tlb_load:
 703         /* load the next available TLB index.
 704         */
 705         lwz     r9, tlb_4xx_index@l(0)
 706         addi    r9, r9, 1
 707         andi.   r9, r9, (PPC40X_TLB_SIZE-1)
 708         stw     r9, tlb_4xx_index@l(0)
 709
 710 6:
 711         /*
 712          * Clear out the software-only bits in the PTE to generate the
 713          * TLB_DATA value.  These are the bottom 2 bits of the RPM, the
 714          * top 3 bits of the zone field, and M.
 715          */
 716         li      r12, 0x0ce2
 717         andc    r11, r11, r12
 718
 719         tlbwe   r11, r9, TLB_DATA               /* Load TLB LO */
 720         tlbwe   r10, r9, TLB_TAG                /* Load TLB HI */
 721
 722         /* Done...restore registers and get out of here.
 723         */
 724 #ifdef CONFIG_403GCX
 725         lwz     r12, 12(r0)
 726         lwz     r11, 8(r0)
 727         mtspr   SPRN_PID, r12
 728         mtcr    r11
 729         lwz     r9, 4(r0)
 730         lwz     r12, 0(r0)
 731 #else
 732         mfspr   r12, SPRN_SPRG_SCRATCH5
 733         mfspr   r11, SPRN_SPRG_SCRATCH6
 734         mtspr   SPRN_PID, r12
 735         mtcr    r11
 736         mfspr   r9, SPRN_SPRG_SCRATCH4
 737         mfspr   r12, SPRN_SPRG_SCRATCH3
 738 #endif
 739         mfspr   r11, SPRN_SPRG_SCRATCH1
 740         mfspr   r10, SPRN_SPRG_SCRATCH0
 741         PPC405_ERR77_SYNC
 742         rfi                     /* Should sync shadow TLBs */
 743         b       .               /* prevent prefetch past rfi */
 744
 745 /* This is where the main kernel code starts.
 746  */
 747 start_here:
 748
 749         /* ptr to current */
 750         lis     r2,init_task@h
 751         ori     r2,r2,init_task@l
 752
 753         /* ptr to phys current thread */
 754         tophys(r4,r2)
 755         addi    r4,r4,THREAD    /* init task's THREAD */
 756         mtspr   SPRN_SPRG_THREAD,r4
 757
 758         /* stack */
 759         lis     r1,init_thread_union@ha
 760         addi    r1,r1,init_thread_union@l
 761         li      r0,0
 762         stwu    r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1)
 763
 764         bl      early_init      /* We have to do this with MMU on */
 765
 766 /*
 767  * Decide what sort of machine this is and initialize the MMU.
 768  */
 769 #ifdef CONFIG_KASAN
 770         bl      kasan_early_init
 771 #endif
 772         li      r3,0
 773         mr      r4,r31
 774         bl      machine_init
 775         bl      MMU_init
 776
 777 /* Go back to running unmapped so we can load up new values
 778  * and change to using our exception vectors.
 779  * On the 4xx, all we have to do is invalidate the TLB to clear
 780  * the old 16M byte TLB mappings.
 781  */
 782         lis     r4,2f@h
 783         ori     r4,r4,2f@l
 784         tophys(r4,r4)
 785         lis     r3,(MSR_KERNEL & ~(MSR_IR|MSR_DR))@h
 786         ori     r3,r3,(MSR_KERNEL & ~(MSR_IR|MSR_DR))@l
 787         mtspr   SPRN_SRR0,r4
 788         mtspr   SPRN_SRR1,r3
 789         rfi
 790         b       .               /* prevent prefetch past rfi */
 791
 792 /* Load up the kernel context */
 793 2:
 794         sync                    /* Flush to memory before changing TLB */
 795         tlbia
 796         isync                   /* Flush shadow TLBs */
 797
 798         /* set up the PTE pointers for the Abatron bdiGDB.
 799         */
 800         lis     r6, swapper_pg_dir@h
 801         ori     r6, r6, swapper_pg_dir@l
 802         lis     r5, abatron_pteptrs@h
 803         ori     r5, r5, abatron_pteptrs@l
 804         stw     r5, 0xf0(r0)    /* Must match your Abatron config file */
 805         tophys(r5,r5)
 806         stw     r6, 0(r5)
 807
 808 /* Now turn on the MMU for real! */
 809         lis     r4,MSR_KERNEL@h
 810         ori     r4,r4,MSR_KERNEL@l
 811         lis     r3,start_kernel@h
 812         ori     r3,r3,start_kernel@l
 813         mtspr   SPRN_SRR0,r3
 814         mtspr   SPRN_SRR1,r4
 815         rfi                     /* enable MMU and jump to start_kernel */
 816         b       .               /* prevent prefetch past rfi */
 817
 818 /* Set up the initial MMU state so we can do the first level of
 819  * kernel initialization.  This maps the first 16 MBytes of memory 1:1
 820  * virtual to physical and more importantly sets the cache mode.
 821  */
 822 initial_mmu:
 823         tlbia                   /* Invalidate all TLB entries */
 824         isync
 825
 826         /* We should still be executing code at physical address 0x0000xxxx
 827          * at this point. However, start_here is at virtual address
 828          * 0xC000xxxx. So, set up a TLB mapping to cover this once
 829          * translation is enabled.
 830          */
 831
 832         lis     r3,KERNELBASE@h         /* Load the kernel virtual address */
 833         ori     r3,r3,KERNELBASE@l
 834         tophys(r4,r3)                   /* Load the kernel physical address */
 835
 836         iccci   r0,r3                   /* Invalidate the i-cache before use */
 837
 838         /* Load the kernel PID.
 839         */
 840         li      r0,0
 841         mtspr   SPRN_PID,r0
 842         sync
 843
 844         /* Configure and load one entry into TLB slots 63 */
 845         clrrwi  r4,r4,10                /* Mask off the real page number */
 846         ori     r4,r4,(TLB_WR | TLB_EX) /* Set the write and execute bits */
 847
 848         clrrwi  r3,r3,10                /* Mask off the effective page number */
 849         ori     r3,r3,(TLB_VALID | TLB_PAGESZ(PAGESZ_16M))
 850
 851         li      r0,63                    /* TLB slot 63 */
 852
 853         tlbwe   r4,r0,TLB_DATA          /* Load the data portion of the entry */
 854         tlbwe   r3,r0,TLB_TAG           /* Load the tag portion of the entry */
 855
 856         isync
 857
 858         /* Establish the exception vector base
 859         */
 860         lis     r4,KERNELBASE@h         /* EVPR only uses the high 16-bits */
 861         tophys(r0,r4)                   /* Use the physical address */
 862         mtspr   SPRN_EVPR,r0
 863
 864         blr
 865
 866 _GLOBAL(abort)
 867         mfspr   r13,SPRN_DBCR0
 868         oris    r13,r13,DBCR0_RST_SYSTEM@h
 869         mtspr   SPRN_DBCR0,r13
 870
 871 _GLOBAL(set_context)
 872
 873 #ifdef CONFIG_BDI_SWITCH
 874         /* Context switch the PTE pointer for the Abatron BDI2000.
 875          * The PGDIR is the second parameter.
 876          */
 877         lis     r5, abatron_pteptrs@ha
 878         stw     r4, abatron_pteptrs@l + 0x4(r5)
 879 #endif
 880         sync
 881         mtspr   SPRN_PID,r3
 882         isync                           /* Need an isync to flush shadow */
 883                                         /* TLBs after changing PID */
 884         blr
 885
 886 /* We put a few things here that have to be page-aligned. This stuff
 887  * goes at the beginning of the data segment, which is page-aligned.
 888  */
 889         .data
 890         .align  12
 891         .globl  sdata
 892 sdata:
 893         .globl  empty_zero_page
 894 empty_zero_page:
 895         .space  4096
 896 EXPORT_SYMBOL(empty_zero_page)
 897         .globl  swapper_pg_dir
 898 swapper_pg_dir:
 899         .space  PGD_TABLE_SIZE
 900
 901 /* Room for two PTE pointers, usually the kernel and current user pointers
 902  * to their respective root page table.
 903  */
 904 abatron_pteptrs:
 905         .space  8