1 // SPDX-License-Identifier: GPL-2.0
3 * IPU remoteproc driver for various SoCs
5 * Copyright (C) 2019 Texas Instruments Incorporated - https://www.ti.com/
6 * Angela Stegmaier <angelabaker@ti.com>
7 * Venkateswara Rao Mandela <venkat.mandela@ti.com>
8 * Keerthy <j-keerthy@ti.com>
15 #include <dm/device_compat.h>
18 #include <dm/of_access.h>
19 #include <fs_loader.h>
20 #include <remoteproc.h>
28 #include <power-domain.h>
34 #include <linux/bitmap.h>
36 #define IPU1_LOAD_ADDR (0xa17ff000)
37 #define MAX_REMOTECORE_BIN_SIZE (8 * 0x100000)
45 #define IPU2_LOAD_ADDR (IPU1_LOAD_ADDR + MAX_REMOTECORE_BIN_SIZE)
48 #define PAGESIZE_1M 0x0
49 #define PAGESIZE_64K 0x1
50 #define PAGESIZE_4K 0x2
51 #define PAGESIZE_16M 0x3
54 #define ELEMSIZE_8 0x0
55 #define ELEMSIZE_16 0x1
56 #define ELEMSIZE_32 0x2
60 #define PGT_SMALLPAGE_SIZE 0x00001000
61 #define PGT_LARGEPAGE_SIZE 0x00010000
62 #define PGT_SECTION_SIZE 0x00100000
63 #define PGT_SUPERSECTION_SIZE 0x01000000
65 #define PGT_L1_DESC_PAGE 0x00001
66 #define PGT_L1_DESC_SECTION 0x00002
67 #define PGT_L1_DESC_SUPERSECTION 0x40002
69 #define PGT_L1_DESC_PAGE_MASK 0xfffffC00
70 #define PGT_L1_DESC_SECTION_MASK 0xfff00000
71 #define PGT_L1_DESC_SUPERSECTION_MASK 0xff000000
73 #define PGT_L1_DESC_SMALLPAGE_INDEX_SHIFT 12
74 #define PGT_L1_DESC_LARGEPAGE_INDEX_SHIFT 16
75 #define PGT_L1_DESC_SECTION_INDEX_SHIFT 20
76 #define PGT_L1_DESC_SUPERSECTION_INDEX_SHIFT 24
78 #define PGT_L2_DESC_SMALLPAGE 0x02
79 #define PGT_L2_DESC_LARGEPAGE 0x01
81 #define PGT_L2_DESC_SMALLPAGE_MASK 0xfffff000
82 #define PGT_L2_DESC_LARGEPAGE_MASK 0xffff0000
85 * The memory for the page tables (256 KB per IPU) is placed just before
86 * the carveout memories for the remote processors. 16 KB of memory is
87 * needed for the L1 page table (4096 entries * 4 bytes per 1 MB section).
88 * Any smaller page (64 KB or 4 KB) entries are supported through L2 page
89 * tables (1 KB per table). The remaining 240 KB can provide support for
90 * 240 L2 page tables. Any remoteproc firmware image requiring more than
91 * 240 L2 page table entries would need more memory to be reserved.
93 #define PAGE_TABLE_SIZE_L1 (0x00004000)
94 #define PAGE_TABLE_SIZE_L2 (0x400)
95 #define MAX_NUM_L2_PAGE_TABLES (240)
96 #define PAGE_TABLE_SIZE_L2_TOTAL (MAX_NUM_L2_PAGE_TABLES * PAGE_TABLE_SIZE_L2)
97 #define PAGE_TABLE_SIZE (PAGE_TABLE_SIZE_L1 + (PAGE_TABLE_SIZE_L2_TOTAL))
100 * struct omap_rproc_mem - internal memory structure
101 * @cpu_addr: MPU virtual address of the memory region
102 * @bus_addr: bus address used to access the memory region
103 * @dev_addr: device address of the memory region from DSP view
104 * @size: size of the memory region
106 struct omap_rproc_mem
{
107 void __iomem
*cpu_addr
;
108 phys_addr_t bus_addr
;
113 struct ipu_privdata
{
114 struct omap_rproc_mem mem
;
115 struct list_head mappings
;
119 struct udevice
*rdev
;
122 typedef int (*handle_resource_t
) (void *, int offset
, int avail
);
124 unsigned int *page_table_l1
= (unsigned int *)0x0;
125 unsigned int *page_table_l2
= (unsigned int *)0x0;
128 * Set maximum carveout size to 96 MB
130 #define DRA7_RPROC_MAX_CO_SIZE (96 * 0x100000)
133 * These global variables are used for deriving the MMU page tables. They
134 * are initialized for each core with the appropriate values. The length
135 * of the array mem_bitmap is set as per a 96 MB carveout which the
136 * maximum set aside in the current memory map.
138 unsigned long mem_base
;
139 unsigned long mem_size
;
142 mem_bitmap
[BITS_TO_LONGS(DRA7_RPROC_MAX_CO_SIZE
>> PAGE_SHIFT
)];
143 unsigned long mem_count
;
145 unsigned int pgtable_l2_map
[MAX_NUM_L2_PAGE_TABLES
];
146 unsigned int pgtable_l2_cnt
;
148 void *ipu_alloc_mem(struct udevice
*dev
, unsigned long len
, unsigned long align
)
151 unsigned long pageno
;
154 count
= ((len
+ (PAGE_SIZE
- 1)) & ~(PAGE_SIZE
- 1)) >> PAGE_SHIFT
;
155 mask
= (1 << align
) - 1;
157 bitmap_find_next_zero_area(mem_bitmap
, mem_count
, 0, count
, mask
);
158 debug("%s: count %d mask %#lx pageno %#lx\n", __func__
, count
, mask
,
161 if (pageno
>= mem_count
) {
162 debug("%s: %s Error allocating memory; "
163 "Please check carveout size\n", __FILE__
, __func__
);
167 bitmap_set(mem_bitmap
, pageno
, count
);
168 return (void *)(mem_base
+ (pageno
<< PAGE_SHIFT
));
171 int find_pagesz(unsigned int virt
, unsigned int phys
, unsigned int len
)
174 unsigned int min_align
= __ffs(virt
);
176 if (min_align
> __ffs(phys
))
177 min_align
= __ffs(phys
);
179 if (min_align
>= PGT_L1_DESC_SUPERSECTION_INDEX_SHIFT
&&
181 pg_sz_ind
= PAGESIZE_16M
;
184 if (min_align
>= PGT_L1_DESC_SECTION_INDEX_SHIFT
&&
186 pg_sz_ind
= PAGESIZE_1M
;
189 if (min_align
>= PGT_L1_DESC_LARGEPAGE_INDEX_SHIFT
&&
191 pg_sz_ind
= PAGESIZE_64K
;
194 if (min_align
>= PGT_L1_DESC_SMALLPAGE_INDEX_SHIFT
&&
196 pg_sz_ind
= PAGESIZE_4K
;
204 int get_l2_pg_tbl_addr(unsigned int virt
, unsigned int *pg_tbl_addr
)
209 unsigned int tag
= (virt
& PGT_L1_DESC_SECTION_MASK
);
212 for (i
= 0; (i
< pgtable_l2_cnt
) && (match_found
== 0); i
++) {
213 if (tag
== pgtable_l2_map
[i
]) {
215 ((unsigned int)page_table_l2
) +
216 (i
* PAGE_TABLE_SIZE_L2
);
222 if (match_found
== 0 && i
< MAX_NUM_L2_PAGE_TABLES
) {
223 pgtable_l2_map
[i
] = tag
;
226 ((unsigned int)page_table_l2
) + (i
* PAGE_TABLE_SIZE_L2
);
234 config_l2_pagetable(unsigned int virt
, unsigned int phys
,
235 unsigned int pg_sz
, unsigned int pg_tbl_addr
)
238 unsigned int desc
= 0;
240 unsigned int *pg_tbl
= (unsigned int *)pg_tbl_addr
;
243 * Pick bit 19:12 of the virtual address as index
245 unsigned int index
= (virt
& (~PGT_L1_DESC_SECTION_MASK
)) >> PAGE_SHIFT
;
250 (phys
& PGT_L2_DESC_LARGEPAGE_MASK
) | PGT_L2_DESC_LARGEPAGE
;
251 for (i
= 0; i
< 16; i
++)
252 pg_tbl
[index
+ i
] = desc
;
257 (phys
& PGT_L2_DESC_SMALLPAGE_MASK
) | PGT_L2_DESC_SMALLPAGE
;
258 pg_tbl
[index
] = desc
;
269 ipu_config_pagetable(struct udevice
*dev
, unsigned int virt
, unsigned int phys
,
273 unsigned int l
= len
;
277 unsigned int pg_tbl_l2_addr
= 0;
278 unsigned int tmp_pgsz
;
280 if ((len
& 0x0FFF) != 0)
284 pg_sz
= find_pagesz(virt
, phys
, l
);
285 index
= virt
>> PGT_L1_DESC_SECTION_INDEX_SHIFT
;
288 * 16 MB super section
292 * Program the next 16 descriptors
295 (phys
& PGT_L1_DESC_SUPERSECTION_MASK
) |
296 PGT_L1_DESC_SUPERSECTION
;
297 for (i
= 0; i
< 16; i
++)
298 page_table_l1
[index
+ i
] = desc
;
299 l
-= PGT_SUPERSECTION_SIZE
;
300 phys
+= PGT_SUPERSECTION_SIZE
;
301 virt
+= PGT_SUPERSECTION_SIZE
;
308 (phys
& PGT_L1_DESC_SECTION_MASK
) |
310 page_table_l1
[index
] = desc
;
311 l
-= PGT_SECTION_SIZE
;
312 phys
+= PGT_SECTION_SIZE
;
313 virt
+= PGT_SECTION_SIZE
;
320 if (pg_sz
== PAGESIZE_64K
)
325 err
= get_l2_pg_tbl_addr(virt
, &pg_tbl_l2_addr
);
328 ("Unable to get level 2 PT address\n");
332 config_l2_pagetable(virt
, phys
, pg_sz
,
335 (pg_tbl_l2_addr
& PGT_L1_DESC_PAGE_MASK
) |
337 page_table_l1
[index
] = desc
;
351 int da_to_pa(struct udevice
*dev
, int da
)
353 struct rproc_mem_entry
*maps
= NULL
;
354 struct ipu_privdata
*priv
= dev_get_priv(dev
);
356 list_for_each_entry(maps
, &priv
->mappings
, node
) {
357 if (da
>= maps
->da
&& da
< (maps
->da
+ maps
->len
))
358 return maps
->dma
+ (da
- maps
->da
);
364 u32
ipu_config_mmu(u32 core_id
, struct rproc
*cfg
)
370 * Clear the entire pagetable location before programming the
371 * address into the MMU
373 memset((void *)cfg
->page_table_addr
, 0x00, PAGE_TABLE_SIZE
);
375 for (i
= 0; i
< cfg
->num_iommus
; i
++) {
376 u32 mmu_base
= cfg
->mmu_base_addr
[i
];
378 __raw_writel((int)cfg
->page_table_addr
, mmu_base
+ 0x4c);
379 reg
= __raw_readl(mmu_base
+ 0x88);
382 * enable bus-error back
384 __raw_writel(reg
| 0x1, mmu_base
+ 0x88);
387 * Enable the MMU IRQs during MMU programming for the
388 * late attachcase. This is to allow the MMU fault to be
389 * detected by the kernel.
391 * MULTIHITFAULT|EMMUMISS|TRANSLATIONFAULT|TABLEWALKFAULT
393 __raw_writel(0x1E, mmu_base
+ 0x1c);
396 * emutlbupdate|TWLENABLE|MMUENABLE
398 __raw_writel(0x6, mmu_base
+ 0x44);
405 * enum ipu_mem - PRU core memory range identifiers
414 static int ipu_start(struct udevice
*dev
)
416 struct ipu_privdata
*priv
;
417 struct reset_ctl reset
;
418 struct rproc
*cfg
= NULL
;
421 priv
= dev_get_priv(dev
);
423 cfg
= rproc_cfg_arr
[priv
->id
];
424 if (cfg
->config_peripherals
)
425 cfg
->config_peripherals(priv
->id
, cfg
);
428 * Start running the remote core
430 ret
= reset_get_by_index(dev
, 0, &reset
);
432 dev_err(dev
, "%s: error getting reset index %d\n", __func__
, 0);
436 ret
= reset_deassert(&reset
);
438 dev_err(dev
, "%s: error deasserting reset %d\n", __func__
, 0);
442 ret
= reset_get_by_index(dev
, 1, &reset
);
444 dev_err(dev
, "%s: error getting reset index %d\n", __func__
, 1);
448 ret
= reset_deassert(&reset
);
450 dev_err(dev
, "%s: error deasserting reset %d\n", __func__
, 1);
457 static int ipu_stop(struct udevice
*dev
)
463 * ipu_init() - Initialize the remote processor
464 * @dev: rproc device pointer
466 * Return: 0 if all went ok, else return appropriate error
468 static int ipu_init(struct udevice
*dev
)
473 static int ipu_add_res(struct udevice
*dev
, struct rproc_mem_entry
*mapping
)
475 struct ipu_privdata
*priv
= dev_get_priv(dev
);
477 list_add_tail(&mapping
->node
, &priv
->mappings
);
481 static int ipu_load(struct udevice
*dev
, ulong addr
, ulong size
)
483 Elf32_Ehdr
*ehdr
; /* Elf header structure pointer */
484 Elf32_Phdr
*phdr
; /* Program header structure pointer */
490 ehdr
= (Elf32_Ehdr
*)addr
;
491 phdr
= (Elf32_Phdr
*)(addr
+ ehdr
->e_phoff
);
493 * Load each program header
495 for (i
= 0; i
< ehdr
->e_phnum
; ++i
) {
496 memcpy(&proghdr
, phdr
, sizeof(Elf32_Phdr
));
498 if (proghdr
.p_type
!= PT_LOAD
) {
503 va
= proghdr
.p_paddr
;
504 pa
= da_to_pa(dev
, va
);
506 proghdr
.p_paddr
= pa
;
508 void *dst
= (void *)(uintptr_t)proghdr
.p_paddr
;
509 void *src
= (void *)addr
+ proghdr
.p_offset
;
511 debug("Loading phdr %i to 0x%p (%i bytes)\n", i
, dst
,
513 if (proghdr
.p_filesz
)
514 memcpy(dst
, src
, proghdr
.p_filesz
);
516 flush_cache((unsigned long)dst
, proghdr
.p_memsz
);
524 static const struct dm_rproc_ops ipu_ops
= {
529 .add_res
= ipu_add_res
,
530 .config_pagetable
= ipu_config_pagetable
,
531 .alloc_mem
= ipu_alloc_mem
,
535 * If the remotecore binary expects any peripherals to be setup before it has
536 * booted, configure them here.
538 * These functions are left empty by default as their operation is usecase
542 u32
ipu1_config_peripherals(u32 core_id
, struct rproc
*cfg
)
547 u32
ipu2_config_peripherals(u32 core_id
, struct rproc
*cfg
)
552 struct rproc_intmem_to_l3_mapping ipu1_intmem_to_l3_mapping
= {
559 .priv_addr
= 0x55020000,
560 .l3_addr
= 0x58820000,
565 struct rproc_intmem_to_l3_mapping ipu2_intmem_to_l3_mapping
= {
572 .priv_addr
= 0x55020000,
573 .l3_addr
= 0x55020000,
578 struct rproc ipu1_config
= {
580 .mmu_base_addr
= {0x58882000, 0},
581 .load_addr
= IPU1_LOAD_ADDR
,
583 .firmware_name
= "dra7-ipu1-fw.xem4",
584 .config_mmu
= ipu_config_mmu
,
585 .config_peripherals
= ipu1_config_peripherals
,
586 .intmem_to_l3_mapping
= &ipu1_intmem_to_l3_mapping
589 struct rproc ipu2_config
= {
591 .mmu_base_addr
= {0x55082000, 0},
592 .load_addr
= IPU2_LOAD_ADDR
,
594 .firmware_name
= "dra7-ipu2-fw.xem4",
595 .config_mmu
= ipu_config_mmu
,
596 .config_peripherals
= ipu2_config_peripherals
,
597 .intmem_to_l3_mapping
= &ipu2_intmem_to_l3_mapping
600 struct rproc
*rproc_cfg_arr
[2] = {
601 [IPU2
] = &ipu2_config
,
602 [IPU1
] = &ipu1_config
,
605 u32
spl_pre_boot_core(struct udevice
*dev
, u32 core_id
)
607 struct rproc
*cfg
= NULL
;
608 unsigned long load_elf_status
= 0;
611 cfg
= rproc_cfg_arr
[core_id
];
613 * Check for valid elf image
615 if (!valid_elf_image(cfg
->load_addr
))
618 if (rproc_find_resource_table(dev
, cfg
->load_addr
, &tablesz
))
619 cfg
->has_rsc_table
= 1;
621 cfg
->has_rsc_table
= 0;
626 if (cfg
->config_mmu
&& cfg
->has_rsc_table
)
627 cfg
->config_mmu(core_id
, cfg
);
630 * Load the remote core. Fill the page table of the first(possibly
631 * only) IOMMU during ELF loading. Copy the page table to the second
632 * IOMMU before running the remote core.
635 page_table_l1
= (unsigned int *)cfg
->page_table_addr
;
637 (unsigned int *)(cfg
->page_table_addr
+ PAGE_TABLE_SIZE_L1
);
638 mem_base
= cfg
->cma_base
;
639 mem_size
= cfg
->cma_size
;
640 memset(mem_bitmap
, 0x00, sizeof(mem_bitmap
));
641 mem_count
= (cfg
->cma_size
>> PAGE_SHIFT
);
644 * Clear variables used for level 2 page table allocation
646 memset(pgtable_l2_map
, 0x00, sizeof(pgtable_l2_map
));
649 load_elf_status
= rproc_parse_resource_table(dev
, cfg
);
650 if (load_elf_status
== 0) {
651 debug("load_elf_image_phdr returned error for core %s\n",
656 flush_cache(cfg
->page_table_addr
, PAGE_TABLE_SIZE
);
661 static fdt_addr_t
ipu_parse_mem_nodes(struct udevice
*dev
, char *name
,
662 int privid
, fdt_size_t
*sizep
)
668 ret
= ofnode_read_u32(dev_ofnode(dev
), name
, &sp
);
670 dev_err(dev
, "memory-region node fetch failed %d\n", ret
);
674 mem_node
= ofnode_get_by_phandle(sp
);
675 if (!ofnode_valid(mem_node
))
678 return ofnode_get_addr_size_index(mem_node
, 0, sizep
);
682 * ipu_probe() - Basic probe
683 * @dev: corresponding k3 remote processor device
685 * Return: 0 if all goes good, else appropriate error message.
687 static int ipu_probe(struct udevice
*dev
)
689 struct ipu_privdata
*priv
;
690 struct rproc
*cfg
= NULL
;
691 struct reset_ctl reset
;
692 static const char *const ipu_mem_names
[] = { "l2ram" };
696 priv
= dev_get_priv(dev
);
699 devfdt_get_addr_size_name(dev
,
701 (fdt_addr_t
*)&priv
->mem
.size
);
703 ret
= reset_get_by_index(dev
, 2, &reset
);
705 dev_err(dev
, "%s: error getting reset index %d\n", __func__
, 2);
709 ret
= reset_deassert(&reset
);
711 dev_err(dev
, "%s: error deasserting reset %d\n", __func__
, 2);
715 if (priv
->mem
.bus_addr
== FDT_ADDR_T_NONE
) {
716 dev_err(dev
, "%s bus address not found\n", ipu_mem_names
[0]);
719 priv
->mem
.cpu_addr
= map_physmem(priv
->mem
.bus_addr
,
720 priv
->mem
.size
, MAP_NOCACHE
);
722 if (devfdt_get_addr(dev
) == 0x58820000)
727 cfg
= rproc_cfg_arr
[priv
->id
];
728 cfg
->cma_base
= ipu_parse_mem_nodes(dev
, "memory-region", priv
->id
,
730 cfg
->cma_size
= sizep
;
732 cfg
->page_table_addr
= ipu_parse_mem_nodes(dev
, "pg-tbl", priv
->id
,
736 "ID %d memory %8s: bus addr %pa size 0x%zx va %p da 0x%x\n",
737 priv
->id
, ipu_mem_names
[0], &priv
->mem
.bus_addr
,
738 priv
->mem
.size
, priv
->mem
.cpu_addr
, priv
->mem
.dev_addr
);
740 INIT_LIST_HEAD(&priv
->mappings
);
741 if (spl_pre_boot_core(dev
, priv
->id
))
747 static const struct udevice_id ipu_ids
[] = {
748 {.compatible
= "ti,dra7-ipu"},
752 U_BOOT_DRIVER(ipu
) = {
755 .id
= UCLASS_REMOTEPROC
,
758 .priv_auto
= sizeof(struct ipu_privdata
),