mmu.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755
  1. /*
  2. * Copyright (c) 2006-2025 RT-Thread Development Team
  3. *
  4. * SPDX-License-Identifier: Apache-2.0
  5. *
  6. * Change Logs:
  7. * Date Author Notes
  8. * 2021-01-30 lizhirui first version
  9. * 2022-12-13 WangXiaoyao Port to new mm
  10. * 2023-10-12 Shell Add permission control API
  11. */
  12. #include <rtthread.h>
  13. #include <stddef.h>
  14. #include <stdint.h>
  15. #define DBG_TAG "hw.mmu"
  16. #define DBG_LVL DBG_INFO
  17. #include <rtdbg.h>
  18. #include <board.h>
  19. #include <cache.h>
  20. #include <mm_aspace.h>
  21. #include <mm_page.h>
  22. #include <mmu.h>
  23. #include <riscv_mmu.h>
  24. #include <tlb.h>
  25. #ifdef RT_USING_SMART
  26. #include <board.h>
  27. #include <ioremap.h>
  28. #include <lwp_user_mm.h>
  29. #endif
  30. #ifndef RT_USING_SMART
  31. #define USER_VADDR_START 0
  32. #endif
  33. static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr);
  34. static void *current_mmu_table = RT_NULL;
  35. volatile __attribute__((aligned(4 * 1024)))
  36. rt_ubase_t MMUTable[__SIZE(VPN2_BIT)];
  37. /**
  38. * @brief Switch the current address space to the specified one.
  39. *
  40. * This function is responsible for switching the address space by updating the page table
  41. * and related hardware state. The behavior depends on whether the architecture supports
  42. * Address Space Identifiers (ASIDs), devided by macro definition of ARCH_USING_ASID.
  43. *
  44. * @param aspace Pointer to the address space structure containing the new page table.
  45. *
  46. * @note If ASID is supported (`ARCH_USING_ASID` is defined), the function will call
  47. * `rt_hw_asid_switch_pgtbl` to switch the page table and update the ASID.
  48. * Otherwise, it will directly write the `satp` CSR to switch the page table
  49. * and invalidate the TLB.
  50. */
  51. #ifdef ARCH_USING_ASID
  52. void rt_hw_aspace_switch(rt_aspace_t aspace)
  53. {
  54. uintptr_t page_table = (uintptr_t)rt_kmem_v2p(aspace->page_table);
  55. current_mmu_table = aspace->page_table;
  56. rt_hw_asid_switch_pgtbl(aspace, page_table);
  57. }
  58. #else /* !ARCH_USING_ASID */
  59. void rt_hw_aspace_switch(rt_aspace_t aspace)
  60. {
  61. uintptr_t page_table = (uintptr_t)rt_kmem_v2p(aspace->page_table);
  62. current_mmu_table = aspace->page_table;
  63. write_csr(satp, (((size_t)SATP_MODE) << SATP_MODE_OFFSET) |
  64. ((rt_ubase_t)page_table >> PAGE_OFFSET_BIT));
  65. rt_hw_tlb_invalidate_all_local();
  66. }
  67. void rt_hw_asid_init(void)
  68. {
  69. }
  70. #endif /* ARCH_USING_ASID */
  71. /* get current page table. */
  72. void *rt_hw_mmu_tbl_get()
  73. {
  74. return current_mmu_table;
  75. }
  76. /* Map a single virtual address page to a physical address page in the page table. */
  77. static int _map_one_page(struct rt_aspace *aspace, void *va, void *pa,
  78. size_t attr)
  79. {
  80. rt_ubase_t l1_off, l2_off, l3_off;
  81. rt_ubase_t *mmu_l1, *mmu_l2, *mmu_l3;
  82. l1_off = GET_L1((size_t)va);
  83. l2_off = GET_L2((size_t)va);
  84. l3_off = GET_L3((size_t)va);
  85. mmu_l1 = ((rt_ubase_t *)aspace->page_table) + l1_off;
  86. if (PTE_USED(*mmu_l1))
  87. {
  88. mmu_l2 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET);
  89. }
  90. else
  91. {
  92. mmu_l2 = (rt_ubase_t *)rt_pages_alloc(0);
  93. if (mmu_l2)
  94. {
  95. rt_memset(mmu_l2, 0, PAGE_SIZE);
  96. rt_hw_cpu_dcache_clean(mmu_l2, PAGE_SIZE);
  97. *mmu_l1 = COMBINEPTE((rt_ubase_t)VPN_TO_PPN(mmu_l2, PV_OFFSET),
  98. PAGE_DEFAULT_ATTR_NEXT);
  99. rt_hw_cpu_dcache_clean(mmu_l1, sizeof(*mmu_l1));
  100. }
  101. else
  102. {
  103. return -1;
  104. }
  105. }
  106. if (PTE_USED(*(mmu_l2 + l2_off)))
  107. {
  108. RT_ASSERT(!PAGE_IS_LEAF(*(mmu_l2 + l2_off)));
  109. mmu_l3 =
  110. (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)), PV_OFFSET);
  111. }
  112. else
  113. {
  114. mmu_l3 = (rt_ubase_t *)rt_pages_alloc(0);
  115. if (mmu_l3)
  116. {
  117. rt_memset(mmu_l3, 0, PAGE_SIZE);
  118. rt_hw_cpu_dcache_clean(mmu_l3, PAGE_SIZE);
  119. *(mmu_l2 + l2_off) =
  120. COMBINEPTE((rt_ubase_t)VPN_TO_PPN(mmu_l3, PV_OFFSET),
  121. PAGE_DEFAULT_ATTR_NEXT);
  122. rt_hw_cpu_dcache_clean(mmu_l2, sizeof(*mmu_l2));
  123. /* declares a reference to parent page table */
  124. rt_page_ref_inc((void *)mmu_l2, 0);
  125. }
  126. else
  127. {
  128. return -1;
  129. }
  130. }
  131. RT_ASSERT(!PTE_USED(*(mmu_l3 + l3_off)));
  132. /* declares a reference to parent page table */
  133. rt_page_ref_inc((void *)mmu_l3, 0);
  134. *(mmu_l3 + l3_off) = COMBINEPTE((rt_ubase_t)pa, attr);
  135. rt_hw_cpu_dcache_clean(mmu_l3 + l3_off, sizeof(*(mmu_l3 + l3_off)));
  136. return 0;
  137. }
  138. /**
  139. * @brief Maps a virtual address space to a physical address space.
  140. *
  141. * This function maps a specified range of virtual addresses to a range of physical addresses
  142. * and sets the attributes of the page table entries (PTEs). If an error occurs during the
  143. * mapping process, the function will automatically roll back any partially completed mappings.
  144. *
  145. * @param aspace Pointer to the address space structure containing the page table information.
  146. * @param v_addr The starting virtual address to be mapped.
  147. * @param p_addr The starting physical address to be mapped.
  148. * @param size The size of the memory to be mapped (in bytes).
  149. * @param attr The attributes of the page table entries (e.g., read/write permissions, cache policies).
  150. *
  151. * @return On success, returns the starting virtual address `v_addr`;
  152. * On failure, returns `NULL`.
  153. *
  154. * @note This function will not override existing page table entries.
  155. * @warning The caller must ensure that `v_addr` and `p_addr` are page-aligned,
  156. * and `size` is a multiple of the page size.
  157. *
  158. */
  159. void *rt_hw_mmu_map(struct rt_aspace *aspace, void *v_addr, void *p_addr,
  160. size_t size, size_t attr)
  161. {
  162. int ret = -1;
  163. void *unmap_va = v_addr;
  164. size_t npages = size >> ARCH_PAGE_SHIFT;
  165. /* TODO trying with HUGEPAGE here */
  166. while (npages--)
  167. {
  168. MM_PGTBL_LOCK(aspace);
  169. ret = _map_one_page(aspace, v_addr, p_addr, attr);
  170. MM_PGTBL_UNLOCK(aspace);
  171. if (ret != 0)
  172. {
  173. /* error, undo map */
  174. while (unmap_va != v_addr)
  175. {
  176. MM_PGTBL_LOCK(aspace);
  177. _unmap_area(aspace, unmap_va);
  178. MM_PGTBL_UNLOCK(aspace);
  179. unmap_va += ARCH_PAGE_SIZE;
  180. }
  181. break;
  182. }
  183. v_addr += ARCH_PAGE_SIZE;
  184. p_addr += ARCH_PAGE_SIZE;
  185. }
  186. if (ret == 0)
  187. {
  188. return unmap_va;
  189. }
  190. return NULL;
  191. }
  192. /* unmap page table entry */
  193. static void _unmap_pte(rt_ubase_t *pentry, rt_ubase_t *lvl_entry[], int level)
  194. {
  195. int loop_flag = 1;
  196. while (loop_flag)
  197. {
  198. loop_flag = 0;
  199. *pentry = 0;
  200. rt_hw_cpu_dcache_clean(pentry, sizeof(*pentry));
  201. /* we don't handle level 0, which is maintained by caller */
  202. if (level > 0)
  203. {
  204. void *page = (void *)((rt_ubase_t)pentry & ~ARCH_PAGE_MASK);
  205. /* decrease reference from child page to parent */
  206. rt_pages_free(page, 0);
  207. int free = rt_page_ref_get(page, 0);
  208. if (free == 1)
  209. {
  210. rt_pages_free(page, 0);
  211. pentry = lvl_entry[--level];
  212. loop_flag = 1;
  213. }
  214. }
  215. }
  216. }
  217. /* Unmaps a virtual address range (1GB/2MB/4KB according to actual page level) from the page table. */
  218. static size_t _unmap_area(struct rt_aspace *aspace, void *v_addr)
  219. {
  220. rt_ubase_t loop_va = __UMASKVALUE((rt_ubase_t)v_addr, PAGE_OFFSET_MASK);
  221. size_t unmapped = 0;
  222. int i = 0;
  223. rt_ubase_t lvl_off[3];
  224. rt_ubase_t *lvl_entry[3];
  225. lvl_off[0] = (rt_ubase_t)GET_L1(loop_va);
  226. lvl_off[1] = (rt_ubase_t)GET_L2(loop_va);
  227. lvl_off[2] = (rt_ubase_t)GET_L3(loop_va);
  228. unmapped = 1 << (ARCH_PAGE_SHIFT + ARCH_INDEX_WIDTH * 2ul);
  229. rt_ubase_t *pentry;
  230. lvl_entry[i] = ((rt_ubase_t *)aspace->page_table + lvl_off[i]);
  231. pentry = lvl_entry[i];
  232. /* check if lvl_entry[0] is valid. if no, return 0 directly. */
  233. if (!PTE_USED(*pentry))
  234. {
  235. return 0;
  236. }
  237. /* find leaf page table entry */
  238. while (PTE_USED(*pentry) && !PAGE_IS_LEAF(*pentry))
  239. {
  240. i += 1;
  241. if (i >= 3)
  242. {
  243. unmapped = 0;
  244. break;
  245. }
  246. lvl_entry[i] = ((rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*pentry), PV_OFFSET) +
  247. lvl_off[i]);
  248. pentry = lvl_entry[i];
  249. unmapped >>= ARCH_INDEX_WIDTH;
  250. }
  251. /* clear PTE & setup its */
  252. if (PTE_USED(*pentry))
  253. {
  254. _unmap_pte(pentry, lvl_entry, i);
  255. }
  256. else
  257. {
  258. unmapped = 0; /* invalid pte, return 0. */
  259. }
  260. return unmapped;
  261. }
  262. /**
  263. * @brief Unmaps a range of virtual memory addresses from the specified address space.
  264. *
  265. * This function is responsible for unmapping a contiguous region of virtual memory
  266. * from the given address space. It handles multiple pages and ensures thread safety
  267. * by locking the page table during the unmapping operation.
  268. *
  269. * @param aspace Pointer to the address space structure from which the memory will be unmapped.
  270. * @param v_addr Starting virtual address to unmap. Must be page-aligned.
  271. * @param size Size of the memory region to unmap. Must be page-aligned.
  272. *
  273. * @note The caller must ensure that both `v_addr` and `size` are page-aligned.
  274. *
  275. * @details The function operates in a loop, unmapping memory in chunks. It uses the
  276. * `_unmap_area` function to perform the actual unmapping, which is called within a
  277. * locked section to ensure thread safety. The loop continues until the entire region
  278. * is unmapped.
  279. *
  280. * @see _unmap_area
  281. * @note unmap is different from map that it can handle multiple pages
  282. */
  283. void rt_hw_mmu_unmap(struct rt_aspace *aspace, void *v_addr, size_t size)
  284. {
  285. /* caller guarantee that v_addr & size are page aligned */
  286. if (!aspace->page_table)
  287. {
  288. return;
  289. }
  290. size_t unmapped = 0;
  291. while (size > 0)
  292. {
  293. MM_PGTBL_LOCK(aspace);
  294. unmapped = _unmap_area(aspace, v_addr);
  295. MM_PGTBL_UNLOCK(aspace);
  296. /* when unmapped == 0, region not exist in pgtbl */
  297. if (!unmapped || unmapped > size) break;
  298. size -= unmapped;
  299. v_addr += unmapped;
  300. }
  301. }
  302. #ifdef RT_USING_SMART
  303. static inline void _init_region(void *vaddr, size_t size)
  304. {
  305. rt_ioremap_start = vaddr;
  306. rt_ioremap_size = size;
  307. rt_mpr_start = rt_ioremap_start - rt_mpr_size;
  308. LOG_D("rt_ioremap_start: %p, rt_mpr_start: %p", rt_ioremap_start,
  309. rt_mpr_start);
  310. }
  311. #else
  312. static inline void _init_region(void *vaddr, size_t size)
  313. {
  314. rt_mpr_start = vaddr - rt_mpr_size;
  315. }
  316. #endif
  317. #if defined(RT_USING_SMART) && defined(ARCH_REMAP_KERNEL)
  318. #define KERN_SPACE_START ((void *)KERNEL_VADDR_START)
  319. #define KERN_SPACE_SIZE (0xfffffffffffff000UL - KERNEL_VADDR_START + 0x1000)
  320. #else
  321. #define KERN_SPACE_START ((void *)0x1000)
  322. #define KERN_SPACE_SIZE ((size_t)USER_VADDR_START - 0x1000)
  323. #endif
  324. /**
  325. * @brief Initialize the MMU (Memory Management Unit) mapping.
  326. *
  327. * This function initializes the MMU mapping, incluing these steps as follows:
  328. * 1. Check the validity of the input parameters,
  329. * 2. Calculate the start and end virtual addresses based on the input virtual address and size.
  330. * 3. Convert the virtual addresses to PPN2 indices.
  331. * 4. Check the initialization of the page table. If any entry in the page table within
  332. * the specified range is non-zero, it returns -1.
  333. * 5. It initializes the kernel address space using rt_aspace_init() and initializes the specified region
  334. * using _init_region.
  335. *
  336. * @param aspace Pointer to the address space. Must not be NULL.
  337. * @param v_address The starting virtual address.
  338. * @param size The size of the virtual address space.
  339. * @param vtable Pointer to the page table. Must not be NULL.
  340. * @param pv_off The page table offset.
  341. *
  342. * @return Returns 0 if the initialization is successful. Returns -1 if any input parameter is invalid
  343. * or the page table initialization check fails.
  344. */
  345. int rt_hw_mmu_map_init(rt_aspace_t aspace, void *v_address, rt_ubase_t size,
  346. rt_ubase_t *vtable, rt_ubase_t pv_off)
  347. {
  348. size_t l1_off, va_s, va_e;
  349. if ((!aspace) || (!vtable))
  350. {
  351. return -1;
  352. }
  353. va_s = (rt_ubase_t)v_address;
  354. va_e = ((rt_ubase_t)v_address) + size - 1;
  355. if (va_e < va_s)
  356. {
  357. return -1;
  358. }
  359. /* convert address to PPN2 index */
  360. va_s = GET_L1(va_s);
  361. va_e = GET_L1(va_e);
  362. if (va_s == 0)
  363. {
  364. return -1;
  365. }
  366. /* vtable initialization check */
  367. for (l1_off = va_s; l1_off <= va_e; l1_off++)
  368. {
  369. size_t v = vtable[l1_off];
  370. if (v)
  371. {
  372. return -1;
  373. }
  374. }
  375. rt_aspace_init(&rt_kernel_space, KERN_SPACE_START, KERN_SPACE_SIZE, vtable);
  376. _init_region(v_address, size);
  377. return 0;
  378. }
  379. const static int max_level =
  380. (ARCH_VADDR_WIDTH - ARCH_PAGE_SHIFT) / ARCH_INDEX_WIDTH;
  381. static inline uintptr_t _get_level_size(int level)
  382. {
  383. return 1ul << (ARCH_PAGE_SHIFT + (max_level - level) * ARCH_INDEX_WIDTH);
  384. }
  385. static rt_ubase_t *_query(struct rt_aspace *aspace, void *vaddr, int *level)
  386. {
  387. rt_ubase_t l1_off, l2_off, l3_off;
  388. rt_ubase_t *mmu_l1, *mmu_l2, *mmu_l3;
  389. rt_ubase_t pa;
  390. l1_off = GET_L1((rt_uintptr_t)vaddr);
  391. l2_off = GET_L2((rt_uintptr_t)vaddr);
  392. l3_off = GET_L3((rt_uintptr_t)vaddr);
  393. if (!aspace)
  394. {
  395. LOG_W("%s: no aspace", __func__);
  396. return RT_NULL;
  397. }
  398. mmu_l1 = ((rt_ubase_t *)aspace->page_table) + l1_off;
  399. if (PTE_USED(*mmu_l1))
  400. {
  401. if (*mmu_l1 & PTE_XWR_MASK)
  402. {
  403. *level = 1;
  404. return mmu_l1;
  405. }
  406. mmu_l2 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*mmu_l1), PV_OFFSET);
  407. if (PTE_USED(*(mmu_l2 + l2_off)))
  408. {
  409. if (*(mmu_l2 + l2_off) & PTE_XWR_MASK)
  410. {
  411. *level = 2;
  412. return mmu_l2 + l2_off;
  413. }
  414. mmu_l3 = (rt_ubase_t *)PPN_TO_VPN(GET_PADDR(*(mmu_l2 + l2_off)),
  415. PV_OFFSET);
  416. if (PTE_USED(*(mmu_l3 + l3_off)))
  417. {
  418. *level = 3;
  419. return mmu_l3 + l3_off;
  420. }
  421. }
  422. }
  423. return RT_NULL;
  424. }
  425. /**
  426. * @brief Translate a virtual address to a physical address.
  427. *
  428. * This function translates a given virtual address (`vaddr`) to its corresponding
  429. * physical address (`paddr`) using the page table in the specified address space (`aspace`).
  430. *
  431. * @param aspace Pointer to the address space structure containing the page table.
  432. * @param vaddr The virtual address to be translated.
  433. *
  434. * @return The translated physical address. If the translation fails, `ARCH_MAP_FAILED` is returned.
  435. *
  436. * @note The function queries the page table entry (PTE) for the virtual address using `_query`.
  437. * If a valid PTE is found, the physical address is extracted and combined with the offset
  438. * from the virtual address. If no valid PTE is found, a debug log is recorded, and
  439. * `ARCH_MAP_FAILED` is returned.
  440. */
  441. void *rt_hw_mmu_v2p(struct rt_aspace *aspace, void *vaddr)
  442. {
  443. int level;
  444. rt_ubase_t *pte = _query(aspace, vaddr, &level);
  445. uintptr_t paddr;
  446. if (pte)
  447. {
  448. paddr = GET_PADDR(*pte);
  449. paddr |= ((intptr_t)vaddr & (_get_level_size(level) - 1));
  450. }
  451. else
  452. {
  453. LOG_D("%s: failed at %p", __func__, vaddr);
  454. paddr = (uintptr_t)ARCH_MAP_FAILED;
  455. }
  456. return (void *)paddr;
  457. }
  458. static int _noncache(rt_base_t *pte)
  459. {
  460. return 0;
  461. }
  462. static int _cache(rt_base_t *pte)
  463. {
  464. return 0;
  465. }
  466. static int (*control_handler[MMU_CNTL_DUMMY_END])(rt_base_t *pte)=
  467. {
  468. [MMU_CNTL_CACHE] = _cache,
  469. [MMU_CNTL_NONCACHE] = _noncache,
  470. };
  471. /**
  472. * @brief Control the page table entries (PTEs) for a specified virtual address range.
  473. *
  474. * This function applies a control command (e.g., cache control) to the page table entries
  475. * (PTEs) corresponding to the specified virtual address range (`vaddr` to `vaddr + size`).
  476. *
  477. * @param aspace Pointer to the address space structure containing the page table.
  478. * @param vaddr The starting virtual address of the range.
  479. * @param size The size of the virtual address range.
  480. * @param cmd The control command to apply (e.g., `MMU_CNTL_CACHE`, `MMU_CNTL_NONCACHE`.etc.).
  481. *
  482. * @return `RT_EOK` on success, or an error code (`-RT_EINVAL` or `-RT_ENOSYS`) on failure.
  483. *
  484. * @note The function uses the `control_handler` array to map the command to a handler function.
  485. * It iterates over the virtual address range, queries the PTEs, and applies the handler
  486. * to each valid PTE. If the command is invalid, `-RT_ENOSYS` is returned.
  487. */
  488. int rt_hw_mmu_control(struct rt_aspace *aspace, void *vaddr, size_t size,
  489. enum rt_mmu_cntl cmd)
  490. {
  491. int level;
  492. int err = -RT_EINVAL;
  493. void *vend = vaddr + size;
  494. int (*handler)(rt_base_t *pte);
  495. if (cmd >= 0 && cmd < MMU_CNTL_DUMMY_END)
  496. {
  497. handler = control_handler[cmd];
  498. while (vaddr < vend)
  499. {
  500. rt_base_t *pte = _query(aspace, vaddr, &level);
  501. void *range_end = vaddr + _get_level_size(level);
  502. RT_ASSERT(range_end <= vend);
  503. if (pte)
  504. {
  505. err = handler(pte);
  506. RT_ASSERT(err == RT_EOK);
  507. }
  508. vaddr = range_end;
  509. }
  510. }
  511. else
  512. {
  513. err = -RT_ENOSYS;
  514. }
  515. return err;
  516. }
  517. /**
  518. * @brief setup Page Table for kernel space. It's a fixed map
  519. * and all mappings cannot be changed after initialization.
  520. *
  521. * Memory region in struct mem_desc must be page aligned,
  522. * otherwise is a failure and no report will be
  523. * returned.
  524. *
  525. * @param aspace Pointer to the address space structure.
  526. * @param mdesc Pointer to the array of memory descriptors.
  527. * @param desc_nr Number of memory descriptors in the array.
  528. */
  529. void rt_hw_mmu_setup(rt_aspace_t aspace, struct mem_desc *mdesc, int desc_nr)
  530. {
  531. void *err;
  532. for (size_t i = 0; i < desc_nr; i++)
  533. {
  534. size_t attr;
  535. switch (mdesc->attr)
  536. {
  537. case NORMAL_MEM:
  538. attr = MMU_MAP_K_RWCB;
  539. break;
  540. case NORMAL_NOCACHE_MEM:
  541. attr = MMU_MAP_K_RWCB;
  542. break;
  543. case DEVICE_MEM:
  544. attr = MMU_MAP_K_DEVICE;
  545. break;
  546. default:
  547. attr = MMU_MAP_K_DEVICE;
  548. }
  549. struct rt_mm_va_hint hint = {
  550. .flags = MMF_MAP_FIXED,
  551. .limit_start = aspace->start,
  552. .limit_range_size = aspace->size,
  553. .map_size = mdesc->vaddr_end - mdesc->vaddr_start + 1,
  554. .prefer = (void *)mdesc->vaddr_start};
  555. if (mdesc->paddr_start == (rt_uintptr_t)ARCH_MAP_FAILED)
  556. mdesc->paddr_start = mdesc->vaddr_start + PV_OFFSET;
  557. rt_aspace_map_phy_static(aspace, &mdesc->varea, &hint, attr,
  558. mdesc->paddr_start >> MM_PAGE_SHIFT, &err);
  559. mdesc++;
  560. }
  561. rt_hw_asid_init();
  562. rt_hw_aspace_switch(&rt_kernel_space);
  563. rt_page_cleanup();
  564. }
  565. #define SATP_BASE ((rt_ubase_t)SATP_MODE << SATP_MODE_OFFSET)
  566. extern unsigned int __bss_end;
  567. /**
  568. * @brief Early memory setup function for hardware initialization.
  569. *
  570. * This function performs early memory setup tasks, including:
  571. * - Calculating the physical-to-virtual (PV) offset.
  572. * - Setting up initial page tables for identity mapping and text region relocation.
  573. * - Applying new memory mappings by updating the SATP register.
  574. *
  575. * @note This function is typically called during the early stages of system initialization (startup_gcc.S),
  576. * before the memory management system is fully operational.
  577. * Here the identity mapping is implemented by a 1-stage page table, whose page size is 1GB.
  578. */
  579. void rt_hw_mem_setup_early(void)
  580. {
  581. rt_ubase_t pv_off;
  582. rt_ubase_t ps = 0x0;
  583. rt_ubase_t vs = 0x0;
  584. rt_ubase_t *early_pgtbl = (rt_ubase_t *)(((size_t)&__bss_end + 4095) & ~0xfff);
  585. /* calculate pv_offset */
  586. void *symb_pc;
  587. void *symb_linker;
  588. __asm__ volatile("la %0, _start\n" : "=r"(symb_pc));
  589. __asm__ volatile("la %0, _start_link_addr\n" : "=r"(symb_linker));
  590. symb_linker = *(void **)symb_linker;
  591. pv_off = symb_pc - symb_linker;
  592. rt_kmem_pvoff_set(pv_off);
  593. if (pv_off)
  594. {
  595. if (pv_off & ((1ul << (ARCH_INDEX_WIDTH * 2 + ARCH_PAGE_SHIFT)) - 1))
  596. {
  597. LOG_E("%s: not aligned virtual address. pv_offset %p", __func__,
  598. pv_off);
  599. RT_ASSERT(0);
  600. }
  601. /**
  602. * identical mapping,
  603. * PC are still at lower region before relocating to high memory
  604. */
  605. rt_ubase_t pg_idx ;
  606. ps = (rt_ubase_t)symb_pc & (~(L1_PAGE_SIZE - 1));
  607. pg_idx = GET_L1(ps);
  608. early_pgtbl[pg_idx] = COMBINEPTE(ps, MMU_MAP_EARLY);
  609. /* relocate text region */
  610. __asm__ volatile("la %0, _start\n" : "=r"(ps));
  611. ps &= ~(L1_PAGE_SIZE - 1);
  612. vs = ps - pv_off;
  613. /* relocate region */
  614. rt_ubase_t vs_idx = GET_L1(vs);
  615. rt_ubase_t ve_idx = GET_L1(vs + 0x80000000);
  616. for (size_t i = vs_idx; i < ve_idx; i++)
  617. {
  618. early_pgtbl[i] = COMBINEPTE(ps, MMU_MAP_EARLY);
  619. ps += L1_PAGE_SIZE;
  620. }
  621. /* apply new mapping */
  622. asm volatile("sfence.vma x0, x0");
  623. write_csr(satp, SATP_BASE | ((size_t)early_pgtbl >> PAGE_OFFSET_BIT));
  624. asm volatile("sfence.vma x0, x0");
  625. }
  626. /* return to lower text section */
  627. }
  628. /**
  629. * @brief Creates and initializes a new MMU page table.
  630. *
  631. * This function allocates a new MMU page table, copies the kernel space
  632. * page table into it, and flushes the data cache to ensure consistency.
  633. *
  634. * @return
  635. * - A pointer to the newly allocated MMU page table on success.
  636. * - RT_NULL if the allocation fails.
  637. */
  638. void *rt_hw_mmu_pgtbl_create(void)
  639. {
  640. rt_ubase_t *mmu_table;
  641. mmu_table = (rt_ubase_t *)rt_pages_alloc_ext(0, PAGE_ANY_AVAILABLE);
  642. if (!mmu_table)
  643. {
  644. return RT_NULL;
  645. }
  646. rt_memcpy(mmu_table, rt_kernel_space.page_table, ARCH_PAGE_SIZE);
  647. rt_hw_cpu_dcache_ops(RT_HW_CACHE_FLUSH, mmu_table, ARCH_PAGE_SIZE);
  648. return mmu_table;
  649. }
  650. /**
  651. * @brief Deletes an MMU page table.
  652. *
  653. * This function frees the memory allocated for the given MMU page table.
  654. *
  655. * @param pgtbl Pointer to the MMU page table to be deleted.
  656. */
  657. void rt_hw_mmu_pgtbl_delete(void *pgtbl)
  658. {
  659. rt_pages_free(pgtbl, 0);
  660. }