mm_page.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655
  1. /*
  2. * Copyright (c) 2006-2022, RT-Thread Development Team
  3. *
  4. * SPDX-License-Identifier: Apache-2.0
  5. *
  6. * Change Logs:
  7. * Date Author Notes
  8. * 2019-11-01 Jesven The first version
  9. * 2022-12-13 WangXiaoyao Hot-pluggable, extensible
  10. * page management algorithm
  11. */
  12. #include <rtthread.h>
  13. #include <stddef.h>
  14. #include <stdint.h>
  15. #include <string.h>
  16. #include "mm_fault.h"
  17. #include "mm_private.h"
  18. #include "mm_aspace.h"
  19. #include "mm_flag.h"
  20. #include "mm_page.h"
  21. #include <mmu.h>
  22. #define DBG_TAG "mm.page"
  23. #define DBG_LVL DBG_WARNING
  24. #include <rtdbg.h>
  25. RT_CTASSERT(order_huge_pg, RT_PAGE_MAX_ORDER > ARCH_PAGE_SHIFT - 2);
  26. RT_CTASSERT(size_width, sizeof(rt_size_t) == sizeof(void *));
  27. #ifdef RT_USING_SMART
  28. #include "lwp_arch_comm.h"
  29. #endif /* RT_USING_SMART */
  30. static rt_size_t init_mpr_align_start;
  31. static rt_size_t init_mpr_align_end;
  32. static void *init_mpr_cont_start;
  33. static struct rt_varea mpr_varea;
  34. static struct rt_page *page_list[RT_PAGE_MAX_ORDER];
  35. #define page_start ((rt_page_t)rt_mpr_start)
  36. static rt_size_t page_nr;
  37. static rt_size_t early_offset;
  38. static const char *get_name(rt_varea_t varea)
  39. {
  40. return "master-page-record";
  41. }
  42. static void hint_free(rt_mm_va_hint_t hint)
  43. {
  44. hint->flags = MMF_MAP_FIXED;
  45. hint->limit_start = rt_kernel_space.start;
  46. hint->limit_range_size = rt_kernel_space.size;
  47. hint->prefer = rt_mpr_start;
  48. }
  49. static void on_page_fault(struct rt_varea *varea, struct rt_aspace_fault_msg *msg)
  50. {
  51. void *init_start = (void *)init_mpr_align_start;
  52. void *init_end = (void *)init_mpr_align_end;
  53. if (msg->fault_vaddr < init_end && msg->fault_vaddr >= init_start)
  54. {
  55. rt_size_t offset = msg->fault_vaddr - init_start;
  56. msg->response.status = MM_FAULT_STATUS_OK;
  57. msg->response.vaddr = init_mpr_cont_start + offset;
  58. msg->response.size = ARCH_PAGE_SIZE;
  59. }
  60. else
  61. {
  62. void *raw_page = rt_pages_alloc(0);
  63. msg->response.status = MM_FAULT_STATUS_OK;
  64. msg->response.vaddr = raw_page;
  65. msg->response.size = ARCH_PAGE_SIZE;
  66. }
  67. }
  68. static struct rt_mem_obj mm_page_mapper = {
  69. .get_name = get_name,
  70. .on_page_fault = on_page_fault,
  71. .hint_free = hint_free,
  72. };
  73. static inline void *page_to_addr(rt_page_t page)
  74. {
  75. return (void *)((page - page_start) << ARCH_PAGE_SHIFT) - PV_OFFSET;
  76. }
  77. static inline rt_page_t addr_to_page(rt_page_t pg_start, void *addr)
  78. {
  79. addr += PV_OFFSET;
  80. return &pg_start[((uintptr_t)addr >> ARCH_PAGE_SHIFT)];
  81. }
  82. #define FLOOR(val, align) (((rt_size_t)(val) + (align)-1) & ~((align)-1))
  83. const rt_size_t shadow_mask =
  84. ((1ul << (RT_PAGE_MAX_ORDER + ARCH_PAGE_SHIFT - 1)) - 1);
  85. const rt_size_t rt_mpr_size = FLOOR(
  86. ((1ul << (ARCH_VADDR_WIDTH - ARCH_PAGE_SHIFT))) * sizeof(struct rt_page),
  87. ARCH_PAGE_SIZE);
  88. void *rt_mpr_start;
  89. rt_weak int rt_hw_clz(unsigned long n)
  90. {
  91. return __builtin_clzl(n);
  92. }
  93. rt_weak int rt_hw_ctz(unsigned long n)
  94. {
  95. return __builtin_ctzl(n);
  96. }
  97. rt_size_t rt_page_bits(rt_size_t size)
  98. {
  99. int bit = sizeof(rt_size_t) * 8 - rt_hw_clz(size) - 1;
  100. if ((size ^ (1UL << bit)) != 0)
  101. {
  102. bit++;
  103. }
  104. bit -= ARCH_PAGE_SHIFT;
  105. if (bit < 0)
  106. {
  107. bit = 0;
  108. }
  109. return bit;
  110. }
  111. struct rt_page *rt_page_addr2page(void *addr)
  112. {
  113. return addr_to_page(page_start, addr);
  114. }
  115. void *rt_page_page2addr(struct rt_page *p)
  116. {
  117. return page_to_addr(p);
  118. }
  119. static inline struct rt_page *buddy_get(struct rt_page *p,
  120. rt_uint32_t size_bits)
  121. {
  122. rt_size_t addr;
  123. addr = (rt_size_t)rt_page_page2addr(p);
  124. addr ^= (1UL << (size_bits + ARCH_PAGE_SHIFT));
  125. return rt_page_addr2page((void *)addr);
  126. }
  127. static void page_remove(struct rt_page *p, rt_uint32_t size_bits)
  128. {
  129. if (p->pre)
  130. {
  131. p->pre->next = p->next;
  132. }
  133. else
  134. {
  135. page_list[size_bits] = p->next;
  136. }
  137. if (p->next)
  138. {
  139. p->next->pre = p->pre;
  140. }
  141. p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  142. }
  143. static void page_insert(struct rt_page *p, rt_uint32_t size_bits)
  144. {
  145. p->next = page_list[size_bits];
  146. if (p->next)
  147. {
  148. p->next->pre = p;
  149. }
  150. p->pre = 0;
  151. page_list[size_bits] = p;
  152. p->size_bits = size_bits;
  153. }
  154. static void _pages_ref_inc(struct rt_page *p, rt_uint32_t size_bits)
  155. {
  156. struct rt_page *page_head;
  157. int idx;
  158. /* find page group head */
  159. idx = p - page_start;
  160. idx = idx & ~((1UL << size_bits) - 1);
  161. page_head = page_start + idx;
  162. page_head = (void *)page_head + early_offset;
  163. page_head->ref_cnt++;
  164. }
  165. static int _pages_ref_get(struct rt_page *p, rt_uint32_t size_bits)
  166. {
  167. struct rt_page *page_head;
  168. int idx;
  169. /* find page group head */
  170. idx = p - page_start;
  171. idx = idx & ~((1UL << size_bits) - 1);
  172. page_head = page_start + idx;
  173. return page_head->ref_cnt;
  174. }
  175. static int _pages_free(struct rt_page *p, rt_uint32_t size_bits)
  176. {
  177. rt_uint32_t level = size_bits;
  178. struct rt_page *buddy;
  179. RT_ASSERT(p >= page_start);
  180. RT_ASSERT((void *)p < rt_mpr_start + rt_mpr_size);
  181. RT_ASSERT(rt_kmem_v2p(p));
  182. RT_ASSERT(p->ref_cnt > 0);
  183. RT_ASSERT(p->size_bits == ARCH_ADDRESS_WIDTH_BITS);
  184. RT_ASSERT(size_bits < RT_PAGE_MAX_ORDER);
  185. p->ref_cnt--;
  186. if (p->ref_cnt != 0)
  187. {
  188. return 0;
  189. }
  190. while (level < RT_PAGE_MAX_ORDER - 1)
  191. {
  192. buddy = buddy_get(p, level);
  193. if (buddy && buddy->size_bits == level)
  194. {
  195. page_remove(buddy, level);
  196. p = (p < buddy) ? p : buddy;
  197. level++;
  198. }
  199. else
  200. {
  201. break;
  202. }
  203. }
  204. page_insert(p, level);
  205. return 1;
  206. }
  207. static struct rt_page *_pages_alloc(rt_uint32_t size_bits)
  208. {
  209. struct rt_page *p;
  210. if (page_list[size_bits])
  211. {
  212. p = page_list[size_bits];
  213. page_remove(p, size_bits);
  214. }
  215. else
  216. {
  217. rt_uint32_t level;
  218. for (level = size_bits + 1; level < RT_PAGE_MAX_ORDER; level++)
  219. {
  220. if (page_list[level])
  221. {
  222. break;
  223. }
  224. }
  225. if (level == RT_PAGE_MAX_ORDER)
  226. {
  227. return 0;
  228. }
  229. p = page_list[level];
  230. page_remove(p, level);
  231. while (level > size_bits)
  232. {
  233. page_insert(p, level - 1);
  234. p = buddy_get(p, level - 1);
  235. level--;
  236. }
  237. }
  238. p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  239. p->ref_cnt = 1;
  240. return p;
  241. }
  242. static void _early_page_remove(rt_page_t page, rt_uint32_t size_bits)
  243. {
  244. rt_page_t page_cont = (void *)page + early_offset;
  245. if (page_cont->pre)
  246. {
  247. rt_page_t pre_cont = (void *)page_cont->pre + early_offset;
  248. pre_cont->next = page_cont->next;
  249. }
  250. else
  251. {
  252. page_list[size_bits] = page_cont->next;
  253. }
  254. if (page_cont->next)
  255. {
  256. rt_page_t next_cont = (void *)page_cont->next + early_offset;
  257. next_cont->pre = page_cont->pre;
  258. }
  259. page_cont->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  260. }
  261. static void _early_page_insert(rt_page_t page, int size_bits)
  262. {
  263. RT_ASSERT((void *)page >= rt_mpr_start &&
  264. (void *)page - rt_mpr_start < +rt_mpr_size);
  265. rt_page_t page_cont = (void *)page + early_offset;
  266. page_cont->next = page_list[size_bits];
  267. if (page_cont->next)
  268. {
  269. rt_page_t next_cont = (void *)page_cont->next + early_offset;
  270. next_cont->pre = page;
  271. }
  272. page_cont->pre = 0;
  273. page_list[size_bits] = page;
  274. page_cont->size_bits = size_bits;
  275. }
  276. static struct rt_page *_early_pages_alloc(rt_uint32_t size_bits)
  277. {
  278. struct rt_page *p;
  279. if (page_list[size_bits])
  280. {
  281. p = page_list[size_bits];
  282. _early_page_remove(p, size_bits);
  283. }
  284. else
  285. {
  286. rt_uint32_t level;
  287. for (level = size_bits + 1; level < RT_PAGE_MAX_ORDER; level++)
  288. {
  289. if (page_list[level])
  290. {
  291. break;
  292. }
  293. }
  294. if (level == RT_PAGE_MAX_ORDER)
  295. {
  296. return 0;
  297. }
  298. p = page_list[level];
  299. _early_page_remove(p, level);
  300. while (level > size_bits)
  301. {
  302. _early_page_insert(p, level - 1);
  303. p = buddy_get(p, level - 1);
  304. level--;
  305. }
  306. }
  307. rt_page_t page_cont = (void *)p + early_offset;
  308. page_cont->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  309. page_cont->ref_cnt = 1;
  310. return p;
  311. }
  312. int rt_page_ref_get(void *addr, rt_uint32_t size_bits)
  313. {
  314. struct rt_page *p;
  315. rt_base_t level;
  316. int ref;
  317. p = rt_page_addr2page(addr);
  318. level = rt_hw_interrupt_disable();
  319. ref = _pages_ref_get(p, size_bits);
  320. rt_hw_interrupt_enable(level);
  321. return ref;
  322. }
  323. void rt_page_ref_inc(void *addr, rt_uint32_t size_bits)
  324. {
  325. struct rt_page *p;
  326. rt_base_t level;
  327. p = rt_page_addr2page(addr);
  328. level = rt_hw_interrupt_disable();
  329. _pages_ref_inc(p, size_bits);
  330. rt_hw_interrupt_enable(level);
  331. }
  332. static rt_page_t (*pages_alloc_handler)(rt_uint32_t size_bits);
  333. void *rt_pages_alloc(rt_uint32_t size_bits)
  334. {
  335. void *alloc_buf = RT_NULL;
  336. struct rt_page *p;
  337. rt_base_t level;
  338. level = rt_hw_interrupt_disable();
  339. p = pages_alloc_handler(size_bits);
  340. rt_hw_interrupt_enable(level);
  341. if (p)
  342. {
  343. alloc_buf = page_to_addr(p);
  344. }
  345. return alloc_buf;
  346. }
  347. int rt_pages_free(void *addr, rt_uint32_t size_bits)
  348. {
  349. struct rt_page *p;
  350. int real_free = 0;
  351. p = rt_page_addr2page(addr);
  352. if (p)
  353. {
  354. rt_base_t level;
  355. level = rt_hw_interrupt_disable();
  356. real_free = _pages_free(p, size_bits);
  357. rt_hw_interrupt_enable(level);
  358. }
  359. return real_free;
  360. }
  361. void rt_page_list(void) __attribute__((alias("list_page")));
  362. void list_page(void)
  363. {
  364. int i;
  365. rt_size_t total = 0;
  366. rt_base_t level;
  367. level = rt_hw_interrupt_disable();
  368. for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
  369. {
  370. struct rt_page *p = page_list[i];
  371. rt_kprintf("level %d ", i);
  372. while (p)
  373. {
  374. total += (1UL << i);
  375. rt_kprintf("[0x%08p]", rt_page_page2addr(p));
  376. p = p->next;
  377. }
  378. rt_kprintf("\n");
  379. }
  380. rt_hw_interrupt_enable(level);
  381. rt_kprintf("free pages is 0x%08lx (%ld KB)\n", total, total * ARCH_PAGE_SIZE / 1024);
  382. rt_kprintf("-------------------------------\n");
  383. }
  384. MSH_CMD_EXPORT(list_page, show page info);
  385. void rt_page_get_info(rt_size_t *total_nr, rt_size_t *free_nr)
  386. {
  387. int i;
  388. rt_size_t total_free = 0;
  389. rt_base_t level;
  390. level = rt_hw_interrupt_disable();
  391. for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
  392. {
  393. struct rt_page *p = page_list[i];
  394. while (p)
  395. {
  396. total_free += (1UL << i);
  397. p = p->next;
  398. }
  399. }
  400. rt_hw_interrupt_enable(level);
  401. *total_nr = page_nr;
  402. *free_nr = total_free;
  403. }
  404. void rt_page_init(rt_region_t reg)
  405. {
  406. int i;
  407. rt_region_t shadow;
  408. /* inclusive start, exclusive end */
  409. reg.start += ARCH_PAGE_MASK;
  410. reg.start &= ~ARCH_PAGE_MASK;
  411. reg.end &= ~ARCH_PAGE_MASK;
  412. if (reg.end <= reg.start)
  413. {
  414. LOG_E("region end(%p) must greater than start(%p)", reg.start, reg.end);
  415. RT_ASSERT(0);
  416. }
  417. page_nr = ((reg.end - reg.start) >> ARCH_PAGE_SHIFT);
  418. shadow.start = reg.start & ~shadow_mask;
  419. shadow.end = FLOOR(reg.end, shadow_mask + 1);
  420. LOG_D("[Init page] start: 0x%lx, end: 0x%lx, total: 0x%lx", reg.start,
  421. reg.end, page_nr);
  422. int err;
  423. /* init free list */
  424. for (i = 0; i < RT_PAGE_MAX_ORDER; i++)
  425. {
  426. page_list[i] = 0;
  427. }
  428. /* map MPR area */
  429. err = rt_aspace_map_static(&rt_kernel_space, &mpr_varea, &rt_mpr_start,
  430. rt_mpr_size, MMU_MAP_K_RWCB, MMF_MAP_FIXED,
  431. &mm_page_mapper, 0);
  432. if (err != RT_EOK)
  433. {
  434. LOG_E("MPR map failed with size %lx at %p", rt_mpr_size, rt_mpr_start);
  435. RT_ASSERT(0);
  436. }
  437. /* calculate footprint */
  438. init_mpr_align_start =
  439. (rt_size_t)addr_to_page(page_start, (void *)shadow.start) &
  440. ~ARCH_PAGE_MASK;
  441. init_mpr_align_end =
  442. FLOOR(addr_to_page(page_start, (void *)shadow.end), ARCH_PAGE_SIZE);
  443. rt_size_t init_mpr_size = init_mpr_align_end - init_mpr_align_start;
  444. rt_size_t init_mpr_npage = init_mpr_size >> ARCH_PAGE_SHIFT;
  445. init_mpr_cont_start = (void *)reg.start;
  446. void *init_mpr_cont_end = init_mpr_cont_start + init_mpr_size;
  447. early_offset = init_mpr_cont_start - (void *)init_mpr_align_start;
  448. rt_page_t mpr_cont = rt_mpr_start + early_offset;
  449. /* mark init mpr pages as illegal */
  450. rt_page_t head_cont = addr_to_page(mpr_cont, (void *)reg.start);
  451. rt_page_t tail_cont = addr_to_page(mpr_cont, (void *)reg.end);
  452. for (rt_page_t iter = head_cont; iter < tail_cont; iter++)
  453. {
  454. iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  455. }
  456. /* mark shadow pages as illegal */
  457. rt_page_t shad_head_cont = addr_to_page(mpr_cont, (void *)shadow.start);
  458. for (rt_page_t iter = shad_head_cont; iter < head_cont; iter++)
  459. {
  460. iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  461. }
  462. rt_page_t shad_tail_cont = addr_to_page(mpr_cont, (void *)shadow.end);
  463. for (rt_page_t iter = tail_cont; iter < shad_tail_cont; iter++)
  464. {
  465. iter->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  466. }
  467. /* insert reserved pages to list */
  468. reg.start = (rt_size_t)init_mpr_cont_end;
  469. const int max_order = RT_PAGE_MAX_ORDER + ARCH_PAGE_SHIFT - 1;
  470. while (reg.start != reg.end)
  471. {
  472. struct rt_page *p;
  473. int align_bits;
  474. int size_bits;
  475. size_bits =
  476. ARCH_ADDRESS_WIDTH_BITS - 1 - rt_hw_clz(reg.end - reg.start);
  477. align_bits = rt_hw_ctz(reg.start);
  478. if (align_bits < size_bits)
  479. {
  480. size_bits = align_bits;
  481. }
  482. if (size_bits > max_order)
  483. {
  484. size_bits = max_order;
  485. }
  486. p = addr_to_page(mpr_cont, (void *)reg.start);
  487. p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  488. p->ref_cnt = 0;
  489. /* insert to list */
  490. _early_page_insert((void *)p - early_offset,
  491. size_bits - ARCH_PAGE_SHIFT);
  492. reg.start += (1UL << size_bits);
  493. }
  494. pages_alloc_handler = _early_pages_alloc;
  495. /* doing the page table bushiness */
  496. if (rt_aspace_load_page(&rt_kernel_space, (void *)init_mpr_align_start, init_mpr_npage))
  497. {
  498. LOG_E("%s: failed to load pages", __func__);
  499. RT_ASSERT(0);
  500. }
  501. if (rt_hw_mmu_tbl_get() == rt_kernel_space.page_table)
  502. rt_page_cleanup();
  503. }
  504. static int _load_mpr_area(void *head, void *tail)
  505. {
  506. int err = 0;
  507. void *iter = (void *)((uintptr_t)head & ~ARCH_PAGE_MASK);
  508. tail = (void *)FLOOR(tail, ARCH_PAGE_SIZE);
  509. while (iter != tail)
  510. {
  511. void *paddr = rt_kmem_v2p(iter);
  512. if (paddr == ARCH_MAP_FAILED)
  513. {
  514. err = rt_aspace_load_page(&rt_kernel_space, iter, 1);
  515. if (err != RT_EOK)
  516. {
  517. LOG_E("%s: failed to load page", __func__);
  518. break;
  519. }
  520. }
  521. iter += ARCH_PAGE_SIZE;
  522. }
  523. return err;
  524. }
  525. int rt_page_install(rt_region_t region)
  526. {
  527. int err = -RT_EINVAL;
  528. if (region.end != region.start && !(region.start & ARCH_PAGE_MASK) &&
  529. !(region.end & ARCH_PAGE_MASK) &&
  530. !((region.end - region.start) & shadow_mask))
  531. {
  532. void *head = addr_to_page(page_start, (void *)region.start);
  533. void *tail = addr_to_page(page_start, (void *)region.end);
  534. page_nr += ((region.end - region.start) >> ARCH_PAGE_SHIFT);
  535. err = _load_mpr_area(head, tail);
  536. if (err == RT_EOK)
  537. {
  538. while (region.start != region.end)
  539. {
  540. struct rt_page *p;
  541. int size_bits;
  542. size_bits = RT_PAGE_MAX_ORDER - 1;
  543. p = addr_to_page(page_start, (void *)region.start);
  544. p->size_bits = ARCH_ADDRESS_WIDTH_BITS;
  545. p->ref_cnt = 1;
  546. _pages_free(p, size_bits);
  547. region.start += (1UL << (size_bits + ARCH_PAGE_SHIFT));
  548. }
  549. }
  550. }
  551. return err;
  552. }
  553. void rt_page_cleanup(void)
  554. {
  555. early_offset = 0;
  556. pages_alloc_handler = _pages_alloc;
  557. }