lwp_user_mm.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660
  1. /*
  2. * Copyright (c) 2006-2021, RT-Thread Development Team
  3. *
  4. * SPDX-License-Identifier: Apache-2.0
  5. *
  6. * Change Logs:
  7. * Date Author Notes
  8. * 2019-10-28 Jesven first version
  9. * 2021-02-06 lizhirui fixed fixed vtable size problem
  10. * 2021-02-12 lizhirui add 64-bit support for lwp_brk
  11. * 2021-02-19 lizhirui add riscv64 support for lwp_user_accessable and lwp_get_from_user
  12. * 2021-06-07 lizhirui modify user space bound check
  13. */
  14. #include <rtthread.h>
  15. #include <rthw.h>
  16. #include <string.h>
  17. #ifdef ARCH_MM_MMU
  18. #include <lwp.h>
  19. #include <lwp_arch.h>
  20. #include <lwp_mm.h>
  21. #include <lwp_user_mm.h>
  22. #include <mm_aspace.h>
  23. #include <mm_fault.h>
  24. #include <mm_flag.h>
  25. #include <mm_page.h>
  26. #include <mmu.h>
  27. #include <page.h>
  28. #define DBG_TAG "LwP"
  29. #define DBG_LVL DBG_LOG
  30. #include <rtdbg.h>
  31. static void _init_lwp_objs(struct rt_lwp_objs *lwp_objs, rt_aspace_t aspace);
  32. int lwp_user_space_init(struct rt_lwp *lwp, rt_bool_t is_fork)
  33. {
  34. int err = -RT_ENOMEM;
  35. lwp->lwp_obj = rt_malloc(sizeof(struct rt_lwp_objs));
  36. _init_lwp_objs(lwp->lwp_obj, lwp->aspace);
  37. if (lwp->lwp_obj)
  38. {
  39. err = arch_user_space_init(lwp);
  40. if (!is_fork && err == RT_EOK)
  41. {
  42. void *addr = (void *)USER_STACK_VSTART;
  43. err = rt_aspace_map(lwp->aspace, &addr,
  44. USER_STACK_VEND - USER_STACK_VSTART,
  45. MMU_MAP_U_RWCB, 0, &lwp->lwp_obj->mem_obj, 0);
  46. }
  47. }
  48. return err;
  49. }
  50. void lwp_aspace_switch(struct rt_thread *thread)
  51. {
  52. struct rt_lwp *lwp = RT_NULL;
  53. rt_aspace_t aspace;
  54. void *from_tbl;
  55. if (thread->lwp)
  56. {
  57. lwp = (struct rt_lwp *)thread->lwp;
  58. aspace = lwp->aspace;
  59. }
  60. else
  61. aspace = &rt_kernel_space;
  62. from_tbl = rt_hw_mmu_tbl_get();
  63. if (aspace->page_table != from_tbl)
  64. {
  65. rt_hw_aspace_switch(aspace);
  66. }
  67. }
  68. void lwp_unmap_user_space(struct rt_lwp *lwp)
  69. {
  70. arch_user_space_free(lwp);
  71. rt_free(lwp->lwp_obj);
  72. }
  73. static const char *user_get_name(rt_varea_t varea)
  74. {
  75. char *name;
  76. if (varea->flag & MMF_TEXT)
  77. {
  78. name = "user.text";
  79. }
  80. else
  81. {
  82. if (varea->start == (void *)USER_STACK_VSTART)
  83. {
  84. name = "user.stack";
  85. }
  86. else if (varea->start >= (void *)USER_HEAP_VADDR &&
  87. varea->start < (void *)USER_HEAP_VEND)
  88. {
  89. name = "user.heap";
  90. }
  91. else
  92. {
  93. name = "user.data";
  94. }
  95. }
  96. return name;
  97. }
  98. static void _user_do_page_fault(struct rt_varea *varea,
  99. struct rt_mm_fault_msg *msg)
  100. {
  101. struct rt_lwp_objs *lwp_objs;
  102. lwp_objs = rt_container_of(varea->mem_obj, struct rt_lwp_objs, mem_obj);
  103. if (lwp_objs->source)
  104. {
  105. void *paddr = rt_hw_mmu_v2p(lwp_objs->source, msg->vaddr);
  106. if (paddr != ARCH_MAP_FAILED)
  107. {
  108. void *vaddr;
  109. vaddr = paddr - PV_OFFSET;
  110. if (!(varea->flag & MMF_TEXT))
  111. {
  112. void *cp = rt_pages_alloc(0);
  113. if (cp)
  114. {
  115. memcpy(cp, vaddr, ARCH_PAGE_SIZE);
  116. rt_varea_insert_page(varea, cp);
  117. msg->response.status = MM_FAULT_STATUS_OK;
  118. msg->response.vaddr = cp;
  119. msg->response.size = ARCH_PAGE_SIZE;
  120. }
  121. else
  122. {
  123. LOG_W("%s: page alloc failed at %p", __func__,
  124. varea->start);
  125. }
  126. }
  127. else
  128. {
  129. rt_page_t page = rt_page_addr2page(vaddr);
  130. page->ref_cnt += 1;
  131. rt_varea_insert_page(varea, vaddr);
  132. msg->response.status = MM_FAULT_STATUS_OK;
  133. msg->response.vaddr = vaddr;
  134. msg->response.size = ARCH_PAGE_SIZE;
  135. }
  136. }
  137. else if (!(varea->flag & MMF_TEXT))
  138. {
  139. /* if data segment not exist in source do a fallback */
  140. rt_mm_dummy_mapper.on_page_fault(varea, msg);
  141. }
  142. }
  143. else /* if (!lwp_objs->source), no aspace as source data */
  144. {
  145. rt_mm_dummy_mapper.on_page_fault(varea, msg);
  146. }
  147. }
  148. static void _init_lwp_objs(struct rt_lwp_objs *lwp_objs, rt_aspace_t aspace)
  149. {
  150. lwp_objs->source = NULL;
  151. lwp_objs->mem_obj.get_name = user_get_name;
  152. lwp_objs->mem_obj.hint_free = NULL;
  153. lwp_objs->mem_obj.on_page_fault = _user_do_page_fault;
  154. lwp_objs->mem_obj.on_page_offload = rt_mm_dummy_mapper.on_page_offload;
  155. lwp_objs->mem_obj.on_varea_open = rt_mm_dummy_mapper.on_varea_open;
  156. lwp_objs->mem_obj.on_varea_close = rt_mm_dummy_mapper.on_varea_close;
  157. }
  158. static void *_lwp_map_user(struct rt_lwp *lwp, void *map_va, size_t map_size,
  159. int text)
  160. {
  161. void *va = map_va;
  162. int ret = 0;
  163. size_t flags = MMF_PREFETCH;
  164. if (text)
  165. flags |= MMF_TEXT;
  166. rt_mem_obj_t mem_obj = &lwp->lwp_obj->mem_obj;
  167. ret = rt_aspace_map(lwp->aspace, &va, map_size, MMU_MAP_U_RWCB, flags,
  168. mem_obj, 0);
  169. if (ret != RT_EOK)
  170. {
  171. va = RT_NULL;
  172. LOG_I("lwp_map_user: failed to map %lx with size %lx with errno %d", map_va,
  173. map_size, ret);
  174. }
  175. return va;
  176. }
  177. int lwp_unmap_user(struct rt_lwp *lwp, void *va)
  178. {
  179. int err;
  180. err = rt_aspace_unmap(lwp->aspace, va, 1);
  181. return err;
  182. }
  183. static void _dup_varea(rt_varea_t varea, struct rt_lwp *src_lwp,
  184. rt_aspace_t dst)
  185. {
  186. void *vaddr = varea->start;
  187. void *vend = vaddr + varea->size;
  188. if (vaddr < (void *)USER_STACK_VSTART || vaddr >= (void *)USER_STACK_VEND)
  189. {
  190. while (vaddr != vend)
  191. {
  192. void *paddr;
  193. paddr = lwp_v2p(src_lwp, vaddr);
  194. if (paddr != ARCH_MAP_FAILED)
  195. {
  196. rt_aspace_load_page(dst, vaddr, 1);
  197. }
  198. vaddr += ARCH_PAGE_SIZE;
  199. }
  200. }
  201. else
  202. {
  203. while (vaddr != vend)
  204. {
  205. vend -= ARCH_PAGE_SIZE;
  206. void *paddr;
  207. paddr = lwp_v2p(src_lwp, vend);
  208. if (paddr != ARCH_MAP_FAILED)
  209. {
  210. rt_aspace_load_page(dst, vend, 1);
  211. }
  212. else
  213. {
  214. break;
  215. }
  216. }
  217. }
  218. }
  219. int lwp_dup_user(rt_varea_t varea, void *arg)
  220. {
  221. int err;
  222. struct rt_lwp *self_lwp = lwp_self();
  223. struct rt_lwp *new_lwp = (struct rt_lwp *)arg;
  224. void *pa = RT_NULL;
  225. void *va = RT_NULL;
  226. rt_mem_obj_t mem_obj = varea->mem_obj;
  227. if (!mem_obj)
  228. {
  229. /* duplicate a physical mapping */
  230. pa = lwp_v2p(self_lwp, (void *)varea->start);
  231. RT_ASSERT(pa != ARCH_MAP_FAILED);
  232. struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED,
  233. .limit_range_size = new_lwp->aspace->size,
  234. .limit_start = new_lwp->aspace->start,
  235. .prefer = varea->start,
  236. .map_size = varea->size};
  237. err = rt_aspace_map_phy(new_lwp->aspace, &hint, varea->attr,
  238. MM_PA_TO_OFF(pa), &va);
  239. if (err != RT_EOK)
  240. {
  241. LOG_W("%s: aspace map failed at %p with size %p", __func__,
  242. varea->start, varea->size);
  243. }
  244. }
  245. else
  246. {
  247. /* duplicate a mem_obj backing mapping */
  248. va = varea->start;
  249. err = rt_aspace_map(new_lwp->aspace, &va, varea->size, varea->attr,
  250. varea->flag, &new_lwp->lwp_obj->mem_obj,
  251. varea->offset);
  252. if (err != RT_EOK)
  253. {
  254. LOG_W("%s: aspace map failed at %p with size %p", __func__,
  255. varea->start, varea->size);
  256. }
  257. else
  258. {
  259. /* loading page frames for !MMF_PREFETCH varea */
  260. if (!(varea->flag & MMF_PREFETCH))
  261. {
  262. _dup_varea(varea, self_lwp, new_lwp->aspace);
  263. }
  264. }
  265. }
  266. if (va != (void *)varea->start)
  267. {
  268. return -1;
  269. }
  270. return 0;
  271. }
  272. int lwp_unmap_user_phy(struct rt_lwp *lwp, void *va)
  273. {
  274. return lwp_unmap_user(lwp, va);
  275. }
  276. void *lwp_map_user(struct rt_lwp *lwp, void *map_va, size_t map_size, int text)
  277. {
  278. void *ret = RT_NULL;
  279. size_t offset = 0;
  280. if (!map_size)
  281. {
  282. return 0;
  283. }
  284. offset = (size_t)map_va & ARCH_PAGE_MASK;
  285. map_size += (offset + ARCH_PAGE_SIZE - 1);
  286. map_size &= ~ARCH_PAGE_MASK;
  287. map_va = (void *)((size_t)map_va & ~ARCH_PAGE_MASK);
  288. ret = _lwp_map_user(lwp, map_va, map_size, text);
  289. if (ret)
  290. {
  291. ret = (void *)((char *)ret + offset);
  292. }
  293. return ret;
  294. }
  295. void *lwp_map_user_phy(struct rt_lwp *lwp, void *map_va, void *map_pa,
  296. size_t map_size, int cached)
  297. {
  298. int err;
  299. void *va;
  300. size_t offset = 0;
  301. if (!map_size)
  302. {
  303. return 0;
  304. }
  305. if (map_va)
  306. {
  307. if (((size_t)map_va & ARCH_PAGE_MASK) !=
  308. ((size_t)map_pa & ARCH_PAGE_MASK))
  309. {
  310. return 0;
  311. }
  312. }
  313. offset = (size_t)map_pa & ARCH_PAGE_MASK;
  314. map_size += (offset + ARCH_PAGE_SIZE - 1);
  315. map_size &= ~ARCH_PAGE_MASK;
  316. map_pa = (void *)((size_t)map_pa & ~ARCH_PAGE_MASK);
  317. struct rt_mm_va_hint hint = {.flags = MMF_MAP_FIXED,
  318. .limit_range_size = lwp->aspace->size,
  319. .limit_start = lwp->aspace->start,
  320. .prefer = map_va,
  321. .map_size = map_size};
  322. rt_size_t attr = cached ? MMU_MAP_U_RWCB : MMU_MAP_U_RW;
  323. err =
  324. rt_aspace_map_phy(lwp->aspace, &hint, attr, MM_PA_TO_OFF(map_pa), &va);
  325. if (err != RT_EOK)
  326. {
  327. va = RT_NULL;
  328. LOG_W("%s", __func__);
  329. }
  330. else
  331. {
  332. va += offset;
  333. }
  334. return va;
  335. }
  336. rt_base_t lwp_brk(void *addr)
  337. {
  338. rt_base_t ret = -1;
  339. struct rt_lwp *lwp = RT_NULL;
  340. rt_mm_lock();
  341. lwp = rt_thread_self()->lwp;
  342. if ((size_t)addr <= lwp->end_heap)
  343. {
  344. ret = (rt_base_t)lwp->end_heap;
  345. }
  346. else
  347. {
  348. size_t size = 0;
  349. void *va = RT_NULL;
  350. if ((size_t)addr <= USER_HEAP_VEND)
  351. {
  352. size = (((size_t)addr - lwp->end_heap) + ARCH_PAGE_SIZE - 1) &
  353. ~ARCH_PAGE_MASK;
  354. va = lwp_map_user(lwp, (void *)lwp->end_heap, size, 0);
  355. }
  356. if (va)
  357. {
  358. lwp->end_heap += size;
  359. ret = lwp->end_heap;
  360. }
  361. }
  362. rt_mm_unlock();
  363. return ret;
  364. }
  365. #define MAP_ANONYMOUS 0x20
  366. void *lwp_mmap2(void *addr, size_t length, int prot, int flags, int fd,
  367. off_t pgoffset)
  368. {
  369. void *ret = (void *)-1;
  370. if (fd == -1)
  371. {
  372. ret = lwp_map_user(lwp_self(), addr, length, 0);
  373. if (ret)
  374. {
  375. if ((flags & MAP_ANONYMOUS) != 0)
  376. {
  377. rt_memset(ret, 0, length);
  378. }
  379. }
  380. else
  381. {
  382. ret = (void *)-1;
  383. }
  384. }
  385. else
  386. {
  387. struct dfs_fd *d;
  388. d = fd_get(fd);
  389. if (d && d->vnode->type == FT_DEVICE)
  390. {
  391. struct dfs_mmap2_args mmap2;
  392. mmap2.addr = addr;
  393. mmap2.length = length;
  394. mmap2.prot = prot;
  395. mmap2.flags = flags;
  396. mmap2.pgoffset = pgoffset;
  397. mmap2.ret = (void *)-1;
  398. if (dfs_file_mmap2(d, &mmap2) == 0)
  399. {
  400. ret = mmap2.ret;
  401. }
  402. }
  403. }
  404. return ret;
  405. }
  406. int lwp_munmap(void *addr)
  407. {
  408. int ret = 0;
  409. rt_mm_lock();
  410. ret = lwp_unmap_user(lwp_self(), addr);
  411. rt_mm_unlock();
  412. return ret;
  413. }
  414. size_t lwp_get_from_user(void *dst, void *src, size_t size)
  415. {
  416. struct rt_lwp *lwp = RT_NULL;
  417. /* check src */
  418. if (src < (void *)USER_VADDR_START)
  419. {
  420. return 0;
  421. }
  422. if (src >= (void *)USER_VADDR_TOP)
  423. {
  424. return 0;
  425. }
  426. if ((void *)((char *)src + size) > (void *)USER_VADDR_TOP)
  427. {
  428. return 0;
  429. }
  430. lwp = lwp_self();
  431. if (!lwp)
  432. {
  433. return 0;
  434. }
  435. return lwp_data_get(lwp, dst, src, size);
  436. }
  437. size_t lwp_put_to_user(void *dst, void *src, size_t size)
  438. {
  439. struct rt_lwp *lwp = RT_NULL;
  440. /* check dst */
  441. if (dst < (void *)USER_VADDR_START)
  442. {
  443. return 0;
  444. }
  445. if (dst >= (void *)USER_VADDR_TOP)
  446. {
  447. return 0;
  448. }
  449. if ((void *)((char *)dst + size) > (void *)USER_VADDR_TOP)
  450. {
  451. return 0;
  452. }
  453. lwp = lwp_self();
  454. if (!lwp)
  455. {
  456. return 0;
  457. }
  458. return lwp_data_put(lwp, dst, src, size);
  459. }
  460. int lwp_user_accessable(void *addr, size_t size)
  461. {
  462. void *addr_start = RT_NULL, *addr_end = RT_NULL, *next_page = RT_NULL;
  463. void *tmp_addr = RT_NULL;
  464. struct rt_lwp *lwp = lwp_self();
  465. if (!lwp)
  466. {
  467. return 0;
  468. }
  469. if (!size || !addr)
  470. {
  471. return 0;
  472. }
  473. addr_start = addr;
  474. addr_end = (void *)((char *)addr + size);
  475. #ifdef ARCH_RISCV64
  476. if (addr_start < (void *)USER_VADDR_START)
  477. {
  478. return 0;
  479. }
  480. #else
  481. if (addr_start >= (void *)USER_VADDR_TOP)
  482. {
  483. return 0;
  484. }
  485. if (addr_end > (void *)USER_VADDR_TOP)
  486. {
  487. return 0;
  488. }
  489. #endif
  490. next_page =
  491. (void *)(((size_t)addr_start + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1));
  492. do
  493. {
  494. size_t len = (char *)next_page - (char *)addr_start;
  495. if (size < len)
  496. {
  497. len = size;
  498. }
  499. tmp_addr = lwp_v2p(lwp, addr_start);
  500. if (tmp_addr == ARCH_MAP_FAILED)
  501. {
  502. if ((rt_ubase_t)addr_start >= USER_STACK_VSTART && (rt_ubase_t)addr_start < USER_STACK_VEND)
  503. tmp_addr = *(void **)addr_start;
  504. else
  505. return 0;
  506. }
  507. addr_start = (void *)((char *)addr_start + len);
  508. size -= len;
  509. next_page = (void *)((char *)next_page + ARCH_PAGE_SIZE);
  510. } while (addr_start < addr_end);
  511. return 1;
  512. }
  513. /* src is in mmu_info space, dst is in current thread space */
  514. size_t lwp_data_get(struct rt_lwp *lwp, void *dst, void *src, size_t size)
  515. {
  516. size_t copy_len = 0;
  517. void *addr_start = RT_NULL, *addr_end = RT_NULL, *next_page = RT_NULL;
  518. void *tmp_dst = RT_NULL, *tmp_src = RT_NULL;
  519. if (!size || !dst)
  520. {
  521. return 0;
  522. }
  523. tmp_dst = dst;
  524. addr_start = src;
  525. addr_end = (void *)((char *)src + size);
  526. next_page =
  527. (void *)(((size_t)addr_start + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1));
  528. do
  529. {
  530. size_t len = (char *)next_page - (char *)addr_start;
  531. if (size < len)
  532. {
  533. len = size;
  534. }
  535. tmp_src = lwp_v2p(lwp, addr_start);
  536. if (tmp_src == ARCH_MAP_FAILED)
  537. {
  538. break;
  539. }
  540. tmp_src = (void *)((char *)tmp_src - PV_OFFSET);
  541. rt_memcpy(tmp_dst, tmp_src, len);
  542. tmp_dst = (void *)((char *)tmp_dst + len);
  543. addr_start = (void *)((char *)addr_start + len);
  544. size -= len;
  545. next_page = (void *)((char *)next_page + ARCH_PAGE_SIZE);
  546. copy_len += len;
  547. } while (addr_start < addr_end);
  548. return copy_len;
  549. }
  550. /* dst is in kernel space, src is in current thread space */
  551. size_t lwp_data_put(struct rt_lwp *lwp, void *dst, void *src, size_t size)
  552. {
  553. size_t copy_len = 0;
  554. void *addr_start = RT_NULL, *addr_end = RT_NULL, *next_page = RT_NULL;
  555. void *tmp_dst = RT_NULL, *tmp_src = RT_NULL;
  556. if (!size || !dst)
  557. {
  558. return 0;
  559. }
  560. tmp_src = src;
  561. addr_start = dst;
  562. addr_end = (void *)((char *)dst + size);
  563. next_page =
  564. (void *)(((size_t)addr_start + ARCH_PAGE_SIZE) & ~(ARCH_PAGE_SIZE - 1));
  565. do
  566. {
  567. size_t len = (char *)next_page - (char *)addr_start;
  568. if (size < len)
  569. {
  570. len = size;
  571. }
  572. tmp_dst = lwp_v2p(lwp, addr_start);
  573. if (tmp_dst == ARCH_MAP_FAILED)
  574. {
  575. break;
  576. }
  577. tmp_dst = (void *)((char *)tmp_dst - PV_OFFSET);
  578. rt_memcpy(tmp_dst, tmp_src, len);
  579. tmp_src = (void *)((char *)tmp_src + len);
  580. addr_start = (void *)((char *)addr_start + len);
  581. size -= len;
  582. next_page = (void *)((char *)next_page + ARCH_PAGE_SIZE);
  583. copy_len += len;
  584. } while (addr_start < addr_end);
  585. return copy_len;
  586. }
  587. #endif