1 // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
20 #include <freertos/FreeRTOS.h>
21 #include <freertos/task.h>
22 #include <freertos/semphr.h>
23 #include <rom/spi_flash.h>
24 #include <rom/cache.h>
26 #include <soc/dport_reg.h>
27 #include "sdkconfig.h"
30 #include "esp_spi_flash.h"
31 #include "esp_flash_encrypt.h"
33 #include "cache_utils.h"
34 #include "esp_spiram.h"
37 // Enable built-in checks in queue.h in debug builds
40 #include "rom/queue.h"
42 #define REGIONS_COUNT 4
43 #define PAGES_PER_REGION 64
44 #define INVALID_ENTRY_VAL 0x100
45 #define VADDR0_START_ADDR 0x3F400000
46 #define VADDR1_START_ADDR 0x40000000
47 #define VADDR1_FIRST_USABLE_ADDR 0x400D0000
48 #define PRO_IRAM0_FIRST_USABLE_PAGE ((VADDR1_FIRST_USABLE_ADDR - VADDR1_START_ADDR) / SPI_FLASH_MMU_PAGE_SIZE + 64)
50 /* Ensure pages in a region haven't been marked as written via
51 spi_flash_mark_modified_region(). If the page has
52 been written, flush the entire flash cache before returning.
54 This ensures stale cache entries are never read after fresh calls
55 to spi_flash_mmap(), while keeping the number of cache flushes to a
58 Returns true if cache was flushed.
61 static bool spi_flash_ensure_unmodified_region(size_t start_addr, size_t length);
63 typedef struct mmap_entry_{
67 LIST_ENTRY(mmap_entry_) entries;
71 static LIST_HEAD(mmap_entries_head, mmap_entry_) s_mmap_entries_head =
72 LIST_HEAD_INITIALIZER(s_mmap_entries_head);
73 static uint8_t s_mmap_page_refcnt[REGIONS_COUNT * PAGES_PER_REGION] = {0};
74 static uint32_t s_mmap_last_handle = 0;
77 static void IRAM_ATTR spi_flash_mmap_init()
79 if (s_mmap_page_refcnt[0] != 0) {
80 return; /* mmap data already initialised */
83 DPORT_STALL_OTHER_CPU_START();
84 for (int i = 0; i < REGIONS_COUNT * PAGES_PER_REGION; ++i) {
85 uint32_t entry_pro = DPORT_PRO_FLASH_MMU_TABLE[i];
86 uint32_t entry_app = DPORT_APP_FLASH_MMU_TABLE[i];
88 if (entry_pro != entry_app) {
89 // clean up entries used by boot loader
90 entry_pro = DPORT_FLASH_MMU_TABLE_INVALID_VAL;
91 DPORT_PRO_FLASH_MMU_TABLE[i] = DPORT_FLASH_MMU_TABLE_INVALID_VAL;
93 if ((entry_pro & INVALID_ENTRY_VAL) == 0 && (i == 0 || i == PRO_IRAM0_FIRST_USABLE_PAGE || entry_pro != 0)) {
94 s_mmap_page_refcnt[i] = 1;
96 DPORT_PRO_FLASH_MMU_TABLE[i] = DPORT_FLASH_MMU_TABLE_INVALID_VAL;
97 DPORT_APP_FLASH_MMU_TABLE[i] = DPORT_FLASH_MMU_TABLE_INVALID_VAL;
100 DPORT_STALL_OTHER_CPU_END();
103 static void IRAM_ATTR get_mmu_region(spi_flash_mmap_memory_t memory, int* out_begin, int* out_size,uint32_t* region_addr)
105 if (memory == SPI_FLASH_MMAP_DATA) {
109 *region_addr = VADDR0_START_ADDR;
111 // only part of VAddr1 is usable, so adjust for that
112 *out_begin = PRO_IRAM0_FIRST_USABLE_PAGE;
113 *out_size = 3 * 64 - *out_begin;
114 *region_addr = VADDR1_FIRST_USABLE_ADDR;
118 esp_err_t IRAM_ATTR spi_flash_mmap(size_t src_addr, size_t size, spi_flash_mmap_memory_t memory,
119 const void** out_ptr, spi_flash_mmap_handle_t* out_handle)
122 if (src_addr & 0xffff) {
123 return ESP_ERR_INVALID_ARG;
125 if (src_addr + size > g_rom_flashchip.chip_size) {
126 return ESP_ERR_INVALID_ARG;
128 // region which should be mapped
129 int phys_page = src_addr / SPI_FLASH_MMU_PAGE_SIZE;
130 int page_count = (size + SPI_FLASH_MMU_PAGE_SIZE - 1) / SPI_FLASH_MMU_PAGE_SIZE;
131 //prepare a linear pages array to feed into spi_flash_mmap_pages
132 int *pages=malloc(sizeof(int)*page_count);
134 return ESP_ERR_NO_MEM;
136 for (int i = 0; i < page_count; i++) {
137 pages[i] = phys_page+i;
139 ret=spi_flash_mmap_pages(pages, page_count, memory, out_ptr, out_handle);
144 esp_err_t IRAM_ATTR spi_flash_mmap_pages(int *pages, size_t page_count, spi_flash_mmap_memory_t memory,
145 const void** out_ptr, spi_flash_mmap_handle_t* out_handle)
148 bool did_flush, need_flush = false;
150 return ESP_ERR_INVALID_ARG;
152 for (int i = 0; i < page_count; i++) {
153 if (pages[i] < 0 || pages[i]*SPI_FLASH_MMU_PAGE_SIZE >= g_rom_flashchip.chip_size) {
154 return ESP_ERR_INVALID_ARG;
157 mmap_entry_t* new_entry = (mmap_entry_t*) heap_caps_malloc(sizeof(mmap_entry_t), MALLOC_CAP_INTERNAL|MALLOC_CAP_8BIT);
158 if (new_entry == 0) {
159 return ESP_ERR_NO_MEM;
162 spi_flash_disable_interrupts_caches_and_other_cpu();
165 for (int i = 0; i < page_count; i++) {
166 if (spi_flash_ensure_unmodified_region(pages[i]*SPI_FLASH_MMU_PAGE_SIZE, SPI_FLASH_MMU_PAGE_SIZE)) {
170 spi_flash_mmap_init();
171 // figure out the memory region where we should look for pages
172 int region_begin; // first page to check
173 int region_size; // number of pages to check
174 uint32_t region_addr; // base address of memory region
175 get_mmu_region(memory,®ion_begin,®ion_size,®ion_addr);
176 if (region_size < page_count) {
177 return ESP_ERR_NO_MEM;
179 // The following part searches for a range of MMU entries which can be used.
180 // Algorithm is essentially naïve strstr algorithm, except that unused MMU
181 // entries are treated as wildcards.
183 int end = region_begin + region_size - page_count;
184 for (start = region_begin; start <= end; ++start) {
187 DPORT_STALL_OTHER_CPU_START();
188 for (pos = start; pos < start + page_count; ++pos, ++pageno) {
189 int table_val = (int) DPORT_PRO_FLASH_MMU_TABLE[pos];
190 uint8_t refcnt = s_mmap_page_refcnt[pos];
191 if (refcnt != 0 && table_val != pages[pageno]) {
195 DPORT_STALL_OTHER_CPU_END();
196 // whole mapping range matched, bail out
197 if (pos - start == page_count) {
201 // checked all the region(s) and haven't found anything?
205 ret = ESP_ERR_NO_MEM;
207 // set up mapping using pages
209 DPORT_STALL_OTHER_CPU_START();
210 for (int i = start; i != start + page_count; ++i, ++pageno) {
211 // sanity check: we won't reconfigure entries with non-zero reference count
212 assert(s_mmap_page_refcnt[i] == 0 ||
213 (DPORT_PRO_FLASH_MMU_TABLE[i] == pages[pageno] &&
214 DPORT_APP_FLASH_MMU_TABLE[i] == pages[pageno]));
215 if (s_mmap_page_refcnt[i] == 0) {
216 if (DPORT_PRO_FLASH_MMU_TABLE[i] != pages[pageno] || DPORT_APP_FLASH_MMU_TABLE[i] != pages[pageno]) {
217 DPORT_PRO_FLASH_MMU_TABLE[i] = pages[pageno];
218 DPORT_APP_FLASH_MMU_TABLE[i] = pages[pageno];
222 ++s_mmap_page_refcnt[i];
224 DPORT_STALL_OTHER_CPU_END();
225 LIST_INSERT_HEAD(&s_mmap_entries_head, new_entry, entries);
226 new_entry->page = start;
227 new_entry->count = page_count;
228 new_entry->handle = ++s_mmap_last_handle;
229 *out_handle = new_entry->handle;
230 *out_ptr = (void*) (region_addr + (start - region_begin) * SPI_FLASH_MMU_PAGE_SIZE);
234 /* This is a temporary fix for an issue where some
235 cache reads may see stale data.
237 Working on a long term fix that doesn't require invalidating
240 if (!did_flush && need_flush) {
241 #if CONFIG_SPIRAM_SUPPORT
242 esp_spiram_writeback_cache();
248 spi_flash_enable_interrupts_caches_and_other_cpu();
249 if (*out_ptr == NULL) {
255 void IRAM_ATTR spi_flash_munmap(spi_flash_mmap_handle_t handle)
257 spi_flash_disable_interrupts_caches_and_other_cpu();
259 // look for handle in linked list
260 for (it = LIST_FIRST(&s_mmap_entries_head); it != NULL; it = LIST_NEXT(it, entries)) {
261 if (it->handle == handle) {
262 // for each page, decrement reference counter
263 // if reference count is zero, disable MMU table entry to
264 // facilitate debugging of use-after-free conditions
265 DPORT_STALL_OTHER_CPU_START();
266 for (int i = it->page; i < it->page + it->count; ++i) {
267 assert(s_mmap_page_refcnt[i] > 0);
268 if (--s_mmap_page_refcnt[i] == 0) {
269 DPORT_PRO_FLASH_MMU_TABLE[i] = INVALID_ENTRY_VAL;
270 DPORT_APP_FLASH_MMU_TABLE[i] = INVALID_ENTRY_VAL;
273 DPORT_STALL_OTHER_CPU_END();
274 LIST_REMOVE(it, entries);
278 spi_flash_enable_interrupts_caches_and_other_cpu();
280 assert(0 && "invalid handle, or handle already unmapped");
285 void spi_flash_mmap_dump()
287 spi_flash_mmap_init();
289 for (it = LIST_FIRST(&s_mmap_entries_head); it != NULL; it = LIST_NEXT(it, entries)) {
290 printf("handle=%d page=%d count=%d\n", it->handle, it->page, it->count);
292 for (int i = 0; i < REGIONS_COUNT * PAGES_PER_REGION; ++i) {
293 if (s_mmap_page_refcnt[i] != 0) {
294 printf("page %d: refcnt=%d paddr=%d\n",
295 i, (int) s_mmap_page_refcnt[i], DPORT_PRO_FLASH_MMU_TABLE[i]);
300 uint32_t spi_flash_mmap_get_free_pages(spi_flash_mmap_memory_t memory)
302 spi_flash_mmap_init();
304 int region_begin; // first page to check
305 int region_size; // number of pages to check
306 uint32_t region_addr; // base address of memory region
307 get_mmu_region(memory,®ion_begin,®ion_size,®ion_addr);
308 DPORT_STALL_OTHER_CPU_START();
309 for (int i = region_begin; i < region_begin + region_size; ++i) {
310 if (s_mmap_page_refcnt[i] == 0 && DPORT_PRO_FLASH_MMU_TABLE[i] == INVALID_ENTRY_VAL) {
314 DPORT_STALL_OTHER_CPU_END();
316 return count - 1; //don't sure mmap src_addr,if src_addr not align 64K bytes,max need one more pages
322 /* 256-bit (up to 16MB of 64KB pages) bitset of all flash pages
323 that have been written to since last cache flush.
325 Before mmaping a page, need to flush caches if that page has been
328 Note: It's possible to do some additional performance tweaks to
329 this algorithm, as we actually only need to flush caches if a page
330 was first mmapped, then written to, then is about to be mmaped a
331 second time. This is a fair bit more complex though, so unless
332 there's an access pattern that this would significantly boost then
333 it's probably not worth it.
335 static uint32_t written_pages[256/32];
337 static bool update_written_pages(size_t start_addr, size_t length, bool mark);
339 void IRAM_ATTR spi_flash_mark_modified_region(size_t start_addr, size_t length)
341 update_written_pages(start_addr, length, true);
344 static IRAM_ATTR bool spi_flash_ensure_unmodified_region(size_t start_addr, size_t length)
346 return update_written_pages(start_addr, length, false);
349 /* generic implementation for the previous two functions */
350 static inline IRAM_ATTR bool update_written_pages(size_t start_addr, size_t length, bool mark)
352 /* align start_addr & length to full MMU pages */
353 uint32_t page_start_addr = start_addr & ~(SPI_FLASH_MMU_PAGE_SIZE-1);
354 length += (start_addr - page_start_addr);
355 length = (length + SPI_FLASH_MMU_PAGE_SIZE - 1) & ~(SPI_FLASH_MMU_PAGE_SIZE-1);
356 for (uint32_t addr = page_start_addr; addr < page_start_addr + length; addr += SPI_FLASH_MMU_PAGE_SIZE) {
357 int page = addr / SPI_FLASH_MMU_PAGE_SIZE;
359 return false; /* invalid address */
363 uint32_t bit = 1 << (page % 32);
366 written_pages[idx] |= bit;
367 } else if (written_pages[idx] & bit) {
368 /* it is tempting to write a version of this that only
369 flushes each CPU's cache as needed. However this is
370 tricky because mmaped memory can be used on un-pinned
371 cores, or the pointer passed between CPUs.
373 #if CONFIG_SPIRAM_SUPPORT
374 esp_spiram_writeback_cache();
377 #ifndef CONFIG_FREERTOS_UNICORE
380 bzero(written_pages, sizeof(written_pages));
388 uint32_t spi_flash_cache2phys(const void *cached)
390 intptr_t c = (intptr_t)cached;
392 if (c >= VADDR1_START_ADDR && c < VADDR1_FIRST_USABLE_ADDR) {
393 /* IRAM address, doesn't map to flash */
394 return SPI_FLASH_CACHE2PHYS_FAIL;
396 else if (c < VADDR1_FIRST_USABLE_ADDR) {
397 /* expect cache is in DROM */
398 cache_page = (c - VADDR0_START_ADDR) / SPI_FLASH_MMU_PAGE_SIZE;
400 /* expect cache is in IROM */
401 cache_page = (c - VADDR1_START_ADDR) / SPI_FLASH_MMU_PAGE_SIZE + 64;
404 if (cache_page >= 256) {
405 /* cached address was not in IROM or DROM */
406 return SPI_FLASH_CACHE2PHYS_FAIL;
408 DPORT_STALL_OTHER_CPU_START();
409 uint32_t phys_page = DPORT_PRO_FLASH_MMU_TABLE[cache_page];
410 DPORT_STALL_OTHER_CPU_END();
411 if (phys_page == INVALID_ENTRY_VAL) {
412 /* page is not mapped */
413 return SPI_FLASH_CACHE2PHYS_FAIL;
415 uint32_t phys_offs = phys_page * SPI_FLASH_MMU_PAGE_SIZE;
416 return phys_offs | (c & (SPI_FLASH_MMU_PAGE_SIZE-1));
420 const void *spi_flash_phys2cache(uint32_t phys_offs, spi_flash_mmap_memory_t memory)
422 uint32_t phys_page = phys_offs / SPI_FLASH_MMU_PAGE_SIZE;
423 int start, end, page_delta;
426 if (memory == SPI_FLASH_MMAP_DATA) {
429 base = VADDR0_START_ADDR;
432 start = PRO_IRAM0_FIRST_USABLE_PAGE;
434 base = VADDR1_START_ADDR;
438 DPORT_STALL_OTHER_CPU_START();
439 for (int i = start; i < end; i++) {
440 if (DPORT_PRO_FLASH_MMU_TABLE[i] == phys_page) {
442 intptr_t cache_page = base + (SPI_FLASH_MMU_PAGE_SIZE * i);
443 DPORT_STALL_OTHER_CPU_END();
444 return (const void *) (cache_page | (phys_offs & (SPI_FLASH_MMU_PAGE_SIZE-1)));
447 DPORT_STALL_OTHER_CPU_END();