1 // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
19 #include <sys/param.h> // For MIN/MAX(a, b)
21 #include <freertos/FreeRTOS.h>
22 #include <freertos/task.h>
23 #include <freertos/semphr.h>
24 #include <rom/spi_flash.h>
25 #include <rom/cache.h>
27 #include <soc/dport_reg.h>
28 #include "sdkconfig.h"
31 #include "esp_spi_flash.h"
34 #include "esp_flash_partitions.h"
35 #include "esp_ota_ops.h"
36 #include "cache_utils.h"
38 /* bytes erased by SPIEraseBlock() ROM function */
39 #define BLOCK_ERASE_SIZE 65536
41 /* Limit number of bytes written/read in a single SPI operation,
42 as these operations disable all higher priority tasks from running.
44 #define MAX_WRITE_CHUNK 8192
45 #define MAX_READ_CHUNK 16384
47 static const char *TAG __attribute__((unused)) = "spi_flash";
49 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
50 static spi_flash_counters_t s_flash_stats;
52 #define COUNTER_START() uint32_t ts_begin = xthal_get_ccount()
53 #define COUNTER_STOP(counter) \
55 s_flash_stats.counter.count++; \
56 s_flash_stats.counter.time += (xthal_get_ccount() - ts_begin) / (esp_clk_cpu_freq() / 1000000); \
59 #define COUNTER_ADD_BYTES(counter, size) \
61 s_flash_stats.counter.bytes += size; \
65 #define COUNTER_START()
66 #define COUNTER_STOP(counter)
67 #define COUNTER_ADD_BYTES(counter, size)
69 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS
71 static esp_err_t spi_flash_translate_rc(esp_rom_spiflash_result_t rc);
73 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_default_ops = {
74 .start = spi_flash_disable_interrupts_caches_and_other_cpu,
75 .end = spi_flash_enable_interrupts_caches_and_other_cpu,
76 .op_lock = spi_flash_op_lock,
77 .op_unlock = spi_flash_op_unlock
80 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_no_os_ops = {
81 .start = spi_flash_disable_interrupts_caches_and_other_cpu_no_os,
82 .end = spi_flash_enable_interrupts_caches_no_os,
87 static const spi_flash_guard_funcs_t *s_flash_guard_ops;
89 #ifdef CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ABORTS
90 #define UNSAFE_WRITE_ADDRESS abort()
92 #define UNSAFE_WRITE_ADDRESS return false
96 /* CHECK_WRITE_ADDRESS macro to fail writes which land in the
97 bootloader, partition table, or running application region.
99 #if CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ALLOWED
100 #define CHECK_WRITE_ADDRESS(ADDR, SIZE)
101 #else /* FAILS or ABORTS */
102 #define CHECK_WRITE_ADDRESS(ADDR, SIZE) do { \
103 if (!is_safe_write_address(ADDR, SIZE)) { \
104 return ESP_ERR_INVALID_ARG; \
107 #endif // CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ALLOWED
109 static __attribute__((unused)) bool is_safe_write_address(size_t addr, size_t size)
112 if (addr <= ESP_PARTITION_TABLE_OFFSET + ESP_PARTITION_TABLE_MAX_LEN) {
113 UNSAFE_WRITE_ADDRESS;
116 const esp_partition_t *p = esp_ota_get_running_partition();
117 if (addr >= p->address && addr < p->address + p->size) {
118 UNSAFE_WRITE_ADDRESS;
120 if (addr < p->address && addr + size > p->address) {
121 UNSAFE_WRITE_ADDRESS;
128 void spi_flash_init()
130 spi_flash_init_lock();
131 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
132 spi_flash_reset_counters();
136 void IRAM_ATTR spi_flash_guard_set(const spi_flash_guard_funcs_t *funcs)
138 s_flash_guard_ops = funcs;
141 const spi_flash_guard_funcs_t *IRAM_ATTR spi_flash_guard_get()
143 return s_flash_guard_ops;
146 size_t IRAM_ATTR spi_flash_get_chip_size()
148 return g_rom_flashchip.chip_size;
151 static inline void IRAM_ATTR spi_flash_guard_start()
153 if (s_flash_guard_ops && s_flash_guard_ops->start) {
154 s_flash_guard_ops->start();
158 static inline void IRAM_ATTR spi_flash_guard_end()
160 if (s_flash_guard_ops && s_flash_guard_ops->end) {
161 s_flash_guard_ops->end();
165 static inline void IRAM_ATTR spi_flash_guard_op_lock()
167 if (s_flash_guard_ops && s_flash_guard_ops->op_lock) {
168 s_flash_guard_ops->op_lock();
172 static inline void IRAM_ATTR spi_flash_guard_op_unlock()
174 if (s_flash_guard_ops && s_flash_guard_ops->op_unlock) {
175 s_flash_guard_ops->op_unlock();
179 static esp_rom_spiflash_result_t IRAM_ATTR spi_flash_unlock()
181 static bool unlocked = false;
183 spi_flash_guard_start();
184 esp_rom_spiflash_result_t rc = esp_rom_spiflash_unlock();
185 spi_flash_guard_end();
186 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
191 return ESP_ROM_SPIFLASH_RESULT_OK;
194 esp_err_t IRAM_ATTR spi_flash_erase_sector(size_t sec)
196 CHECK_WRITE_ADDRESS(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
197 return spi_flash_erase_range(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
200 esp_err_t IRAM_ATTR spi_flash_erase_range(uint32_t start_addr, uint32_t size)
202 CHECK_WRITE_ADDRESS(start_addr, size);
203 if (start_addr % SPI_FLASH_SEC_SIZE != 0) {
204 return ESP_ERR_INVALID_ARG;
206 if (size % SPI_FLASH_SEC_SIZE != 0) {
207 return ESP_ERR_INVALID_SIZE;
209 if (size + start_addr > spi_flash_get_chip_size()) {
210 return ESP_ERR_INVALID_SIZE;
212 size_t start = start_addr / SPI_FLASH_SEC_SIZE;
213 size_t end = start + size / SPI_FLASH_SEC_SIZE;
214 const size_t sectors_per_block = BLOCK_ERASE_SIZE / SPI_FLASH_SEC_SIZE;
216 esp_rom_spiflash_result_t rc;
217 rc = spi_flash_unlock();
218 if (rc == ESP_ROM_SPIFLASH_RESULT_OK) {
219 for (size_t sector = start; sector != end && rc == ESP_ROM_SPIFLASH_RESULT_OK; ) {
220 spi_flash_guard_start();
221 if (sector % sectors_per_block == 0 && end - sector >= sectors_per_block) {
222 rc = esp_rom_spiflash_erase_block(sector / sectors_per_block);
223 sector += sectors_per_block;
224 COUNTER_ADD_BYTES(erase, sectors_per_block * SPI_FLASH_SEC_SIZE);
226 rc = esp_rom_spiflash_erase_sector(sector);
228 COUNTER_ADD_BYTES(erase, SPI_FLASH_SEC_SIZE);
230 spi_flash_guard_end();
234 return spi_flash_translate_rc(rc);
237 /* Wrapper around esp_rom_spiflash_write() that verifies data as written if CONFIG_SPI_FLASH_VERIFY_WRITE is set.
239 If CONFIG_SPI_FLASH_VERIFY_WRITE is not set, this is esp_rom_spiflash_write().
241 static IRAM_ATTR esp_rom_spiflash_result_t spi_flash_write_inner(uint32_t target, const uint32_t *src_addr, int32_t len)
243 #ifndef CONFIG_SPI_FLASH_VERIFY_WRITE
244 return esp_rom_spiflash_write(target, src_addr, len);
245 #else // CONFIG_SPI_FLASH_VERIFY_WRITE
246 esp_rom_spiflash_result_t res = ESP_ROM_SPIFLASH_RESULT_OK;
247 assert(len % sizeof(uint32_t) == 0);
249 uint32_t before_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
250 uint32_t after_buf[ESP_ROM_SPIFLASH_BUFF_BYTE_READ_NUM / sizeof(uint32_t)];
251 int32_t remaining = len;
252 for(int i = 0; i < len; i += sizeof(before_buf)) {
253 int i_w = i / sizeof(uint32_t); // index in words (i is an index in bytes)
255 int32_t read_len = MIN(sizeof(before_buf), remaining);
257 // Read "before" contents from flash
258 res = esp_rom_spiflash_read(target + i, before_buf, read_len);
259 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
263 #ifdef CONFIG_SPI_FLASH_WARN_SETTING_ZERO_TO_ONE
264 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
265 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
267 uint32_t write = src_addr[i_w + r_w];
268 uint32_t before = before_buf[r_w];
269 if ((before & write) != write) {
270 spi_flash_guard_end();
271 ESP_LOGW(TAG, "Write at offset 0x%x requests 0x%08x but will write 0x%08x -> 0x%08x",
272 target + i + r, write, before, before & write);
273 spi_flash_guard_start();
278 res = esp_rom_spiflash_write(target + i, &src_addr[i_w], read_len);
279 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
283 res = esp_rom_spiflash_read(target + i, after_buf, read_len);
284 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
288 for (int r = 0; r < read_len; r += sizeof(uint32_t)) {
289 int r_w = r / sizeof(uint32_t); // index in words (r is index in bytes)
291 uint32_t expected = src_addr[i_w + r_w] & before_buf[r_w];
292 uint32_t actual = after_buf[r_w];
293 if (expected != actual) {
294 #ifdef CONFIG_SPI_FLASH_LOG_FAILED_WRITE
295 spi_flash_guard_end();
296 ESP_LOGE(TAG, "Bad write at offset 0x%x expected 0x%08x readback 0x%08x", target + i + r, expected, actual);
297 spi_flash_guard_start();
299 res = ESP_ROM_SPIFLASH_RESULT_ERR;
302 if (res != ESP_ROM_SPIFLASH_RESULT_OK) {
305 remaining -= read_len;
308 #endif // CONFIG_SPI_FLASH_VERIFY_WRITE
312 esp_err_t IRAM_ATTR spi_flash_write(size_t dst, const void *srcv, size_t size)
314 CHECK_WRITE_ADDRESS(dst, size);
315 // Out of bound writes are checked in ROM code, but we can give better
317 if (dst + size > g_rom_flashchip.chip_size) {
318 return ESP_ERR_INVALID_SIZE;
324 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
326 const uint8_t *srcc = (const uint8_t *) srcv;
328 * Large operations are split into (up to) 3 parts:
329 * - Left padding: 4 bytes up to the first 4-byte aligned destination offset.
331 * - Right padding: 4 bytes from the last 4-byte aligned offset covered.
333 size_t left_off = dst & ~3U;
334 size_t left_size = MIN(((dst + 3) & ~3U) - dst, size);
335 size_t mid_off = left_size;
336 size_t mid_size = (size - left_size) & ~3U;
337 size_t right_off = left_size + mid_size;
338 size_t right_size = size - mid_size - left_size;
340 rc = spi_flash_unlock();
341 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
345 uint32_t t = 0xffffffff;
346 memcpy(((uint8_t *) &t) + (dst - left_off), srcc, left_size);
347 spi_flash_guard_start();
348 rc = spi_flash_write_inner(left_off, &t, 4);
349 spi_flash_guard_end();
350 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
353 COUNTER_ADD_BYTES(write, 4);
356 /* If src buffer is 4-byte aligned as well and is not in a region that requires cache access to be enabled, we
357 * can write directly without buffering in RAM. */
359 bool direct_write = esp_ptr_internal(srcc)
360 && esp_ptr_byte_accessible(srcc)
361 && ((uintptr_t) srcc + mid_off) % 4 == 0;
363 bool direct_write = true;
365 while(mid_size > 0 && rc == ESP_ROM_SPIFLASH_RESULT_OK) {
366 uint32_t write_buf[8];
367 uint32_t write_size = MIN(mid_size, MAX_WRITE_CHUNK);
368 const uint8_t *write_src = srcc + mid_off;
370 write_size = MIN(write_size, sizeof(write_buf));
371 memcpy(write_buf, write_src, write_size);
372 write_src = (const uint8_t *)write_buf;
374 spi_flash_guard_start();
375 rc = spi_flash_write_inner(dst + mid_off, (const uint32_t *) write_src, write_size);
376 spi_flash_guard_end();
377 COUNTER_ADD_BYTES(write, write_size);
378 mid_size -= write_size;
379 mid_off += write_size;
381 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
386 if (right_size > 0) {
387 uint32_t t = 0xffffffff;
388 memcpy(&t, srcc + right_off, right_size);
389 spi_flash_guard_start();
390 rc = spi_flash_write_inner(dst + right_off, &t, 4);
391 spi_flash_guard_end();
392 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
395 COUNTER_ADD_BYTES(write, 4);
400 spi_flash_guard_op_lock();
401 spi_flash_mark_modified_region(dst, size);
402 spi_flash_guard_op_unlock();
404 return spi_flash_translate_rc(rc);
407 esp_err_t IRAM_ATTR spi_flash_write_encrypted(size_t dest_addr, const void *src, size_t size)
409 CHECK_WRITE_ADDRESS(dest_addr, size);
410 const uint8_t *ssrc = (const uint8_t *)src;
411 if ((dest_addr % 16) != 0) {
412 return ESP_ERR_INVALID_ARG;
414 if ((size % 16) != 0) {
415 return ESP_ERR_INVALID_SIZE;
419 esp_rom_spiflash_result_t rc;
420 rc = spi_flash_unlock();
421 if (rc == ESP_ROM_SPIFLASH_RESULT_OK) {
422 /* esp_rom_spiflash_write_encrypted encrypts data in RAM as it writes,
423 so copy to a temporary buffer - 32 bytes at a time.
425 Each call to esp_rom_spiflash_write_encrypted takes a 32 byte "row" of
426 data to encrypt, and each row is two 16 byte AES blocks
427 that share a key (as derived from flash address).
429 uint8_t encrypt_buf[32] __attribute__((aligned(4)));
431 for (size_t i = 0; i < size; i += row_size) {
432 uint32_t row_addr = dest_addr + i;
433 if (i == 0 && (row_addr % 32) != 0) {
434 /* writing to second block of a 32 byte row */
437 /* copy to second block in buffer */
438 memcpy(encrypt_buf + 16, ssrc + i, 16);
439 /* decrypt the first block from flash, will reencrypt to same bytes */
440 spi_flash_read_encrypted(row_addr, encrypt_buf, 16);
441 } else if (size - i == 16) {
442 /* 16 bytes left, is first block of a 32 byte row */
444 /* copy to first block in buffer */
445 memcpy(encrypt_buf, ssrc + i, 16);
446 /* decrypt the second block from flash, will reencrypt to same bytes */
447 spi_flash_read_encrypted(row_addr + 16, encrypt_buf + 16, 16);
449 /* Writing a full 32 byte row (2 blocks) */
451 memcpy(encrypt_buf, ssrc + i, 32);
454 spi_flash_guard_start();
455 rc = esp_rom_spiflash_write_encrypted(row_addr, (uint32_t *)encrypt_buf, 32);
456 spi_flash_guard_end();
457 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
461 bzero(encrypt_buf, sizeof(encrypt_buf));
463 COUNTER_ADD_BYTES(write, size);
466 spi_flash_guard_op_lock();
467 spi_flash_mark_modified_region(dest_addr, size);
468 spi_flash_guard_op_unlock();
470 return spi_flash_translate_rc(rc);
473 esp_err_t IRAM_ATTR spi_flash_read(size_t src, void *dstv, size_t size)
475 // Out of bound reads are checked in ROM code, but we can give better
477 if (src + size > g_rom_flashchip.chip_size) {
478 return ESP_ERR_INVALID_SIZE;
484 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
486 spi_flash_guard_start();
487 /* To simplify boundary checks below, we handle small reads separately. */
489 uint32_t t[6]; /* Enough for 16 bytes + 4 on either side for padding. */
490 uint32_t read_src = src & ~3U;
491 uint32_t left_off = src & 3U;
492 uint32_t read_size = (left_off + size + 3) & ~3U;
493 rc = esp_rom_spiflash_read(read_src, t, read_size);
494 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
497 COUNTER_ADD_BYTES(read, read_size);
498 memcpy(dstv, ((uint8_t *) t) + left_off, size);
501 uint8_t *dstc = (uint8_t *) dstv;
502 intptr_t dsti = (intptr_t) dstc;
504 * Large operations are split into (up to) 3 parts:
505 * - The middle part: from the first 4-aligned position in src to the first
506 * 4-aligned position in dst.
508 size_t src_mid_off = (src % 4 == 0 ? 0 : 4 - (src % 4));
509 size_t dst_mid_off = (dsti % 4 == 0 ? 0 : 4 - (dsti % 4));
510 size_t mid_size = (size - MAX(src_mid_off, dst_mid_off)) & ~3U;
512 * - Once the middle part is in place, src_mid_off bytes from the preceding
513 * 4-aligned source location are added on the left.
515 size_t pad_left_src = src & ~3U;
516 size_t pad_left_size = src_mid_off;
518 * - Finally, the right part is added: from the end of the middle part to
519 * the end. Depending on the alignment of source and destination, this may
520 * be a 4 or 8 byte read from pad_right_src.
522 size_t pad_right_src = (src + pad_left_size + mid_size) & ~3U;
523 size_t pad_right_off = (pad_right_src - src);
524 size_t pad_right_size = (size - pad_right_off);
527 bool direct_read = esp_ptr_internal(dstc)
528 && esp_ptr_byte_accessible(dstc)
529 && ((uintptr_t) dstc + dst_mid_off) % 4 == 0;
531 bool direct_read = true;
534 uint32_t mid_remaining = mid_size;
535 uint32_t mid_read = 0;
536 while (mid_remaining > 0) {
537 uint32_t read_size = MIN(mid_remaining, MAX_READ_CHUNK);
538 uint32_t read_buf[8];
539 uint8_t *read_dst_final = dstc + dst_mid_off + mid_read;
540 uint8_t *read_dst = read_dst_final;
542 read_size = MIN(read_size, sizeof(read_buf));
543 read_dst = (uint8_t *) read_buf;
545 rc = esp_rom_spiflash_read(src + src_mid_off + mid_read,
546 (uint32_t *) read_dst, read_size);
547 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
550 mid_remaining -= read_size;
551 mid_read += read_size;
553 spi_flash_guard_end();
554 memcpy(read_dst_final, read_buf, read_size);
555 spi_flash_guard_start();
556 } else if (mid_remaining > 0) {
557 /* Drop guard momentarily, allows other tasks to preempt */
558 spi_flash_guard_end();
559 spi_flash_guard_start();
562 COUNTER_ADD_BYTES(read, mid_size);
564 * If offsets in src and dst are different, perform an in-place shift
565 * to put destination data into its final position.
566 * Note that the shift can be left (src_mid_off < dst_mid_off) or right.
568 if (src_mid_off != dst_mid_off) {
570 spi_flash_guard_end();
572 memmove(dstc + src_mid_off, dstc + dst_mid_off, mid_size);
574 spi_flash_guard_start();
578 if (pad_left_size > 0) {
580 rc = esp_rom_spiflash_read(pad_left_src, &t, 4);
581 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
584 COUNTER_ADD_BYTES(read, 4);
586 spi_flash_guard_end();
588 memcpy(dstc, ((uint8_t *) &t) + (4 - pad_left_size), pad_left_size);
590 spi_flash_guard_start();
593 if (pad_right_size > 0) {
595 int32_t read_size = (pad_right_size <= 4 ? 4 : 8);
596 rc = esp_rom_spiflash_read(pad_right_src, t, read_size);
597 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
600 COUNTER_ADD_BYTES(read, read_size);
602 spi_flash_guard_end();
604 memcpy(dstc + pad_right_off, t, pad_right_size);
606 spi_flash_guard_start();
610 spi_flash_guard_end();
612 return spi_flash_translate_rc(rc);
615 esp_err_t IRAM_ATTR spi_flash_read_encrypted(size_t src, void *dstv, size_t size)
617 if (src + size > g_rom_flashchip.chip_size) {
618 return ESP_ERR_INVALID_SIZE;
626 spi_flash_mmap_handle_t map_handle;
627 size_t map_src = src & ~(SPI_FLASH_MMU_PAGE_SIZE - 1);
628 size_t map_size = size + (src - map_src);
630 err = spi_flash_mmap(map_src, map_size, SPI_FLASH_MMAP_DATA, (const void **)&map, &map_handle);
634 memcpy(dstv, map + (src - map_src), size);
635 spi_flash_munmap(map_handle);
640 static esp_err_t IRAM_ATTR spi_flash_translate_rc(esp_rom_spiflash_result_t rc)
643 case ESP_ROM_SPIFLASH_RESULT_OK:
645 case ESP_ROM_SPIFLASH_RESULT_TIMEOUT:
646 return ESP_ERR_FLASH_OP_TIMEOUT;
647 case ESP_ROM_SPIFLASH_RESULT_ERR:
649 return ESP_ERR_FLASH_OP_FAIL;
653 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
655 static inline void dump_counter(spi_flash_counter_t *counter, const char *name)
657 ESP_LOGI(TAG, "%s count=%8d time=%8dus bytes=%8d\n", name,
658 counter->count, counter->time, counter->bytes);
661 const spi_flash_counters_t *spi_flash_get_counters()
663 return &s_flash_stats;
666 void spi_flash_reset_counters()
668 memset(&s_flash_stats, 0, sizeof(s_flash_stats));
671 void spi_flash_dump_counters()
673 dump_counter(&s_flash_stats.read, "read ");
674 dump_counter(&s_flash_stats.write, "write");
675 dump_counter(&s_flash_stats.erase, "erase");
678 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS