1 // Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
7 // http://www.apache.org/licenses/LICENSE-2.0
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
19 #include <sys/param.h> // For MIN/MAX(a, b)
21 #include <freertos/FreeRTOS.h>
22 #include <freertos/task.h>
23 #include <freertos/semphr.h>
24 #include <rom/spi_flash.h>
25 #include <rom/cache.h>
27 #include <soc/dport_reg.h>
28 #include "sdkconfig.h"
31 #include "esp_spi_flash.h"
34 #include "esp_flash_partitions.h"
35 #include "esp_ota_ops.h"
36 #include "cache_utils.h"
38 /* bytes erased by SPIEraseBlock() ROM function */
39 #define BLOCK_ERASE_SIZE 65536
41 /* Limit number of bytes written/read in a single SPI operation,
42 as these operations disable all higher priority tasks from running.
44 #define MAX_WRITE_CHUNK 8192
45 #define MAX_READ_CHUNK 16384
47 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
48 static const char *TAG = "spi_flash";
49 static spi_flash_counters_t s_flash_stats;
51 #define COUNTER_START() uint32_t ts_begin = xthal_get_ccount()
52 #define COUNTER_STOP(counter) \
54 s_flash_stats.counter.count++; \
55 s_flash_stats.counter.time += (xthal_get_ccount() - ts_begin) / (esp_clk_cpu_freq() / 1000000); \
58 #define COUNTER_ADD_BYTES(counter, size) \
60 s_flash_stats.counter.bytes += size; \
64 #define COUNTER_START()
65 #define COUNTER_STOP(counter)
66 #define COUNTER_ADD_BYTES(counter, size)
68 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS
70 static esp_err_t spi_flash_translate_rc(esp_rom_spiflash_result_t rc);
72 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_default_ops = {
73 .start = spi_flash_disable_interrupts_caches_and_other_cpu,
74 .end = spi_flash_enable_interrupts_caches_and_other_cpu,
75 .op_lock = spi_flash_op_lock,
76 .op_unlock = spi_flash_op_unlock
79 const DRAM_ATTR spi_flash_guard_funcs_t g_flash_guard_no_os_ops = {
80 .start = spi_flash_disable_interrupts_caches_and_other_cpu_no_os,
81 .end = spi_flash_enable_interrupts_caches_no_os,
86 static const spi_flash_guard_funcs_t *s_flash_guard_ops;
88 #ifdef CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ABORTS
89 #define UNSAFE_WRITE_ADDRESS abort()
91 #define UNSAFE_WRITE_ADDRESS return false
95 /* CHECK_WRITE_ADDRESS macro to fail writes which land in the
96 bootloader, partition table, or running application region.
98 #if CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ALLOWED
99 #define CHECK_WRITE_ADDRESS(ADDR, SIZE)
100 #else /* FAILS or ABORTS */
101 #define CHECK_WRITE_ADDRESS(ADDR, SIZE) do { \
102 if (!is_safe_write_address(ADDR, SIZE)) { \
103 return ESP_ERR_INVALID_ARG; \
106 #endif // CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ALLOWED
108 static __attribute__((unused)) bool is_safe_write_address(size_t addr, size_t size)
111 if (addr <= ESP_PARTITION_TABLE_OFFSET + ESP_PARTITION_TABLE_MAX_LEN) {
112 UNSAFE_WRITE_ADDRESS;
115 const esp_partition_t *p = esp_ota_get_running_partition();
116 if (addr >= p->address && addr < p->address + p->size) {
117 UNSAFE_WRITE_ADDRESS;
119 if (addr < p->address && addr + size > p->address) {
120 UNSAFE_WRITE_ADDRESS;
127 void spi_flash_init()
129 spi_flash_init_lock();
130 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
131 spi_flash_reset_counters();
135 void IRAM_ATTR spi_flash_guard_set(const spi_flash_guard_funcs_t *funcs)
137 s_flash_guard_ops = funcs;
140 size_t IRAM_ATTR spi_flash_get_chip_size()
142 return g_rom_flashchip.chip_size;
145 static inline void IRAM_ATTR spi_flash_guard_start()
147 if (s_flash_guard_ops && s_flash_guard_ops->start) {
148 s_flash_guard_ops->start();
152 static inline void IRAM_ATTR spi_flash_guard_end()
154 if (s_flash_guard_ops && s_flash_guard_ops->end) {
155 s_flash_guard_ops->end();
159 static inline void IRAM_ATTR spi_flash_guard_op_lock()
161 if (s_flash_guard_ops && s_flash_guard_ops->op_lock) {
162 s_flash_guard_ops->op_lock();
166 static inline void IRAM_ATTR spi_flash_guard_op_unlock()
168 if (s_flash_guard_ops && s_flash_guard_ops->op_unlock) {
169 s_flash_guard_ops->op_unlock();
173 static esp_rom_spiflash_result_t IRAM_ATTR spi_flash_unlock()
175 static bool unlocked = false;
177 spi_flash_guard_start();
178 esp_rom_spiflash_result_t rc = esp_rom_spiflash_unlock();
179 spi_flash_guard_end();
180 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
185 return ESP_ROM_SPIFLASH_RESULT_OK;
188 esp_err_t IRAM_ATTR spi_flash_erase_sector(size_t sec)
190 CHECK_WRITE_ADDRESS(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
191 return spi_flash_erase_range(sec * SPI_FLASH_SEC_SIZE, SPI_FLASH_SEC_SIZE);
194 esp_err_t IRAM_ATTR spi_flash_erase_range(uint32_t start_addr, uint32_t size)
196 CHECK_WRITE_ADDRESS(start_addr, size);
197 if (start_addr % SPI_FLASH_SEC_SIZE != 0) {
198 return ESP_ERR_INVALID_ARG;
200 if (size % SPI_FLASH_SEC_SIZE != 0) {
201 return ESP_ERR_INVALID_SIZE;
203 if (size + start_addr > spi_flash_get_chip_size()) {
204 return ESP_ERR_INVALID_SIZE;
206 size_t start = start_addr / SPI_FLASH_SEC_SIZE;
207 size_t end = start + size / SPI_FLASH_SEC_SIZE;
208 const size_t sectors_per_block = BLOCK_ERASE_SIZE / SPI_FLASH_SEC_SIZE;
210 esp_rom_spiflash_result_t rc;
211 rc = spi_flash_unlock();
212 if (rc == ESP_ROM_SPIFLASH_RESULT_OK) {
213 for (size_t sector = start; sector != end && rc == ESP_ROM_SPIFLASH_RESULT_OK; ) {
214 spi_flash_guard_start();
215 if (sector % sectors_per_block == 0 && end - sector > sectors_per_block) {
216 rc = esp_rom_spiflash_erase_block(sector / sectors_per_block);
217 sector += sectors_per_block;
218 COUNTER_ADD_BYTES(erase, sectors_per_block * SPI_FLASH_SEC_SIZE);
220 rc = esp_rom_spiflash_erase_sector(sector);
222 COUNTER_ADD_BYTES(erase, SPI_FLASH_SEC_SIZE);
224 spi_flash_guard_end();
228 return spi_flash_translate_rc(rc);
231 esp_err_t IRAM_ATTR spi_flash_write(size_t dst, const void *srcv, size_t size)
233 CHECK_WRITE_ADDRESS(dst, size);
234 // Out of bound writes are checked in ROM code, but we can give better
236 if (dst + size > g_rom_flashchip.chip_size) {
237 return ESP_ERR_INVALID_SIZE;
243 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
245 const uint8_t *srcc = (const uint8_t *) srcv;
247 * Large operations are split into (up to) 3 parts:
248 * - Left padding: 4 bytes up to the first 4-byte aligned destination offset.
250 * - Right padding: 4 bytes from the last 4-byte aligned offset covered.
252 size_t left_off = dst & ~3U;
253 size_t left_size = MIN(((dst + 3) & ~3U) - dst, size);
254 size_t mid_off = left_size;
255 size_t mid_size = (size - left_size) & ~3U;
256 size_t right_off = left_size + mid_size;
257 size_t right_size = size - mid_size - left_size;
259 rc = spi_flash_unlock();
260 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
264 uint32_t t = 0xffffffff;
265 memcpy(((uint8_t *) &t) + (dst - left_off), srcc, left_size);
266 spi_flash_guard_start();
267 rc = esp_rom_spiflash_write(left_off, &t, 4);
268 spi_flash_guard_end();
269 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
272 COUNTER_ADD_BYTES(write, 4);
275 /* If src buffer is 4-byte aligned as well and is not in a region that requires cache access to be enabled, we
276 * can write directly without buffering in RAM. */
278 bool direct_write = esp_ptr_internal(srcc)
279 && esp_ptr_byte_accessible(srcc)
280 && ((uintptr_t) srcc + mid_off) % 4 == 0;
282 bool direct_write = true;
284 while(mid_size > 0 && rc == ESP_ROM_SPIFLASH_RESULT_OK) {
285 uint32_t write_buf[8];
286 uint32_t write_size = MIN(mid_size, MAX_WRITE_CHUNK);
287 const uint8_t *write_src = srcc + mid_off;
289 write_size = MIN(write_size, sizeof(write_buf));
290 memcpy(write_buf, write_src, write_size);
291 write_src = (const uint8_t *)write_buf;
293 spi_flash_guard_start();
294 rc = esp_rom_spiflash_write(dst + mid_off, (const uint32_t *) write_src, write_size);
295 spi_flash_guard_end();
296 COUNTER_ADD_BYTES(write, write_size);
297 mid_size -= write_size;
298 mid_off += write_size;
300 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
305 if (right_size > 0) {
306 uint32_t t = 0xffffffff;
307 memcpy(&t, srcc + right_off, right_size);
308 spi_flash_guard_start();
309 rc = esp_rom_spiflash_write(dst + right_off, &t, 4);
310 spi_flash_guard_end();
311 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
314 COUNTER_ADD_BYTES(write, 4);
319 spi_flash_guard_op_lock();
320 spi_flash_mark_modified_region(dst, size);
321 spi_flash_guard_op_unlock();
323 return spi_flash_translate_rc(rc);
326 esp_err_t IRAM_ATTR spi_flash_write_encrypted(size_t dest_addr, const void *src, size_t size)
328 CHECK_WRITE_ADDRESS(dest_addr, size);
329 const uint8_t *ssrc = (const uint8_t *)src;
330 if ((dest_addr % 16) != 0) {
331 return ESP_ERR_INVALID_ARG;
333 if ((size % 16) != 0) {
334 return ESP_ERR_INVALID_SIZE;
338 esp_rom_spiflash_result_t rc;
339 rc = spi_flash_unlock();
340 if (rc == ESP_ROM_SPIFLASH_RESULT_OK) {
341 /* esp_rom_spiflash_write_encrypted encrypts data in RAM as it writes,
342 so copy to a temporary buffer - 32 bytes at a time.
344 Each call to esp_rom_spiflash_write_encrypted takes a 32 byte "row" of
345 data to encrypt, and each row is two 16 byte AES blocks
346 that share a key (as derived from flash address).
348 uint8_t encrypt_buf[32] __attribute__((aligned(4)));
350 for (size_t i = 0; i < size; i += row_size) {
351 uint32_t row_addr = dest_addr + i;
352 if (i == 0 && (row_addr % 32) != 0) {
353 /* writing to second block of a 32 byte row */
356 /* copy to second block in buffer */
357 memcpy(encrypt_buf + 16, ssrc + i, 16);
358 /* decrypt the first block from flash, will reencrypt to same bytes */
359 spi_flash_read_encrypted(row_addr, encrypt_buf, 16);
360 } else if (size - i == 16) {
361 /* 16 bytes left, is first block of a 32 byte row */
363 /* copy to first block in buffer */
364 memcpy(encrypt_buf, ssrc + i, 16);
365 /* decrypt the second block from flash, will reencrypt to same bytes */
366 spi_flash_read_encrypted(row_addr + 16, encrypt_buf + 16, 16);
368 /* Writing a full 32 byte row (2 blocks) */
370 memcpy(encrypt_buf, ssrc + i, 32);
373 spi_flash_guard_start();
374 rc = esp_rom_spiflash_write_encrypted(row_addr, (uint32_t *)encrypt_buf, 32);
375 spi_flash_guard_end();
376 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
380 bzero(encrypt_buf, sizeof(encrypt_buf));
382 COUNTER_ADD_BYTES(write, size);
385 spi_flash_guard_op_lock();
386 spi_flash_mark_modified_region(dest_addr, size);
387 spi_flash_guard_op_unlock();
389 return spi_flash_translate_rc(rc);
392 esp_err_t IRAM_ATTR spi_flash_read(size_t src, void *dstv, size_t size)
394 // Out of bound reads are checked in ROM code, but we can give better
396 if (src + size > g_rom_flashchip.chip_size) {
397 return ESP_ERR_INVALID_SIZE;
403 esp_rom_spiflash_result_t rc = ESP_ROM_SPIFLASH_RESULT_OK;
405 spi_flash_guard_start();
406 /* To simplify boundary checks below, we handle small reads separately. */
408 uint32_t t[6]; /* Enough for 16 bytes + 4 on either side for padding. */
409 uint32_t read_src = src & ~3U;
410 uint32_t left_off = src & 3U;
411 uint32_t read_size = (left_off + size + 3) & ~3U;
412 rc = esp_rom_spiflash_read(read_src, t, read_size);
413 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
416 COUNTER_ADD_BYTES(read, read_size);
417 memcpy(dstv, ((uint8_t *) t) + left_off, size);
420 uint8_t *dstc = (uint8_t *) dstv;
421 intptr_t dsti = (intptr_t) dstc;
423 * Large operations are split into (up to) 3 parts:
424 * - The middle part: from the first 4-aligned position in src to the first
425 * 4-aligned position in dst.
427 size_t src_mid_off = (src % 4 == 0 ? 0 : 4 - (src % 4));
428 size_t dst_mid_off = (dsti % 4 == 0 ? 0 : 4 - (dsti % 4));
429 size_t mid_size = (size - MAX(src_mid_off, dst_mid_off)) & ~3U;
431 * - Once the middle part is in place, src_mid_off bytes from the preceding
432 * 4-aligned source location are added on the left.
434 size_t pad_left_src = src & ~3U;
435 size_t pad_left_size = src_mid_off;
437 * - Finally, the right part is added: from the end of the middle part to
438 * the end. Depending on the alignment of source and destination, this may
439 * be a 4 or 8 byte read from pad_right_src.
441 size_t pad_right_src = (src + pad_left_size + mid_size) & ~3U;
442 size_t pad_right_off = (pad_right_src - src);
443 size_t pad_right_size = (size - pad_right_off);
446 bool direct_read = esp_ptr_internal(dstc)
447 && esp_ptr_byte_accessible(dstc)
448 && ((uintptr_t) dstc + dst_mid_off) % 4 == 0;
450 bool direct_read = true;
453 uint32_t mid_remaining = mid_size;
454 uint32_t mid_read = 0;
455 while (mid_remaining > 0) {
456 uint32_t read_size = MIN(mid_remaining, MAX_READ_CHUNK);
457 uint32_t read_buf[8];
458 uint8_t *read_dst_final = dstc + dst_mid_off + mid_read;
459 uint8_t *read_dst = read_dst_final;
461 read_size = MIN(read_size, sizeof(read_buf));
462 read_dst = (uint8_t *) read_buf;
464 rc = esp_rom_spiflash_read(src + src_mid_off + mid_read,
465 (uint32_t *) read_dst, read_size);
466 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
469 mid_remaining -= read_size;
470 mid_read += read_size;
472 spi_flash_guard_end();
473 memcpy(read_dst_final, read_buf, read_size);
474 spi_flash_guard_start();
475 } else if (mid_remaining > 0) {
476 /* Drop guard momentarily, allows other tasks to preempt */
477 spi_flash_guard_end();
478 spi_flash_guard_start();
481 COUNTER_ADD_BYTES(read, mid_size);
483 * If offsets in src and dst are different, perform an in-place shift
484 * to put destination data into its final position.
485 * Note that the shift can be left (src_mid_off < dst_mid_off) or right.
487 if (src_mid_off != dst_mid_off) {
489 spi_flash_guard_end();
491 memmove(dstc + src_mid_off, dstc + dst_mid_off, mid_size);
493 spi_flash_guard_start();
497 if (pad_left_size > 0) {
499 rc = esp_rom_spiflash_read(pad_left_src, &t, 4);
500 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
503 COUNTER_ADD_BYTES(read, 4);
505 spi_flash_guard_end();
507 memcpy(dstc, ((uint8_t *) &t) + (4 - pad_left_size), pad_left_size);
509 spi_flash_guard_start();
512 if (pad_right_size > 0) {
514 int32_t read_size = (pad_right_size <= 4 ? 4 : 8);
515 rc = esp_rom_spiflash_read(pad_right_src, t, read_size);
516 if (rc != ESP_ROM_SPIFLASH_RESULT_OK) {
519 COUNTER_ADD_BYTES(read, read_size);
521 spi_flash_guard_end();
523 memcpy(dstc + pad_right_off, t, pad_right_size);
525 spi_flash_guard_start();
529 spi_flash_guard_end();
531 return spi_flash_translate_rc(rc);
534 esp_err_t IRAM_ATTR spi_flash_read_encrypted(size_t src, void *dstv, size_t size)
536 if (src + size > g_rom_flashchip.chip_size) {
537 return ESP_ERR_INVALID_SIZE;
545 spi_flash_mmap_handle_t map_handle;
546 size_t map_src = src & ~(SPI_FLASH_MMU_PAGE_SIZE - 1);
547 size_t map_size = size + (src - map_src);
549 err = spi_flash_mmap(map_src, map_size, SPI_FLASH_MMAP_DATA, (const void **)&map, &map_handle);
553 memcpy(dstv, map + (src - map_src), size);
554 spi_flash_munmap(map_handle);
559 static esp_err_t IRAM_ATTR spi_flash_translate_rc(esp_rom_spiflash_result_t rc)
562 case ESP_ROM_SPIFLASH_RESULT_OK:
564 case ESP_ROM_SPIFLASH_RESULT_TIMEOUT:
565 return ESP_ERR_FLASH_OP_TIMEOUT;
566 case ESP_ROM_SPIFLASH_RESULT_ERR:
568 return ESP_ERR_FLASH_OP_FAIL;
572 #if CONFIG_SPI_FLASH_ENABLE_COUNTERS
574 static inline void dump_counter(spi_flash_counter_t *counter, const char *name)
576 ESP_LOGI(TAG, "%s count=%8d time=%8dus bytes=%8d\n", name,
577 counter->count, counter->time, counter->bytes);
580 const spi_flash_counters_t *spi_flash_get_counters()
582 return &s_flash_stats;
585 void spi_flash_reset_counters()
587 memset(&s_flash_stats, 0, sizeof(s_flash_stats));
590 void spi_flash_dump_counters()
592 dump_counter(&s_flash_stats.read, "read ");
593 dump_counter(&s_flash_stats.write, "write");
594 dump_counter(&s_flash_stats.erase, "erase");
597 #endif //CONFIG_SPI_FLASH_ENABLE_COUNTERS