diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a9e5d..362440f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,14 +14,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.7.6] (unreleased) ### Changed -- Update dependancy version of atomvm_packbeam to 0.7.5 -- Update dependancy version of atomvm_packbeam to 0.8.0 +- Update dependency version of atomvm_packbeam to 0.7.5 +- Update dependency version of atomvm_packbeam to 0.8.0 +- `offset` for the `esp32_flash` provider is no longer required, and use is deprecated in favor of +`app_partition`, which only needs to be supplied for custom images, or flashing to partitions other +than `main.avm`. +- `port` for the `esp32_flash` provider is now auto discovered if omitted, rather than hard coded +to `/dev/ttyUSB0` which did not work for all chips and host platforms. ### Added - Added dialyzer task to simplify running dialyzer on AtomVM applications. - Added support for rp2350 devices to allow for default detection of the device mount path. -- Added configuration paramenter for setting the path to picotool for the pico_flash task. +- Added configuration parameter for setting the path to picotool for the pico_flash task. - Added escriptize task to build escriptize-like bundled binaries with AtomVM. +- Added `app_partition` parameter to `esp32_flash` task. This is only needed to be provided for +custom partition tables that do not use `main.avm` for the beam application partition name, or to +flash to a custom alternate partition. ### Changed - The `uf2create` task now creates `universal` format uf2 files by default, suitable for both @@ -29,6 +37,16 @@ rp2040 or rp2350 devices. - The `pico_flash` task now checks that a device is an RP2 platform before resetting to `BOOTSEL` mode, preventing interference with other MCUs that may be attached to the host system. - The `pico_flash` task now aborts on all errors rather than trying to continue after a failure. +- The `offset` used by the `esp32_flash` task is now read from the partition table of the device. +When this parameter is provided it will be used to verify the offset of the application partition on +flash matches the expected value. +- The `esp32_flash` task now uses auto discovery for the `port` by default. +- Stacktraces are not shown by default if the `esp32_flash` fails, instead a descriptive error +message is displayed. To view the stacktrace use diagnostic mode. + +### Fixed +- The `esp32_flash` task aborts when an error occurs, rather than attempt to continue after a step +has failed. ## [0.7.5] (2025.05.27) diff --git a/Makefile b/Makefile index 2047f39..d03e6ff 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ doc: rebar3 as doc ex_doc etest: - cd test && ./run.sh + cd test && TEST=1 ./run.sh clean: rm -rf _build diff --git a/README.md b/README.md index 2f90aa1..bef5cb2 100644 --- a/README.md +++ b/README.md @@ -261,24 +261,36 @@ Running this AVM file will boot the `myapp` application automatically, without h You may use the `esp32_flash` task to flash the generated AtomVM packbeam application to the flash storage on an ESP32 device connected over a serial connection. - shell$ rebar3 help atomvm esp32_flash - - Use this plugin to flash an AtomVM packbeam file to an ESP32 device. - - Usage: rebar3 atomvm esp32_flash [-e ] [-c ] [-p ] - [-b ] [-o ] - - -e, --esptool Path to esptool.py - -c, --chip ESP chip (default auto) - -p, --port Device port (default /dev/ttyUSB0) - -b, --baud Baud rate (default 115200) - -o, --offset Offset (default 0x210000) +```shell +shell$ rebar3 help atomvm esp32_flash + +Use this plugin to flash an AtomVM packbeam file to an ESP32 device. + +Usage: rebar3 atomvm esp32_flash [-e ] [-c ] [-p ] + [-b ] [-o ] + [-a ] + + -e, --esptool Path to esptool.py + -c, --chip ESP chip (default auto) + -p, --port Device port (default auto discovery) + -b, --baud Baud rate (default 115200) + -o, --offset Offset (default read from device) *deprecated, use + app_partition. When given, a warning will be issued + if the partition name does not match the expected + name, as long as the offset aligns with a valid + application partition. + -a, --app_partition Application partition name (default main.avm) +``` The `esp32_flash` task will use the `esptool.py` command to flash the ESP32 device. This tool is available via the IDF SDK, or directly via github. The `esptool.py` command is also available via many package managers (e.g., MacOS Homebrew). -By default, the `esp32_flash` task will assume the `esptool.py` command is available on the user's executable path. Alternatively, you may specify the full path to the `esptool.py` command via the `-e` (or `--esptool`) option +The `esp32_flash` task will assume the `esptool.py` command is available on the user's executable +path. Alternatively, you may specify the full path to the `esptool.py` command via the `-e` (or +`--esptool`) option. -By default, the `esp32_flash` task will write to port `/dev/ttyUSB0` at a baud rate of `115200`. You may control the port and baud settings for connecting to your ESP device via the `-port` and `-baud` options to the `esp32_flash` task, e.g., +By default, the `esp32_flash` task uses port auto discovery to find any attached ESP32 device and a +baud rate of `115200`. You may control the port and baud settings for connecting to your ESP32 +device via the `--port` and `--baud` options to the `esp32_flash` task, e.g., shell$ rebar3 atomvm esp32_flash --port /dev/tty.SLAB_USBtoUART --baud 921600 ... @@ -307,7 +319,8 @@ The following table enumerates the properties that may be defined in your projec | `chip` | `string()` | ESP32 chip type | | `port` | `string()` | Device port on which the ESP32 can be located | | `baud` | `integer()` | Device BAUD rate | -| `offset` | `string()` | Offset into which to write AtomVM application | +| `offset` | `string()` | Optionally verify offset on flash matches expected value. Original behavior deprecated: use `app_partition` for custom images or to target a different partition | +| `app_partition` | `string()` | Name of application partition to write AtomVM application for custom partition tables | Example: @@ -320,9 +333,24 @@ Alternatively, the following environment variables may be used to control the ab * `ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_PORT` * `ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_BAUD` * `ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_OFFSET` +* `ATOMVM_REBAR3_PLUGIN_ESP32_APP_PARTITION` Any setting specified on the command line take precedence over settings in `rebar.config`, which in turn take precedence over environment variable settings, which in turn take precedence over the default values specified above. +```note +The behavior of the `offset` configuration option has changed, the correct offset for standard +AtomVM builds are determined by the partition table flashed to the device. Elixir supported builds +are recognized and the correct offset will be used. When using a custom partition table it is +necessary to supply the `app_partition` name. If an offset is given it will be compared to the +address of the discovered `app_partition` and an warning will be given if they do not match, but as +long as the offset aligns to the beginning of a `data` partition with a valid subtype it will be +used. Currently any subtype not reserved for another purpose (such as `fat`, `nvs`, or ESP-IDF +defined subtypes) are considered valid. AtomVM release images use subtype `phy` for the `main.avm` +BEAM application partition, but it is not necessary to supply the name when using a release image +or one of the standard partition tables, or custom tables that use the name `main.avm` for the app +partition. +``` + The `esp32_flash` task depends on the `packbeam` task, so the packbeam file will get automatically built if any changes have been made to its dependencies. ### The `stm32_flash` task diff --git a/UPDATING.md b/UPDATING.md index 8fdd24a..9574ea0 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -6,6 +6,20 @@ # `atomvm_rebar3_plugin` Update Instructions +## (unreleased) + +- Pico 2 (RP2350) devices are recognized and now work with default parameters. Specifying device +path and uf2 flavor for these chipsets is no longer necessary. +- The `esp32_flash` task now reads the application `offset` from the partition table on the device. +If you are using a custom partition table that does not use `main.avm` for the application partition +name you should supply the name used with the `app_partition` parameter. An `offset` may optionally +be supplied to assure the offset of the application partition matches the expected offset, this may +be helpful to assure that specific applications are only flashed to devices with a custom build of +AtomVM. +- The `esp32_flash` task now auto-discovers any attached ESP32 device if the `port` is omitted. +Previously this was hard-coded to `/dev/ttyUSB0`, which did not match many devices or work at all on +MacOS or other platforms. + ## 0.6.* -> 0.7.* - The `atomvm_rebar3_plugin` tasks have been moved into the `atomvm` namespace (from the [`rebar3`](https://rebar3.org) `default` namespace). The "legacy" tasks in the `default` namespace are deprecated, and users will be issued a warning when used. Be sure to use the `atomvm` namespace in any future usage of this plugin, as the deprecated tasks may be removed without warning. E.g., `rebar3 atomvm packbeam ...` diff --git a/src/atomvm_esp32_flash_provider.erl b/src/atomvm_esp32_flash_provider.erl index 75bb63a..a3c1018 100644 --- a/src/atomvm_esp32_flash_provider.erl +++ b/src/atomvm_esp32_flash_provider.erl @@ -30,17 +30,22 @@ -define(OPTS, [ {esptool, $e, "esptool", string, "Path to esptool.py"}, {chip, $c, "chip", string, "ESP chip (default auto)"}, - {port, $p, "port", string, "Device port (default /dev/ttyUSB0)"}, + {port, $p, "port", string, "Device port (default auto discovery)"}, {baud, $b, "baud", integer, "Baud rate (default 115200)"}, - {offset, $o, "offset", string, "Offset (default 0x210000)"} + {offset, $o, "offset", string, + "Offset (default read from device) *deprecated, use app_partition. " + "When given, a warning will be issued if the partition name does not match the " + "expected name, as long as the offset aligns with a valid application partition."}, + {app_partition, $a, "app_partition", string, "Application partition name (default main.avm)"} ]). -define(DEFAULT_OPTS, #{ - esptool => "esptool.py", + esptool => os:find_executable("esptool.py"), chip => "auto", - port => "/dev/ttyUSB0", - baud => 115200, - offset => "0x210000" + port => "auto", + baud => "115200", + offset => "auto", + app_partition => "main.avm" }). %% @@ -81,15 +86,24 @@ do(State) -> maps:get(chip, Opts), maps:get(port, Opts), maps:get(baud, Opts), - maps:get(offset, Opts) + maybe_integer_from_string(maps:get(offset, Opts)), + maps:get(app_partition, Opts), + State ), {ok, State} catch - C:E:S -> + C:rebar_abort:S -> rebar_api:error( - "An error occurred in the ~p task. Class=~p Error=~p Stacktrace=~p~n", [ - ?PROVIDER, C, E, S - ] + "A fatal error occurred in the ~p task.", + [?PROVIDER] + ), + rebar_api:debug("Class=~p, Error=~p~nSTACKTRACE:~n~p~n", [C, rebar_abort, S]), + {error, rebar_abort}; + C:E:S -> + rebar_api:debug("Class=~p, Error=~p~nSTACKTRACE:~n~p~n", [C, E, S]), + rebar_api:abort( + "An unhandled error occurred in the ~p task. Error=~p", + [?PROVIDER, E] ), {error, E} end. @@ -105,6 +119,7 @@ format_error(Reason) -> %% @private get_opts(State) -> {ParsedArgs, _} = rebar_state:command_parsed_args(State), + rebar_api:debug("ParsedArgs = ~w", [ParsedArgs]), RebarOpts = atomvm_rebar3_plugin:get_atomvm_rebar_provider_config(State, ?PROVIDER), ParsedOpts = atomvm_rebar3_plugin:proplist_to_map(ParsedArgs), maps:merge( @@ -127,37 +142,63 @@ env_opts() -> "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_PORT", maps:get(port, ?DEFAULT_OPTS) ), - baud => maybe_convert_string( + baud => maybe_integer_from_string( os:getenv( "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_BAUD", maps:get(baud, ?DEFAULT_OPTS) ) ), - offset => os:getenv( - "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_OFFSET", - maps:get(offset, ?DEFAULT_OPTS) + offset => maybe_integer_from_string( + os:getenv( + "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_OFFSET", + maps:get(offset, ?DEFAULT_OPTS) + ) + ), + app_partition => os:getenv( + "ATOMVM_REBAR3_PLUGIN_ESP32_APP_PARTITION", + maps:get(app_partition, ?DEFAULT_OPTS) ) }. %% @private -maybe_convert_string(S) when is_list(S) -> - list_to_integer(S); -maybe_convert_string(I) -> - I. +-spec maybe_integer_from_string(S :: list() | integer() | auto) -> Value :: integer() | auto. +maybe_integer_from_string(auto) -> + auto; +maybe_integer_from_string("auto") -> + auto; +maybe_integer_from_string(I) when is_integer(I) -> + I; +maybe_integer_from_string(S) when is_list(S) -> + S0 = string:lowercase(string:trim(S)), + case lists:prefix("0x", S0) of + true -> + IntStr = lists:sublist(S0, 3, length(S0)), + list_to_integer(IntStr, 16); + false -> + list_to_integer(S0) + end. %% @private -do_flash(ProjectApps, EspTool, Chip, Port, Baud, Offset) -> - [ProjectAppAVM | _] = [get_avm_file(ProjectApp) || ProjectApp <- ProjectApps], - Portparam = - case Port of - "auto" -> ""; - _ -> ["--port ", Port] +do_flash(ProjectApps, EspTool, Chip, Port, Baud, Offset, Partition, State) -> + {TempFile, EspToolLog} = get_part_tempfiles(), + Address = + case Offset of + auto -> + read_flash_offset(EspTool, Port, Partition, TempFile, EspToolLog); + Val -> + case read_flash_offset(EspTool, Port, Partition, TempFile, EspToolLog) of + Val -> + Val; + _Other -> + {ok, Offset1} = validate_offset(Val, Partition, TempFile), + Offset1 + end end, - Cmd = lists:join(" ", [ - EspTool, + ProjectAppAVM = get_avm_file(hd(ProjectApps)), + + StdArgs = [ "--chip", Chip, - Portparam, "--baud", integer_to_list(Baud), "--before", @@ -165,18 +206,45 @@ do_flash(ProjectApps, EspTool, Chip, Port, Baud, Offset) -> "--after", "hard_reset", "write_flash", - "-u", "--flash_mode", "keep", "--flash_freq", "keep", "--flash_size", "detect", - Offset, + "0x" ++ integer_to_list(Address, 16), ProjectAppAVM - ]), - rebar_api:info("~s~n", [Cmd]), - rebar_api:console("~s", [os:cmd(Cmd)]), + ], + Args = + case Port of + "auto" -> + StdArgs; + _ -> + ["--port", Port | StdArgs] + end, + + AVMApp = filename:basename(ProjectAppAVM), + rebar_api:info("Flashing ~s to device.", [AVMApp]), + + %% The following log output is parsed by the tests and should not be changed or removed. + case os:getenv("TEST") of + false -> + ok; + _ -> + rebar_api:info("~s ~s", [EspTool, lists:flatten(lists:join(" ", Args))]) + end, + + try atomvm_rebar3_plugin:external_command(EspTool, Args, EspToolLog) of + ok -> ok + catch + error:Reason -> + decode_abort_reason(Reason); + C:E:S -> + decode_abort_reason({C, E, S}) + end, + rebar_api:info("Success!", []), + + file:delete(TempFile), ok. %% @private @@ -185,3 +253,120 @@ get_avm_file(App) -> Name = binary_to_list(rebar_app_info:name(App)), DirName = filename:dirname(OutDir), filename:join(DirName, Name ++ ".avm"). + +%% @private +read_flash_offset(EspTool, Port, PartName, TempFile, EspToolLog) -> + rebar_api:info("Reading application partition offset from device...", []), + try esp_part_dump:read_app_offset(EspTool, Port, PartName, TempFile, EspToolLog) of + Offset -> + Offset + catch + error:Reason -> + decode_abort_reason(Reason); + C:E:S -> + decode_abort_reason({C, E, S}) + end. + +validate_offset(Offset, Partition, TempFile) -> + rebar_api:debug("Checking that offset aligns with partition from table on device...", []), + try esp_part_dump:partition_at_offset(Offset, TempFile) of + {Found, _Type} -> + rebar_api:warn( + "The discovered partition ~s at offset 0x~.16B will be used, not the expected partition named ~s.", + [Found, Offset, Partition] + ), + {ok, Offset} + catch + error:Reason -> + decode_abort_reason(Reason); + C:E:S -> + decode_abort_reason({C, E, S}) + end. + +%% @private +get_part_tempfiles() -> + TempFile = create_temp_file("partitions.bin.XXXXX"), + %% we are generating the file name as a side-effect, the partition table dump file should not actually exist yet. + file:delete(TempFile), + + rebar_api:debug("Using partition dump file ~s.", [TempFile]), + + Logfile = create_temp_file("esptool.log.XXXXX"), + rebar_api:debug("Using esptool.py logfile ~s.", [Logfile]), + + {TempFile, Logfile}. + +create_temp_file(Format) -> + Cmd = os:find_executable("mktemp"), + Args = ["-q", Format], + {ok, TempFile} = atomvm_rebar3_plugin:external_command(Cmd, Args, none), + TempFile. + +decode_abort_reason(Reason) -> + case Reason of + invalid_partition_table -> + rebar_api:abort("Invalid partition data!", []); + no_device_dump -> + rebar_api:abort("No partition table retrieved from device.", []); + {partition_not_found, Partition} -> + rebar_api:error("The partition ~s was not found in device partition table!", [Partition]), + rebar_api:abort( + "When using a custom partition table always specify the 'app_partition' NAME.", [] + ); + {invalid_subtype, {PartName, Type}} -> + rebar_api:abort("The partition ~s was found, but used invalid subtype ~w.", [ + PartName, Type + ]); + {invalid_partition_type, PartName} -> + rebar_api:abort( + "The partition ~s is not a data partition!~nOnly data partitions may be used for AtomVM applications", + [ + PartName + ] + ); + corrupt_partition_data -> + rebar_api:abort("Fatal error! Corrupt partition table data!", []); + no_data_partitions -> + rebar_api:abort("The partition table on the device contains no data partitions"); + {"esptool.py failure", Status} -> + rebar_api:abort( + "An error was encountered connecting to device with esptool.py (error ~p)\n" + "Is minicom or serial monitor attached?", + [Status] + ); + {invalid_partition, {Offset, Name, Reason}} -> + rebar_api:abort( + "The configured offset 0x~.16B contains partition ~s, which cannot be used for reason: ~p.~n", + [Offset, Name, Reason] + ); + %% we also want to match on esptool without .py extension for some distributions of the tool, and for .sh + %% extension for the test suite. + {[$e, $s, $p, $t, $o, $o, $l | _], {exit_status, 2}, LogFile} -> + rebar_api:abort( + "Could not establish communication with ESP32 device! Is serial monitor attached?~nLogfile: ~s", + [LogFile] + ); + {Name, {exit_status, Status}, LogFile} -> + rebar_api:abort("External command ~s exited with error ~w~nLogfile saved: ~s", [ + Name, Status, LogFile + ]); + {timeout, Name, min_5} -> + rebar_api:abort("External command ~s failed (5 minute timeout exceeded)", [Name]); + {enoent, CmdName} -> + rebar_api:abort("external command ~s not found", [CmdName]); + {eacces, Cmd} -> + rebar_api:abort("The file at path ~s is not executable, or user lacks permission", [Cmd]); + {Cmd, Args, C, E, Trace} -> + rebar_api:abort( + "Unexpected error using external command ~s (Args = ~s)~nClass = ~p, Error = ~p~nSTACKTRACE:~n~p~n", + [Cmd, Args, C, E, Trace] + ); + {C, E, S} -> + rebar_api:debug("Class=~p, Error=~p~nSTACKTRACE:~n~p~n", [C, E, S]), + rebar_api:abort( + "Unexpected error reading partition table from device. Error = ~p.", + [E] + ); + Reason -> + rebar_api:abort("Unexpected abort! Reason ~w", [Reason]) + end. diff --git a/src/atomvm_rebar3_plugin.erl b/src/atomvm_rebar3_plugin.erl index 1bcb95f..e3a4c40 100644 --- a/src/atomvm_rebar3_plugin.erl +++ b/src/atomvm_rebar3_plugin.erl @@ -22,7 +22,7 @@ -export([init/1]). %% internal API --export([proplist_to_map/1, get_atomvm_rebar_provider_config/2]). +-export([proplist_to_map/1, get_atomvm_rebar_provider_config/2, external_command/3]). -define(PROVIDERS, [ atomvm_bootstrap_provider, @@ -71,3 +71,70 @@ get_atomvm_rebar_provider_config(State, Provider) -> AtomVM -> proplist_to_map(proplists:get_value(Provider, AtomVM, [])) end. + +-spec external_command( + Cmd :: file:name_all(), Args :: [string()], LogFile :: file:name_all() | none +) -> Result :: ok | {ok, iodata()}. +external_command(Cmd, Args, LogFile) -> + CmdName = filename:basename(Cmd), + case LogFile of + none -> + ok; + _ -> + {{Y, M, D}, {H, I, S}} = calendar:local_time(), + LogBegin = io_lib:format("~s executed at ~p/~p/~p ~p:~p:~p~n", [ + CmdName, Y, M, D, H, I, S + ]), + file:write_file(LogFile, LogBegin, [append, {encoding, utf8}]) + end, + CmdPort = + try + open_port({spawn_executable, Cmd}, [ + {args, Args}, stderr_to_stdout, use_stdio, exit_status + ]) + catch + error:enoent -> + error({enoent, CmdName}); + error:eacces -> + error({eacces, Cmd}); + C:E:Trace -> + error({Cmd, Args, C, E, Trace}) + end, + case LogFile of + none -> + wait_for_exit(CmdPort, CmdName, []); + _ -> + log_command_output(CmdPort, CmdName, LogFile) + end. + +-spec wait_for_exit(CmdPort :: port(), CmdName :: string(), Output :: iodata()) -> iodata(). +wait_for_exit(CmdPort, CmdName, Output) -> + receive + {CmdPort, {data, Data}} -> + wait_for_exit(CmdPort, CmdName, [Data | Output]); + {CmdPort, {exit_status, 0}} -> + {ok, string:trim(lists:flatten(lists:reverse(Output)))}; + {CmdPort, {exit_status, Status}} -> + error({CmdName, {exit_status, Status}}) + after 300000 -> + port_close(CmdPort), + error({timeout, CmdName, min_5}) + end. + +-spec log_command_output(CmdPort :: port(), CmdName :: string(), LogFile :: file:name_all()) -> ok. +log_command_output(CmdPort, CmdName, LogFile) -> + receive + {CmdPort, {data, Data}} -> + file:write_file(LogFile, Data, [append, {encoding, utf8}]), + log_command_output(CmdPort, CmdName, LogFile); + {CmdPort, {exit_status, 0}} -> + file:delete(LogFile), + ok; + {CmdPort, {exit_status, Status}} -> + Msg = io_lib:format("Command ~s failed! Exit status ~p.~n", [CmdName, Status]), + file:write_file(LogFile, Msg, [append, {encoding, utf8}]), + error({CmdName, {exit_status, Status}, LogFile}) + after 300000 -> + port_close(CmdPort), + error({timeout, CmdName, min_5}) + end. diff --git a/src/esp_part_dump.erl b/src/esp_part_dump.erl new file mode 100644 index 0000000..5c1136e --- /dev/null +++ b/src/esp_part_dump.erl @@ -0,0 +1,267 @@ +%% +%% Copyright (c) 2025 Winford (UncleGrumpy) +%% All rights reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% +-module(esp_part_dump). + +-export([read_app_offset/5, read_app_offset/2, partition_at_offset/5, partition_at_offset/2]). + +-type fs_type() :: + phy + | avm_app + | undefined + | byte() + | protected + | {reserved, ota | nvs | coredump | nvs_keys | fat | efuse | spiffs | littlefs}. + +-spec read_app_offset( + Esptool :: string(), + Port :: string(), + PartName :: string(), + TempFile :: string(), + LogFile :: file:name_all() +) -> Offset :: non_neg_integer(). +read_app_offset(Esptool, Port, PartName, TempFile, LogFile) -> + Partitions = get_partition_data(Esptool, Port, TempFile, LogFile), + Offset = lookup_partition_by_name(PartName, Partitions), + Offset. + +-spec read_app_offset( + PartName :: string(), File :: string() +) -> Offset :: non_neg_integer(). + +read_app_offset(PartName, File) -> + Partitions = + case file:read_file(File) of + {ok, Data} -> + Data; + {error, enoent} -> + error(no_dump_file) + end, + Offset = lookup_partition_by_name(PartName, Partitions), + Offset. + +-spec lookup_partition_by_name(Name :: string(), Partitions :: binary()) -> + Offset :: non_neg_integer(). +lookup_partition_by_name(Name, Partitions) -> + PartitionsData = parse_data_partitions(Partitions), + case lists:keyfind(Name, 2, PartitionsData) of + false -> + error({partition_not_found, Name}); + {_Offset, Name, protected} -> + error({invalid_partition_type, Name}); + {_Offset, Name, {reserved, SubType}} -> + error({invalid_subtype, {Name, SubType}}); + {Offset, _Name, _SubType} -> + Offset + end. + +-spec partition_at_offset( + Esptool :: string(), + Port :: string(), + Offset :: non_neg_integer(), + TempFile :: file:name_all(), + LogFile :: file:name_all() +) -> + {Name :: string(), Type :: fs_type()}. +partition_at_offset(Esptool, Port, Offset, TempFile, LogFile) -> + Partitions = get_partition_data(Esptool, Port, TempFile, LogFile), + Partition = lookup_partition_by_offset(Offset, Partitions), + Partition. + +-spec partition_at_offset(Offset :: non_neg_integer(), TempFile :: file:name_all()) -> + {Name :: string(), Type :: fs_type()}. +partition_at_offset(Offset, File) -> + Partitions = + case file:read_file(File) of + {ok, Data} -> + Data; + {error, enoent} -> + error(no_dump_file) + end, + Partition = lookup_partition_by_offset(Offset, Partitions), + Partition. + +% @private +-spec lookup_partition_by_offset(Offset :: non_neg_integer(), Partitions :: binary()) -> + {Name :: string(), Type :: fs_type()}. +lookup_partition_by_offset(Offset, Partitions) -> + PartitionsData = parse_data_partitions(Partitions), + case lists:keyfind(Offset, 1, PartitionsData) of + false -> + error({invalid_partition, {Offset, "none", "no partition aligned to address"}}); + {Offset, Name, protected} -> + error({invalid_partition, {Offset, Name, "not a data partition"}}); + {Offset, Name, {reserved, Type}} -> + Reason = io_lib:format("partition reserved for ~s", [Type]), + error({invalid_partition, {Offset, Name, Reason}}); + {Offset, Name, Type} -> + {Name, Type} + end. + +% @private +-spec get_partition_data( + Esptool :: string(), Port :: string(), TempFile :: file:name_all(), LogFile :: file:name_all() +) -> + PartitionData :: binary(). +get_partition_data(Esptool, Port, TempFile, LogFile) -> + case file:read_file(TempFile) of + {ok, Data} -> + Data; + {error, _} -> + dump_device_partition(Esptool, Port, TempFile, LogFile) + end. + +%% @private +-spec dump_device_partition( + Esptool :: string(), Port :: string(), TempFile :: string(), Logfile :: file:name_all() +) -> + PartitionData :: binary(). +dump_device_partition(Esptool, Port, TempFile, Logfile) -> + BaseArgs = [ + "read_flash", + "0x8000", + "0xC00", + TempFile + ], + Args = + case Port of + "auto" -> + BaseArgs; + _ -> + ["--port", Port | BaseArgs] + end, + case os:getenv("TEST") of + false -> + ok; + _ -> + rebar_api:info("~s ~s", [Esptool, lists:flatten(lists:join(" ", Args))]) + end, + + ok = atomvm_rebar3_plugin:external_command(Esptool, Args, Logfile), + + Partition_data = + case file:read_file(TempFile) of + {ok, Data} -> + Data; + {error, enoent} -> + error(no_device_dump); + Error -> + error({file_read, Error}) + end, + Partition_data. + +%% @private +-spec parse_data_partitions(PartitionTable :: binary()) -> + [ + { + Offset :: non_neg_integer(), + Name :: string(), + Type :: fs_type() + } + ]. +parse_data_partitions(PartitionTable) -> + parse_data_partitions(PartitionTable, []). + +parse_data_partitions(<>, PartitionData) -> + case Partition of + %% The default sub-type for AtomVM applications is phy + <<16#aa50:16, 16#01, 16#01, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), phy} | PartitionData + ]); + %% custom sub-type 0xAA can be used to recognize AtomVM applications + <<16#aa50:16, 16#01, 16#aa, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), avm_app} | PartitionData + ]); + %% ESP-IDF sub-type "undefined" is allowed + <<16#aa50:16, 16#01, 16#06, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), undefined} | PartitionData + ]); + %% Other data partition subtypes should not be used (including fat, nvs, coredump, efuse, etc...) + <<16#aa50:16, 16#01, 16#00, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, ota}} + | PartitionData + ]); + <<16#aa50:16, 16#01, 16#02, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, nvs}} + | PartitionData + ]); + <<16#aa50:16, 16#01, 16#03, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, coredump}} + | PartitionData + ]); + <<16#aa50:16, 16#01, 16#04, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, nvs_keys}} + | PartitionData + ]); + <<16#aa50:16, 16#01, 16#05, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, efuse}} + | PartitionData + ]); + <<16#aa50:16, 16#01, 16#81, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, fat}} + | PartitionData + ]); + <<16#aa50:16, 16#01, 16#82, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, spiffs}} + | PartitionData + ]); + <<16#aa50:16, 16#01, 16#83, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), {reserved, littlefs}} + | PartitionData + ]); + %% Catchall to allow any other sub-types + <<16#aa50:16, 16#01, SubType:8, 0, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), SubType} | PartitionData + ]); + %% Non data partitions are off-limits. BEAM applications should only be flashed to data partitions. + <<16#aa50:16, _NonDatatype:3/binary, Address:4/binary, _:3/binary, PName:16/binary, 0:32>> -> + [Name | _End] = binary:split(PName, <<0>>), + parse_data_partitions(Partitions, [ + {binary:decode_unsigned(Address), binary_to_list(Name), protected} | PartitionData + ]); + %% End of table marker + <<16#ebeb:16, 16#ffffffffffffffffffffffffffff:112, _:16/binary>> -> + lists:reverse(PartitionData); + _ -> + error(corrupt_partition_data) + end. diff --git a/test/driver/scripts/README.md b/test/driver/scripts/README.md new file mode 100644 index 0000000..dc07c1f --- /dev/null +++ b/test/driver/scripts/README.md @@ -0,0 +1,41 @@ + +# esptool.sh and partition binary files + +The partition binary files in this directory are used by esptool.sh to simulate the real esptool.py +dumping the partition table from an AtomVM installed device. They were built using esp-idf and the +following partition.csv contents were used to generate the partition tables: + +## partition.bin +This partition table starts with the standard Erlang only partition table, but adds several extra +partitions used for tests to flash to an alternate partition and test failures for invalid partition +types. + +```csv +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +nvs, data, nvs, 0x9000, 0x6000, \ +phy_init, data, phy, 0xf000, 0x1000, \ +factory, app, factory, 0x10000, 0x1C0000, > these are standard partitions +boot.avm, data, phy, 0x1D0000, 0x40000, / +main.avm, data, phy, 0x210000, 0x100000, / +app1.avm, data, 0xAA, 0x310000, 0x070000, \ +bad1, data, fat, 0x380000, 0x010000, > extra test partitions +bad2, app, test, 0x390000, 0x010000 / +``` + +## partition_elixir.bin +This partition table is just the standard Elixir supported build partition table. + +```csv +# Name, Type, SubType, Offset, Size, Flags +# Note: if you change the phy_init or app partition offset, make sure to change the offset in Kconfig.projbuild +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 0x1C0000, +boot.avm, data, 0xAB, 0x1D0000, 0x80000, +main.avm, data, 0xAA, 0x250000, 0x100000 +``` diff --git a/test/driver/scripts/esptool.sh b/test/driver/scripts/esptool.sh new file mode 100755 index 0000000..29658e3 --- /dev/null +++ b/test/driver/scripts/esptool.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env sh +## +## Copyright (c) Winford (UncleGrumpy) +## All rights reserved. +## +# SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later + +binary=${ATOMVM_REBAR3_PLUGIN_PARTITION_DATA:=${ATOMVM_REBAR3_PLUGIN_ESP32_PARTITION_DUMP}} + +while [ "${#}" -gt 0 ]; do + case ${1} in + --port | -p ) shift + if [ "${1}" = "bad" ]; then + exit 2 + fi ;; + read_flash ) shift + if [ "${1}" = "0x8000" ] && { [ "${2}" = "0xC00" ] || [ "${2}" = "0xc00" ]; }; then + target=${3} + cp "${binary}" "${target}" + unset ATOMVM_REBAR3_PLUGIN_PARTITION_DATA + exit 0 + fi ;; + write_flash ) exit 0 ;; + esac + shift +done + +echo "${@}" + +exit 0 diff --git a/test/driver/scripts/partition.bin b/test/driver/scripts/partition.bin new file mode 100644 index 0000000..1f55e3a Binary files /dev/null and b/test/driver/scripts/partition.bin differ diff --git a/test/driver/scripts/partition.bin.license b/test/driver/scripts/partition.bin.license new file mode 100644 index 0000000..e23e315 --- /dev/null +++ b/test/driver/scripts/partition.bin.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: Winford (Uncle Grumpy) diff --git a/test/driver/scripts/partition_elixir.bin b/test/driver/scripts/partition_elixir.bin new file mode 100644 index 0000000..5888934 Binary files /dev/null and b/test/driver/scripts/partition_elixir.bin differ diff --git a/test/driver/scripts/partition_elixir.bin.license b/test/driver/scripts/partition_elixir.bin.license new file mode 100644 index 0000000..e23e315 --- /dev/null +++ b/test/driver/scripts/partition_elixir.bin.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: Apache-2.0 +SPDX-FileCopyrightText: Winford (Uncle Grumpy) diff --git a/test/driver/src/esp32_flash_tests.erl b/test/driver/src/esp32_flash_tests.erl index 1c08287..ec810ae 100644 --- a/test/driver/src/esp32_flash_tests.erl +++ b/test/driver/src/esp32_flash_tests.erl @@ -23,15 +23,17 @@ run(Opts) -> ok = test_flags(Opts), + ok = test_elixir_partition_table(Opts), ok = test_env_overrides(Opts), ok = test_rebar_overrides(Opts), + ok = test_warnings(Opts), + ok = test_errors(Opts), ok. %% @private test_flags(Opts) -> test_flags(Opts, [], [ {"--chip", "auto"}, - {"--port", "/dev/ttyUSB0"}, {"--baud", "115200"}, {"--offset", "0x210000"} ]), @@ -40,16 +42,28 @@ test_flags(Opts) -> Opts, [ {"-c", "esp32c3"}, - {"-p", "/dev/tty.usbserial-0001"} + {"-p", "/dev/tty.usbserial-0001"}, + {"-b", "921600"} ], [ {"--chip", "esp32c3"}, - {"--port", "tty.usbserial-0001"}, - {"--baud", "115200"}, + {"--port", "/dev/tty.usbserial-0001"}, + {"--baud", "921600"}, {"--offset", "0x210000"} ] ), + test_flags( + Opts, + [ + {"-a", "app1.avm"} + ], + [ + {"--chip", "auto"}, + {"--baud", "115200"}, + {"--offset", "0x310000"} + ] + ), ok. %% @private @@ -67,7 +81,23 @@ test_flags(Opts, Flags, FlagExpectList) -> end, FlagExpectList ), - ok = test:expect_contains("_build/default/lib/myapp.avm", Output), + ok = test:expect_contains("Flashing myapp.avm to device.", Output), + + test:tick(). + +test_elixir_partition_table(Opts) -> + AppsDir = maps:get(apps_dir, Opts), + AppDir = test:make_path([AppsDir, "myapp"]), + Offset = 16#250000, + + Cmd = create_esp32_flash_cmd(AppDir, [], [ + {"ATOMVM_REBAR3_PLUGIN_PARTITION_DATA", + os:getenv("ATOMVM_REBAR3_PLUGIN_ESP32_EX_PARTITION_DUMP")} + ]), + Output = test:execute_cmd(Cmd, Opts), + test:debug(Output, Opts), + + ok = test:expect_contains(io_lib:format("0x~.16B", [Offset]), Output), test:tick(). @@ -78,7 +108,7 @@ test_env_overrides(Opts) -> ), test_env_overrides(Opts, "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_CHIP", "esp32", "--chip"), test_env_overrides(Opts, "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_BAUD", "921600", "--baud"), - test_env_overrides(Opts, "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_OFFSET", "0x1000", ""), + test_env_overrides(Opts, "ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_OFFSET", "0x310000", ""), ok. %% @private @@ -123,6 +153,79 @@ test_rebar_overrides(Opts, Flags, EnvVar, Value, Flag, ExpectedValue) -> test:tick(). +%% @private +test_warnings(Opts) -> + test_expect( + Opts, + [ + {"--offset", "0x210000"}, + {"--app_partition", "app1.avm"} + ], + "The discovered partition main.avm at offset 0x210000 will be used, not the expected partition named app1.avm." + ), + ok. + +%% @private +test_errors(Opts) -> + test_expect( + Opts, + [ + {"-a", "fake"} + ], + "The partition fake was not found in device partition table!" + ), + + %% magic parameter to inform the script that emulates dumping the partition data + %% to return an error simulating execution when no device is attached, or communication + %% cannot be established - like when device is attached to serial monitor. + test_expect( + Opts, + [ + {"--port", "bad"} + ], + "Could not establish communication with ESP32 device! Is serial monitor attached?" + ), + + test_expect( + Opts, + [ + {"--app_partition", "app1.avm"}, + {"--offset", "0x210000"} + ], + "The discovered partition main.avm at offset 0x210000 will be used, not the expected partition named app1.avm." + ), + + test_expect( + Opts, + [ + {"-a", "bad1"} + ], + "The partition bad1 was found, but used invalid subtype fat." + ), + + test_expect( + Opts, + [ + {"-a", "bad2"} + ], + "The partition bad2 is not a data partition!" + ), + + ok. + +%% @private +test_expect(Opts, Flags, Expect) -> + AppsDir = maps:get(apps_dir, Opts), + AppDir = test:make_path([AppsDir, "myapp"]), + + Cmd = create_esp32_flash_cmd(AppDir, Flags, []), + Output = test:execute_cmd(Cmd, Opts), + test:debug(Output, Opts), + + ok = test:expect_contains(Expect, Output), + + test:tick(). + %% @private create_esp32_flash_cmd(AppDir, Opts, Env) -> - test:create_rebar3_cmd(AppDir, esp32_flash, [{"-e", "echo"} | Opts], Env). + test:create_rebar3_cmd(AppDir, esp32_flash, Opts, Env). diff --git a/test/driver/src/test.erl b/test/driver/src/test.erl index 22d8845..ac8b1cb 100644 --- a/test/driver/src/test.erl +++ b/test/driver/src/test.erl @@ -143,7 +143,7 @@ execute_cmd(Cmd, Opts) -> debug(Msg, Opts) -> case maps:get(debug, Opts, false) of true -> - io:format("~s~n", [Msg]); + io:format("~p~n", [Msg]); _ -> ok end. diff --git a/test/run.sh b/test/run.sh index 49a5b69..e880f2d 100755 --- a/test/run.sh +++ b/test/run.sh @@ -24,6 +24,14 @@ unset ATOMVM_REBAR3_PLUGIN_PICO_RESET_DEV unset ATOMVM_REBAR3_PLUGIN_UF2CREATE_START +export ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_ESPTOOL="${test_dir}/scripts/esptool.sh" +export ATOMVM_REBAR3_PLUGIN_ESP32_PARTITION_DUMP="${test_dir}/scripts/partition.bin" +export ATOMVM_REBAR3_PLUGIN_ESP32_EX_PARTITION_DUMP="${test_dir}/scripts/partition_elixir.bin" + cd "${test_dir}" rebar3 escriptize ./_build/default/bin/driver -r "$(pwd)" "$@" + +unset ATOMVM_REBAR3_PLUGIN_ESP32_PARTITION_DUMP +unset ATOMVM_REBAR3_PLUGIN_ESP32_EX_PARTITION_DUMP +unset ATOMVM_REBAR3_PLUGIN_ESP32_FLASH_ESPTOOL