builder: improvements needed for esp32 XIP to correctly flash large programs

builder/esp.go

@@ -65,15 +65,6 @@ func makeESPFirmwareImage(infile, outfile, format string) error {
 	// Sort the segments by address. This is what esptool does too.
 	sort.SliceStable(segments, func(i, j int) bool { return segments[i].addr < segments[j].addr })
 
-	// Calculate checksum over the segment data. This is used in the image
-	// footer.
-	checksum := uint8(0xef)
-	for _, segment := range segments {
-		for _, b := range segment.data {
-			checksum ^= b
-		}
-	}
-
 	// Write first to an in-memory buffer, primarily so that we can easily
 	// calculate a hash over the entire image.
 	// An added benefit is that we don't need to check for errors all the time.
@@ -88,6 +79,86 @@ func makeESPFirmwareImage(infile, outfile, format string) error {
 		chip = format[:len(format)-len("-img")]
 	}
 
+	// For ESP32 (original): separate RAM segments (loadable by ROM bootloader)
+	// from flash-mapped segments (DROM/IROM, require MMU setup by startup code).
+	// The ROM bootloader on ESP32 does NOT handle flash-mapped segments —
+	// it tries to memcpy to the virtual address, which crashes.
+	var flashSegments []*espImageSegment
+	if chip == "esp32" {
+		var ramSegments []*espImageSegment
+		for _, seg := range segments {
+			if (seg.addr >= 0x3F400000 && seg.addr < 0x3F800000) || // DROM
+				(seg.addr >= 0x400D0000 && seg.addr < 0x40400000) { // IROM
+				flashSegments = append(flashSegments, seg)
+			} else {
+				ramSegments = append(ramSegments, seg)
+			}
+		}
+		segments = ramSegments
+	}
+
+	// ESP32 flash XIP: compute where the DROM segment will be placed in flash
+	// (page-aligned, right after the RAM segments) and patch the
+	// _drom_flash_addr variable so the startup code can program the cache MMU.
+	// This must happen before the checksum/hash are computed so the patched
+	// value is covered by both.
+	const esp32FlashBase = 0x1000 // esptool flashes the image at 0x1000
+	// The ESP32 flash cache MMU supports configurable page sizes down to 256 B. 64 KiB is the reset/default size.
+	// If the startup code ever changes the MMU page size, this constant must change too.
+	const esp32PageSize = 0x10000 // 64KB MMU pages
+	var esp32DromFlashAddr uint32
+	if chip == "esp32" && len(flashSegments) > 0 {
+		// Compute the size of the RAM portion of the image (everything the ROM
+		// bootloader loads, up to and including the appended SHA256 hash).
+		ramImageSize := 0
+		if makeImage {
+			ramImageSize += 4096
+		}
+		ramImageSize += 24 // image header (8) + trailer fields (16)
+		for _, seg := range segments {
+			ramImageSize += 8 + len(seg.data) // segment header + data (4-aligned)
+		}
+		ramImageSize += 16 - ramImageSize%16 // footer padding + checksum byte
+		ramImageSize += 32                   // appended SHA256 hash
+
+		// DROM flash address must be 64KB page-aligned.
+		esp32DromFlashAddr = uint32(esp32FlashBase+ramImageSize+esp32PageSize-1) &^ (esp32PageSize - 1)
+
+		// Patch _drom_flash_addr in whichever RAM segment contains it.
+		syms, _ := inf.Symbols()
+		var dromSymAddr uint64
+		for _, s := range syms {
+			if s.Name == "_drom_flash_addr" {
+				dromSymAddr = s.Value
+				break
+			}
+		}
+		if dromSymAddr == 0 {
+			return fmt.Errorf("ESP32: _drom_flash_addr symbol not found")
+		}
+		patched := false
+		for _, seg := range segments {
+			if dromSymAddr >= uint64(seg.addr) && dromSymAddr+4 <= uint64(seg.addr)+uint64(len(seg.data)) {
+				off := int(dromSymAddr - uint64(seg.addr))
+				binary.LittleEndian.PutUint32(seg.data[off:], esp32DromFlashAddr)
+				patched = true
+				break
+			}
+		}
+		if !patched {
+			return fmt.Errorf("ESP32: _drom_flash_addr (0x%x) not in any RAM segment", dromSymAddr)
+		}
+	}
+
+	// Calculate checksum over the segment data. This is used in the image
+	// footer.
+	checksum := uint8(0xef)
+	for _, segment := range segments {
+		for _, b := range segment.data {
+			checksum ^= b
+		}
+	}
+
 	if makeImage {
 		// The bootloader starts at 0x1000, or 4096.
 		// TinyGo doesn't use a separate bootloader and runs the entire
@@ -191,6 +262,58 @@ func makeESPFirmwareImage(infile, outfile, format string) error {
 		outf.Write(hash[:])
 	}
 
+	// For ESP32: append flash-mapped segments (DROM/IROM) at page-aligned flash
+	// offsets after the RAM portion. The startup code maps them via the flash
+	// cache MMU (DROM at esp32DromFlashAddr, patched into _drom_flash_addr).
+	if len(flashSegments) > 0 {
+		const flashBase = esp32FlashBase
+		const pageSize = esp32PageSize
+		dromFlashAddr := esp32DromFlashAddr
+
+		// Separate DROM and IROM segments.
+		var dromSegs, iromSegs []*espImageSegment
+		for _, seg := range flashSegments {
+			if seg.addr >= 0x3F400000 && seg.addr < 0x3F800000 {
+				dromSegs = append(dromSegs, seg)
+			} else {
+				iromSegs = append(iromSegs, seg)
+			}
+		}
+
+		// Write DROM segments at the computed page-aligned flash offset.
+		dromSize := 0
+		if len(dromSegs) > 0 {
+			targetImageOffset := int(dromFlashAddr - flashBase)
+			if outf.Len() > targetImageOffset {
+				return fmt.Errorf("ESP32: RAM segments too large (%d bytes), overlap DROM at flash 0x%x", outf.Len(), dromFlashAddr)
+			}
+			outf.Write(make([]byte, targetImageOffset-outf.Len()))
+			for _, seg := range dromSegs {
+				outf.Write(seg.data)
+				dromSize += len(seg.data)
+			}
+		}
+
+		// Write IROM segments immediately after DROM, at the next page boundary.
+		// IROM flash addr = dromFlashAddr + ceil(dromSize/pageSize)*pageSize
+		// (must match the computation in the startup assembly).
+		if len(iromSegs) > 0 {
+			dromPages := (dromSize + pageSize - 1) / pageSize
+			if dromPages == 0 {
+				dromPages = 1
+			}
+			iromFlashAddr := dromFlashAddr + uint32(dromPages)*pageSize
+			targetImageOffset := int(iromFlashAddr - flashBase)
+			if outf.Len() > targetImageOffset {
+				return fmt.Errorf("ESP32: DROM too large, overlaps IROM at flash 0x%x", iromFlashAddr)
+			}
+			outf.Write(make([]byte, targetImageOffset-outf.Len()))
+			for _, seg := range iromSegs {
+				outf.Write(seg.data)
+			}
+		}
+	}
+
 	// QEMU (or more precisely, qemu-system-xtensa from Espressif) expects the
 	// image to be a certain size.
 	if makeImage {

go.mod

@@ -15,22 +15,21 @@ require (
 	github.com/mgechev/revive v1.3.9
 	github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3
 	github.com/tetratelabs/wazero v1.9.0
-	go.bug.st/serial v1.6.4
+	go.bug.st/serial v1.8.0
 	go.bytecodealliance.org v0.6.2
 	go.bytecodealliance.org/cm v0.2.2
 	golang.org/x/net v0.56.0
-	golang.org/x/sys v0.46.0
+	golang.org/x/sys v0.47.0
 	golang.org/x/tools v0.47.0
 	gopkg.in/yaml.v2 v2.4.0
-	tinygo.org/x/espflasher v0.6.1
+	tinygo.org/x/espflasher v0.7.1
 	tinygo.org/x/go-llvm v0.0.0-20260707200325-ddd595b68360
 )
 
 require (
 	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/chavacava/garif v0.1.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
-	github.com/creack/goselect v0.1.2 // indirect
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/fatih/color v1.17.0 // indirect
 	github.com/fatih/structtag v1.2.0 // indirect

go.sum

@@ -8,8 +8,6 @@ github.com/chavacava/garif v0.1.0 h1:2JHa3hbYf5D9dsgseMKAmc/MZ109otzgNFk5s87H9Pc
 github.com/chavacava/garif v0.1.0/go.mod h1:XMyYCkEL58DF0oyW4qDjjnPWONs2HBqYKI+UIPD+Gww=
 github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4=
 github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec=
-github.com/creack/goselect v0.1.2 h1:2DNy14+JPjRBgPzAd1thbQp4BSIihxcBf0IXhQXDRa0=
-github.com/creack/goselect v0.1.2/go.mod h1:a/NhLweNvqIYMuxcMOuWY516Cimucms3DglDzQP3hKY=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -89,8 +87,8 @@ github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc=
 github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/urfave/cli/v3 v3.0.0-beta1 h1:6DTaaUarcM0wX7qj5Hcvs+5Dm3dyUTBbEwIWAjcw9Zg=
 github.com/urfave/cli/v3 v3.0.0-beta1/go.mod h1:FnIeEMYu+ko8zP1F9Ypr3xkZMIDqW3DR92yUtY39q1Y=
-go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A=
-go.bug.st/serial v1.6.4/go.mod h1:nofMJxTeNVny/m6+KaafC6vJGj3miwQZ6vW4BZUGJPI=
+go.bug.st/serial v1.8.0 h1:ZtnmN8aYXtPlTghwSvDWPHKBHL9TM6oFDa+KpSn4SQE=
+go.bug.st/serial v1.8.0/go.mod h1:d0MmS16Qt9b1m06yoYRNUXhRRTJV5Qg2S5EKqQtnayQ=
 go.bytecodealliance.org v0.6.2 h1:Jy4u5DVmSkXgsnwojBhJ+AD/YsJsR3VzVnxF0xRCqTQ=
 go.bytecodealliance.org v0.6.2/go.mod h1:gqjTJm0y9NSksG4py/lSjIQ/SNuIlOQ+hCIEPQwtJgA=
 go.bytecodealliance.org/cm v0.2.2 h1:M9iHS6qs884mbQbIjtLX1OifgyPG9DuMs2iwz8G4WQA=
@@ -107,8 +105,8 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
 golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
 golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
@@ -120,7 +118,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-tinygo.org/x/espflasher v0.6.1 h1:9jfyAP9jGjxF63FQUY2Bml9TFb5fCZYxVgbgR2IjUGs=
-tinygo.org/x/espflasher v0.6.1/go.mod h1:tr5u08HoE67WD5zxJesCiiVF/R1b6Akz3yXwh5zah8U=
+tinygo.org/x/espflasher v0.7.1 h1:oJ+tt58QltP+c5+iHGRdCNNnVaXZ9yYXyWK/F747BVY=
+tinygo.org/x/espflasher v0.7.1/go.mod h1:a3l/4jOSMf9vhaPtEmGbCwtrC5oiZiNKx1i8jUkyyRI=
 tinygo.org/x/go-llvm v0.0.0-20260707200325-ddd595b68360 h1:BOGCjKGwPn2q54p/CNyfGzUDmqyi8YBSm3KlDheaTEU=
 tinygo.org/x/go-llvm v0.0.0-20260707200325-ddd595b68360/go.mod h1:GFbusT2VTA4I+l4j80b17KFK+6whv69Wtny5U+T8RR0=

main.go

@@ -1122,6 +1122,15 @@ const (
 	jtagReset    = "jtag"
 )
 
+var progressFunc = func(current, total int) {
+	pct := float64(current) / float64(total) * 100
+	bar := int(pct / 2)
+	fmt.Printf("\r[%-50s] %6.1f%%", strings.Repeat("#", bar)+strings.Repeat(".", 50-bar), pct)
+	if current >= total {
+		fmt.Println()
+	}
+}
+
 func flashBinUsingEsp32(port, resetMode, tmppath string, options *compileopts.Options) error {
 	opts := espflasher.DefaultOptions()
 	opts.Compress = true
@@ -1152,21 +1161,12 @@ func flashBinUsingEsp32(port, resetMode, tmppath string, options *compileopts.Op
 		return err
 	}
 
-	if err := flasher.EraseFlash(); err != nil {
+	if err := flasher.EraseFlash(progressFunc); err != nil {
 		return fmt.Errorf("erase failed: %v", err)
 	}
 
-	progress := func(current, total int) {
-		pct := float64(current) / float64(total) * 100
-		bar := int(pct / 2)
-		fmt.Printf("\r[%-50s] %6.1f%%", strings.Repeat("#", bar)+strings.Repeat(".", 50-bar), pct)
-		if current >= total {
-			fmt.Println()
-		}
-	}
-
 	// Flash with progress reporting
-	err = flasher.FlashImage(data, offset, progress)
+	err = flasher.FlashImage(data, offset, progressFunc)
 	if err != nil {
 		return err
 	}