Relocation of network buffers

[mrjimenez]

Hi,

I am working with an infineon xmc45_relax_kit in an application with network enabled, and I would like to have the network buffers relocated to DSRAM2 because I am running out of RAM, as can be seen in the linker output:

[288/288] Linking C executable zephyr/zephyr.elf
Memory region         Used Size  Region Size  %age Used
           FLASH:      259152 B       512 KB     49.43%
             RAM:       64020 B        64 KB     97.69%
          PSRAM1:          0 GB        64 KB      0.00%
          DSRAM2:          0 GB        32 KB      0.00%
        IDT_LIST:          0 GB        32 KB      0.00%

I added the following to prj.conf:

CONFIG_CODE_DATA_RELOCATION=y
CONFIG_CODE_DATA_RELOCATION_SRAM=y

And tried adding the following to CMakeLists.txt:

if (BOARD STREQUAL "xmc45_relax_kit")
	zephyr_code_relocate(FILES ${ZEPHYR_BASE}/subsys/net/ip/net_pkt.c LOCATION DSRAM2_DATA)
endif()

also tried:

	zephyr_code_relocate(LIBRARY subsys__net__ip LOCATION DSRAM2_DATA_BSS_NOINIT)

But I always get:

[283/288] Linking C executable zephyr/zephyr_pre0.elf
FAILED: zephyr/zephyr_pre0.elf zephyr/zephyr_pre0.map /home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr/zephyr_pre0.map 
: && ccache /home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/arm-zephyr-eabi-gcc  -gdwarf-4 zephyr/CMakeFiles/zephyr_pre0.dir/misc/empty_file.c.obj -o zephyr/zephyr_pre0.elf  zephyr/CMakeFiles/offsets.dir/./arch/arm/core/offsets/offsets.c.obj  -T  zephyr/linker_zephyr_pre0.cmd  -Wl,-Map,/home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr/zephyr_pre0.map  -Wl,--whole-archive  app/libapp.a  zephyr/libzephyr.a  zephyr/arch/common/libarch__common.a  zephyr/arch/arch/arm/core/libarch__arm__core.a  zephyr/arch/arch/arm/core/cortex_m/libarch__arm__core__cortex_m.a  zephyr/arch/arch/arm/core/mpu/libarch__arm__core__mpu.a  zephyr/lib/libc/picolibc/liblib__libc__picolibc.a  zephyr/lib/libc/common/liblib__libc__common.a  zephyr/lib/posix/options/liblib__posix__options.a  zephyr/lib/net_buf/liblib__net_buf.a  zephyr/lib/os/zvfs/liblib__os__zvfs.a  zephyr/subsys/modbus/libsubsys__modbus.a  zephyr/subsys/random/libsubsys__random.a  zephyr/subsys/net/libsubsys__net.a  zephyr/subsys/net/l2/virtual/libsubsys__net__l2__virtual.a  zephyr/subsys/net/l2/ethernet/libsubsys__net__l2__ethernet.a  zephyr/subsys/net/ip/libsubsys__net__ip.a  zephyr/subsys/net/lib/config/libsubsys__net__lib__config.a  zephyr/drivers/interrupt_controller/libdrivers__interrupt_controller.a  zephyr/drivers/console/libdrivers__console.a  zephyr/drivers/ethernet/libdrivers__ethernet.a  zephyr/drivers/mdio/libdrivers__mdio.a  zephyr/drivers/net/libdrivers__net.a  zephyr/drivers/pinctrl/libdrivers__pinctrl.a  zephyr/drivers/serial/libdrivers__serial.a  zephyr/drivers/timer/libdrivers__timer.a  modules/hal_infineon/libmodules_hal_infineon.a  -Wl,--no-whole-archive  zephyr/kernel/libkernel.a  -L/home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr  zephyr/libcode_relocation_source_lib.a  zephyr/arch/common/libisr_tables.a  -mcpu=cortex-m4  -mthumb  -mabi=aapcs  -mfp16-format=ieee  -mtp=soft  -fuse-ld=bfd  -Wl,--gc-sections  -Wl,--build-id=none  -Wl,--sort-common=descending  -Wl,--sort-section=alignment  -Wl,-u,_OffsetAbsSyms  -Wl,-u,_ConfigAbsSyms  -nostdlib  -static  -Wl,-X  -Wl,-N  -Wl,--orphan-handling=warn  -Wl,-no-pie  -specs=picolibc.specs  -DPICOLIBC_DOUBLE_PRINTF_SCANF -L"/home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/../lib/gcc/arm-zephyr-eabi/12.2.0/thumb/v7e-m/nofp" -lc -lgcc && cd /home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45/zephyr && /usr/bin/cmake -E true
/home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/../lib/gcc/arm-zephyr-eabi/12.2.0/../../../../arm-zephyr-eabi/bin/ld.bfd: zephyr/arch/arch/arm/core/mpu/libarch__arm__core__mpu.a(arm_core_mpu.c.obj):(.rodata.static_regions+0xc): undefined reference to `__ram_text_reloc_start'
/home/myself/zephyr-sdk-0.17.4/arm-zephyr-eabi/bin/../lib/gcc/arm-zephyr-eabi/12.2.0/../../../../arm-zephyr-eabi/bin/ld.bfd: zephyr/arch/arch/arm/core/mpu/libarch__arm__core__mpu.a(arm_core_mpu.c.obj):(.rodata.static_regions+0x10): undefined reference to `__ram_text_reloc_size'
collect2: error: ld returned 1 exit status
ninja: build stopped: subcommand failed.
FATAL ERROR: command exited with status 1: /usr/bin/cmake --build /home/myself/zephyrproject/workspace/apps/f1_network/build_xmc45

Any ideas on what I am missing? If I am doing it wrong, what would be the right way to do it?

Best regards,
Marcelo.

[mrjimenez]

Hi again,

Just fixed the issue by removing CONFIG_CODE_DATA_RELOCATION_SRAM from prj.conf. I surely misunderstood something here.

[288/288] Linking C executable zephyr/zephyr.elf
Memory region         Used Size  Region Size  %age Used
           FLASH:      255212 B       512 KB     48.68%
             RAM:       54780 B        64 KB     83.59%
          PSRAM1:          0 GB        64 KB      0.00%
          DSRAM2:        8628 B        32 KB     26.33%
        IDT_LIST:          0 GB        32 KB      0.00%

[mrjimenez]

Continuing on the topic, even though I can now use DSRAM2, network buffers are still being alocated in RAM.

These are from zephyr.map:

Memory Configuration

Name             Origin             Length             Attributes
FLASH            0x0000000008000000 0x0000000000080000 xr
RAM              0x0000000020000000 0x0000000000010000 xw
PSRAM1           0x0000000010000000 0x0000000000010000 rw
DSRAM2           0x0000000030000000 0x0000000000008000 rw
IDT_LIST         0x00000000ffff7fff 0x0000000000008000 xw
*default*        0x0000000000000000 0xffffffffffffffff
...

net_buf_pool_area
                0x0000000020000448       0x58 load address 0x000000000803e088
                0x0000000020000448                _net_buf_pool_list_start = .
 *(SORT_BY_NAME(SORT_BY_ALIGNMENT(._net_buf_pool.static.*)))
 ._net_buf_pool.static.rx_bufs_
                0x0000000020000448       0x2c zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
 ._net_buf_pool.static.tx_bufs_
                0x0000000020000474       0x2c zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
                0x00000000200004a0                _net_buf_pool_list_end = .

...

 .noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_tx_pkts
                0x000000002000c890      0x8b8 zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
                0x000000002000c890                _k_mem_slab_buf_tx_pkts
 .noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_rx_pkts
                0x000000002000d148     0x1c20 zephyr/subsys/net/ip/libsubsys__net__ip.a(net_pkt.c.obj)
                0x000000002000d148                _k_mem_slab_buf_rx_pkts

And I found this in linker.cmd:

 .dsram2_noinit_reloc (NOLOAD) :
        {
                . = ALIGN(4);
                KEEP(*eth_xmc4xxx.c.obj(.noinit."WEST_TOPDIR/zephyr/drivers/ethernet/eth_xmc4xxx.c".k_mem_slab_buf_tx_frame_slab))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".0))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".0))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".1))
                KEEP(*net_mgmt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_mgmt.c".1))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".0))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".1))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".2))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".3))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_rx_pkts))
                KEEP(*net_pkt.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_pkt.c".k_mem_slab_buf_tx_pkts))
                KEEP(*net_tc.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_tc.c".0))
                KEEP(*net_tc.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/net_tc.c".1))
                KEEP(*sockets_service.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/lib/sockets/sockets_service.c".0))
                KEEP(*tcp.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/tcp.c".0))
                KEEP(*tcp.c.obj(.noinit."WEST_TOPDIR/zephyr/subsys/net/ip/tcp.c".k_mem_slab_buf_tcp_conns_slab))
                . = ALIGN(4);
 } > DSRAM2 AT > DSRAM2
        __dsram2_noinit_reloc_end = .;
        __dsram2_noinit_reloc_start = ADDR(.dsram2_noinit_reloc);
        __dsram2_noinit_reloc_size = __dsram2_noinit_reloc_end - __dsram2_noinit_reloc_start;

Which suggests that this behavior is on purpose. My questions:

1. Where do these KEEP directives come from? I can see __noinit's and __net_buf_align's in the code, is there any relation?
2. Why do the network buffers have to stick to the RAM region? Is there a way to change this behavior?

[jukkar]

The net_pkt slabs are allocated by this macro

zephyr/include/zephyr/net/net_pkt.h

Line 1583 in 6c83b9b

#define NET_PKT_SLAB_DEFINE(name, count) \

and the net_buf pool is allocated by this macro

zephyr/include/zephyr/net/net_pkt.h

Line 1607 in 6c83b9b

#define NET_PKT_DATA_POOL_DEFINE(name, count) \

AFAIK the network packet/buffer data could be allocated also to some other place than the regular ram.