thread '<unnamed>' panicked at 'Send through channel failed: SendError { .. }', src/jsonrpc.rs:74:22

[g1eny0ung]

From https://cloud-native.slack.com/archives/C0193VAV272/p1704451321246929.

Here's the full log: https://gist.github.com/see-quick/6e34e37b559ab86169b251cd96210e69.

Any ideas? @STRRL @cwen0

[see-quick]

I don't know if this helps but I have also provide io chaos logs from chaosctl i.e.

chaosctl debug iochaos kafka-io-chaos-1 -n myproject
[Chaos]: kafka-io-chaos-1

[Pod]: my-cluster-kafka-1

	1. [Mount Information]

	overlay / overlay rw,context="system_u:object_r:container_file_t:s0:c9,c27",relatime,lowerdir=/var/lib/containers/storage/overlay/l/OZIDHLDAGIC4KTZHOFTEQESV2O:/var/lib/containers/storage/overlay/l/JG7YDVM6DFLQDLKGAJ6ALKIWDV:/var/lib/containers/storage/overlay/l/CVJUDOMREVACDMEQAHIAMNPGCM:/var/lib/containers/storage/overlay/l/L3MRMXXM6YCBX4AE5YT7FFI6TR:/var/lib/containers/storage/overlay/l/O6V4GIDP64A2HC25KGAY73K7JY:/var/lib/containers/storage/overlay/l/VRFOZN7LBI4GXBCTUR2ZV24HKH:/var/lib/containers/storage/overlay/l/52LBZZAAC3IP5TFBDSJ3SVRQM2:/var/lib/containers/storage/overlay/l/NGPGPJWVAQ4S2NLWWMVVZCUOVA:/var/lib/containers/storage/overlay/l/U3JACMCT7MRFVEIMM7XRLA3RSV:/var/lib/containers/storage/overlay/l/UWIDC3OWGWQ5ANK7GBSJFKBN4L:/var/lib/containers/storage/overlay/l/7QGITXUY7NKVZRWKUMBYB5MB3N:/var/lib/containers/storage/overlay/l/LZPQGVVH443WIMXVEUGIKACGXE:/var/lib/containers/storage/overlay/l/BXPCBLIS5M3O3ZRBX27XZGXFXJ:/var/lib/containers/storage/overlay/l/PYDWG5KI3LZKVDBFKNVPVFOR5I:/var/lib/containers/storage/overlay/l/FFGBP4F5EILLBUI53C7FCIJXK5:/var/lib/containers/storage/overlay/l/PDON37QKW5SIVIMVMKDGYH75K6:/var/lib/containers/storage/overlay/l/XWBNUDDBICIEOYZWFTZG6Z6UOI:/var/lib/containers/storage/overlay/l/CGVL5BRZXNHZ3L7UJBEDFQNCRF:/var/lib/containers/storage/overlay/l/XYVQTA3OUO6MBA25CQALVOQK5W,upperdir=/var/lib/containers/storage/overlay/ae38c4f416d8a7eacaae19e92a74ab4cd30d8034d3ab5a180098d605e12d1bb4/diff,workdir=/var/lib/containers/storage/overlay/ae38c4f416d8a7eacaae19e92a74ab4cd30d8034d3ab5a180098d605e12d1bb4/work,volatile 0 0
	proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
	tmpfs /dev tmpfs rw,context="system_u:object_r:container_file_t:s0:c9,c27",nosuid,size=65536k,mode=755,inode64 0 0
	devpts /dev/pts devpts rw,context="system_u:object_r:container_file_t:s0:c9,c27",nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=666 0 0
	mqueue /dev/mqueue mqueue rw,seclabel,nosuid,nodev,noexec,relatime 0 0
	sysfs /sys sysfs ro,seclabel,nosuid,nodev,noexec,relatime 0 0
	cgroup /sys/fs/cgroup cgroup2 ro,seclabel,nosuid,nodev,noexec,relatime 0 0
	shm /dev/shm tmpfs rw,context="system_u:object_r:container_file_t:s0:c9,c27",nosuid,nodev,noexec,relatime,size=65536k,inode64 0 0
	tmpfs /etc/resolv.conf tmpfs rw,seclabel,nosuid,nodev,noexec,size=3272884k,nr_inodes=819200,mode=755,inode64 0 0
	tmpfs /etc/hostname tmpfs rw,seclabel,nosuid,nodev,size=3272884k,nr_inodes=819200,mode=755,inode64 0 0
	tmpfs /run/.containerenv tmpfs rw,seclabel,nosuid,nodev,size=3272884k,nr_inodes=819200,mode=755,inode64 0 0
	tmpfs /etc/passwd tmpfs rw,seclabel,nosuid,nodev,noexec,size=3272884k,nr_inodes=819200,mode=755,inode64 0 0
	tmpfs /tmp tmpfs rw,seclabel,relatime,size=5120k,inode64 0 0
	/dev/vda4 /etc/hosts xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,prjquota 0 0
	/dev/vda4 /dev/termination-log xfs rw,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,prjquota 0 0
	tmpfs /run/secrets tmpfs rw,seclabel,nosuid,nodev,size=3272884k,nr_inodes=819200,mode=755,inode64 0 0
	tmpfs /opt/kafka/cluster-ca-certs tmpfs ro,seclabel,relatime,size=15213440k,inode64 0 0
	tmpfs /opt/kafka/broker-certs tmpfs ro,seclabel,relatime,size=15213440k,inode64 0 0
	tmpfs /opt/kafka/client-ca-certs tmpfs ro,seclabel,relatime,size=15213440k,inode64 0 0
	/dev/vda4 /opt/kafka/custom-config xfs ro,seclabel,relatime,attr2,inode64,logbufs=8,logbsize=32k,prjquota 0 0
	tmpfs /var/opt/kafka tmpfs rw,seclabel,relatime,size=4k,inode64 0 0
	/dev/vdc /var/lib/kafka/data-0 ext4 rw,seclabel,relatime 0 0
	tmpfs /run/secrets/kubernetes.io/serviceaccount tmpfs ro,seclabel,relatime,size=15213440k,inode64 0 0
	proc /proc/bus proc ro,nosuid,nodev,noexec,relatime 0 0
	proc /proc/fs proc ro,nosuid,nodev,noexec,relatime 0 0
	proc /proc/irq proc ro,nosuid,nodev,noexec,relatime 0 0
	proc /proc/sys proc ro,nosuid,nodev,noexec,relatime 0 0
	proc /proc/sysrq-trigger proc ro,nosuid,nodev,noexec,relatime 0 0
	tmpfs /proc/acpi tmpfs ro,context="system_u:object_r:container_file_t:s0:c9,c27",relatime,inode64 0 0
	tmpfs /proc/kcore tmpfs rw,context="system_u:object_r:container_file_t:s0:c9,c27",nosuid,size=65536k,mode=755,inode64 0 0
	tmpfs /proc/keys tmpfs rw,context="system_u:object_r:container_file_t:s0:c9,c27",nosuid,size=65536k,mode=755,inode64 0 0
	tmpfs /proc/timer_list tmpfs rw,context="system_u:object_r:container_file_t:s0:c9,c27",nosuid,size=65536k,mode=755,inode64 0 0
	tmpfs /proc/scsi tmpfs ro,context="system_u:object_r:container_file_t:s0:c9,c27",relatime,inode64 0 0
	tmpfs /sys/firmware tmpfs ro,context="system_u:object_r:container_file_t:s0:c9,c27",relatime,inode64 0 0
	

	2. [podiochaos]

	{
	  "VolumeMountPath": "/var/lib/kafka",
	  "Container": "kafka",
	  "Actions": [
	    {
	      "Type": "fault",
	      "path": "/var/lib/kafka/data/*",
	      "percent": 50,
	      "Faults": [
	        {
	          "errno": 5,
	          "weight": 1
	        }
	      ],
	      "Latency": "100ms"
	    },
	    {
	      "Type": "fault",
	      "path": "/var/lib/kafka/data*/*",
	      "percent": 50,
	      "Faults": [
	        {
	          "errno": 5,
	          "weight": 1
	        }
	      ],
	      "Latency": "100ms"
	    },
	    {
	      "Type": "fault",
	      "path": "/var/lib/kafka/data*/*",
	      "percent": 50,
	      "Faults": [
	        {
	          "errno": 5,
	          "weight": 1
	        }
	      ],
	      "Latency": "100ms"
	    },
	    {
	      "Type": "latency",
	      "path": "/var/lib/kafka/data*/*",
	      "percent": 50,
	      "Faults": [
	        {
	          "errno": 0,
	          "weight": 1
	        }
	      ],
	      "Latency": "100ms"
	    }
	  ]
	}

[Hexilee]

SendErr is likely caused by receiver being dropped (child thread exit), we should ignore this err (just report a warning), which is not important.