Skip to content

runtime: a Windows application launched via Steam sometimes freezes #71242

Open
@hajimehoshi

Description

@hajimehoshi
Member

Go version

go version go1.23.2 windows/amd64

Output of go env in your module/workspace:

set GO111MODULE=
set GOARCH=amd64
set GOBIN=
set GOCACHE=C:\Users\hajimehoshi\AppData\Local\go-build
set GOENV=C:\Users\hajimehoshi\AppData\Roaming\go\env
set GOEXE=.exe
set GOEXPERIMENT=
set GOFLAGS=
set GOHOSTARCH=amd64
set GOHOSTOS=windows
set GOINSECURE=
set GOMODCACHE=C:\Users\hajimehoshi\go\pkg\mod
set GONOPROXY=
set GONOSUMDB=
set GOOS=windows
set GOPATH=C:\Users\hajimehoshi\go
set GOPRIVATE=
set GOPROXY=https://proxy.golang.org,direct
set GOROOT=C:\Program Files\Go
set GOSUMDB=sum.golang.org
set GOTMPDIR=
set GOTOOLCHAIN=auto
set GOTOOLDIR=C:\Program Files\Go\pkg\tool\windows_amd64
set GOVCS=
set GOVERSION=go1.23.2
set GODEBUG=
set GOTELEMETRY=local
set GOTELEMETRYDIR=C:\Users\hajimehoshi\AppData\Roaming\go\telemetry
set GCCGO=gccgo
set GOAMD64=v1
set AR=ar
set CC=gcc
set CXX=g++
set CGO_ENABLED=1
set GOMOD=C:\Users\hajimehoshi\ebiten\go.mod
set GOWORK=
set CGO_CFLAGS=-O2 -g
set CGO_CPPFLAGS=
set CGO_CXXFLAGS=-O2 -g
set CGO_FFLAGS=-O2 -g
set CGO_LDFLAGS=-O2 -g
set PKG_CONFIG=pkg-config
set GOGCCFLAGS=-m64 -mthreads -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=C:\Users\HAJIME~1\AppData\Local\Temp\go-build1125585036=/tmp/go-build -gno-record-gcc-switches

What did you do?

Compile this Go program to an execute file (with -ldflags="-H=windowsgui")

EDIT: I could minized the case further. See #71242 (comment)

package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"syscall"
	"time"
	"unsafe"
)

const (
	WS_OVERLAPPEDWINDOW = 0x00000000 | 0x00C00000 | 0x00080000 | 0x00040000 | 0x00020000 | 0x00010000
	CW_USEDEFAULT       = ^0x7fffffff
	SW_SHOW             = 5
	WM_DESTROY          = 2
)

type (
	ATOM      uint16
	HANDLE    uintptr
	HINSTANCE HANDLE
	HICON     HANDLE
	HCURSOR   HANDLE
	HBRUSH    HANDLE
	HWND      HANDLE
	HMENU     HANDLE
)

type WNDCLASSEX struct {
	Size       uint32
	Style      uint32
	WndProc    uintptr
	ClsExtra   int32
	WndExtra   int32
	Instance   HINSTANCE
	Icon       HICON
	Cursor     HCURSOR
	Background HBRUSH
	MenuName   *uint16
	ClassName  *uint16
	IconSm     HICON
}

type RECT struct {
	Left, Top, Right, Bottom int32
}

type POINT struct {
	X, Y int32
}

type MSG struct {
	Hwnd    HWND
	Message uint32
	WParam  uintptr
	LParam  uintptr
	Time    uint32
	Pt      POINT
}

func GetModuleHandle(modulename *uint16) HINSTANCE {
	r, _, _ := syscall.SyscallN(procGetModuleHandle.Addr(), uintptr(unsafe.Pointer(modulename)))
	return HINSTANCE(r)
}

func RegisterClassEx(w *WNDCLASSEX) ATOM {
	r, _, _ := syscall.SyscallN(procRegisterClassEx.Addr(), uintptr(unsafe.Pointer(w)))
	return ATOM(r)
}

func CreateWindowEx(exStyle uint, className, windowName *uint16,
	style uint, x, y, width, height int, parent HWND, menu HMENU,
	instance HINSTANCE, param unsafe.Pointer) HWND {
	r, _, _ := syscall.SyscallN(procCreateWindowEx.Addr(), uintptr(exStyle), uintptr(unsafe.Pointer(className)),
		uintptr(unsafe.Pointer(windowName)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height),
		uintptr(parent), uintptr(menu), uintptr(instance), uintptr(param))
	return HWND(r)
}

func AdjustWindowRect(rect *RECT, style uint, menu bool) bool {
	var iMenu uintptr
	if menu {
		iMenu = 1
	}
	r, _, _ := syscall.SyscallN(procAdjustWindowRect.Addr(), uintptr(unsafe.Pointer(rect)), uintptr(style), iMenu)
	return r != 0
}

func ShowWindow(hwnd HWND, cmdshow int) bool {
	r, _, _ := syscall.SyscallN(procShowWindow.Addr(), uintptr(hwnd), uintptr(cmdshow))
	return r != 0
}

func GetMessage(msg *MSG, hwnd HWND, msgFilterMin, msgFilterMax uint32) int {
	r, _, _ := syscall.SyscallN(procGetMessage.Addr(), uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(msgFilterMin), uintptr(msgFilterMax))
	return int(r)
}

func TranslateMessage(msg *MSG) bool {
	r, _, _ := syscall.SyscallN(procTranslateMessage.Addr(), uintptr(unsafe.Pointer(msg)))
	return r != 0
}

func DispatchMessage(msg *MSG) uintptr {
	r, _, _ := syscall.SyscallN(procDispatchMessage.Addr(), uintptr(unsafe.Pointer(msg)))
	return r
}

func DefWindowProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	r, _, _ := syscall.SyscallN(procDefWindowProc.Addr(), uintptr(hwnd), uintptr(msg), wParam, lParam)
	return r
}

func PostQuitMessage(exitCode int) {
	syscall.SyscallN(procPostQuitMessage.Addr(), uintptr(exitCode))
}

var (
	kernel32            = syscall.NewLazyDLL("kernel32.dll")
	procGetModuleHandle = kernel32.NewProc("GetModuleHandleW")

	user32               = syscall.NewLazyDLL("user32.dll")
	procRegisterClassEx  = user32.NewProc("RegisterClassExW")
	procCreateWindowEx   = user32.NewProc("CreateWindowExW")
	procAdjustWindowRect = user32.NewProc("AdjustWindowRect")
	procShowWindow       = user32.NewProc("ShowWindow")
	procGetMessage       = user32.NewProc("GetMessageW")
	procTranslateMessage = user32.NewProc("TranslateMessage")
	procDispatchMessage  = user32.NewProc("DispatchMessageW")
	procDefWindowProc    = user32.NewProc("DefWindowProcW")
	procPostQuitMessage  = user32.NewProc("PostQuitMessage")
)

func init() {
	runtime.LockOSThread()
}

func main() {
	className, err := syscall.UTF16PtrFromString("Sample Window Class")
	if err != nil {
		panic(err)
	}
	inst := GetModuleHandle(className)

	wc := WNDCLASSEX{
		Size:      uint32(unsafe.Sizeof(WNDCLASSEX{})),
		WndProc:   syscall.NewCallback(wndProc),
		Instance:  inst,
		ClassName: className,
	}

	RegisterClassEx(&wc)

	wr := RECT{
		Left:   0,
		Top:    0,
		Right:  320,
		Bottom: 240,
	}
	title, err := syscall.UTF16PtrFromString("My Title")
	if err != nil {
		panic(err)
	}
	AdjustWindowRect(&wr, WS_OVERLAPPEDWINDOW, false)
	hwnd := CreateWindowEx(
		0, className,
		title,
		WS_OVERLAPPEDWINDOW,
		CW_USEDEFAULT, CW_USEDEFAULT, int(wr.Right-wr.Left), int(wr.Bottom-wr.Top),
		0, 0, inst, nil,
	)
	if hwnd == 0 {
		panic(syscall.GetLastError())
	}

	ShowWindow(hwnd, SW_SHOW)

	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	go func() {
		f, err := os.Create("log.txt")
		if err != nil {
			panic(err)
		}
		defer f.Close()

		log.SetOutput(f)

		for {
			time.Sleep(time.Second)
			var gcStats debug.GCStats
			debug.ReadGCStats(&gcStats)
			log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
			if err := f.Sync(); err != nil {
				panic(err)
			}
		}
	}()

	var msg MSG
	for GetMessage(&msg, 0, 0, 0) != 0 {
		TranslateMessage(&msg)
		DispatchMessage(&msg)
	}
}

func wndProc(hwnd HWND, msg uint32, wparam, lparam uintptr) uintptr {
	switch msg {
	case WM_DESTROY:
		PostQuitMessage(0)
	}
	return DefWindowProc(hwnd, msg, wparam, lparam)
}

Replace an exe file in a Steam game with the compiled exe file, and run it via Steam client.

What did you see happen?

The application sometimes freezes for more than 10 seconds. For example, I saw this log

2025/01/13 23:23:41 LastGC: 2025-01-13 23:23:41.6060533 +0900 JST, NumGC: 51, PauseTotal: 1.3186ms
2025/01/13 23:23:42 LastGC: 2025-01-13 23:23:42.6470915 +0900 JST, NumGC: 102, PauseTotal: 4.5945ms
2025/01/13 23:23:43 LastGC: 2025-01-13 23:23:43.6434896 +0900 JST, NumGC: 152, PauseTotal: 7.8752ms
2025/01/13 23:23:44 LastGC: 2025-01-13 23:23:44.6481645 +0900 JST, NumGC: 204, PauseTotal: 10.224ms
2025/01/13 23:23:45 LastGC: 2025-01-13 23:23:45.6448025 +0900 JST, NumGC: 255, PauseTotal: 12.5067ms
2025/01/13 23:23:46 LastGC: 2025-01-13 23:23:46.6584686 +0900 JST, NumGC: 309, PauseTotal: 14.2881ms
2025/01/13 23:23:47 LastGC: 2025-01-13 23:23:47.6550747 +0900 JST, NumGC: 362, PauseTotal: 17.056ms
2025/01/13 23:23:48 LastGC: 2025-01-13 23:23:48.6681264 +0900 JST, NumGC: 413, PauseTotal: 18.5012ms
2025/01/13 23:23:49 LastGC: 2025-01-13 23:23:49.6577372 +0900 JST, NumGC: 464, PauseTotal: 21.2877ms
2025/01/13 23:23:50 LastGC: 2025-01-13 23:23:50.6675317 +0900 JST, NumGC: 514, PauseTotal: 24.9508ms
2025/01/13 23:23:51 LastGC: 2025-01-13 23:23:51.6642908 +0900 JST, NumGC: 563, PauseTotal: 27.2671ms
2025/01/13 23:23:52 LastGC: 2025-01-13 23:23:52.6696781 +0900 JST, NumGC: 612, PauseTotal: 29.6692ms
2025/01/13 23:23:53 LastGC: 2025-01-13 23:23:53.6818947 +0900 JST, NumGC: 661, PauseTotal: 31.9968ms
2025/01/13 23:23:54 LastGC: 2025-01-13 23:23:54.6830572 +0900 JST, NumGC: 711, PauseTotal: 34.9958ms
2025/01/13 23:23:55 LastGC: 2025-01-13 23:23:55.6889185 +0900 JST, NumGC: 761, PauseTotal: 38.2468ms
2025/01/13 23:23:56 LastGC: 2025-01-13 23:23:56.6869067 +0900 JST, NumGC: 813, PauseTotal: 41.5747ms
2025/01/13 23:23:57 LastGC: 2025-01-13 23:23:57.6920325 +0900 JST, NumGC: 863, PauseTotal: 45.5415ms
2025/01/13 23:24:18 LastGC: 2025-01-13 23:23:58.2810119 +0900 JST, NumGC: 894, PauseTotal: 47.2387ms
2025/01/13 23:24:19 LastGC: 2025-01-13 23:24:19.3442472 +0900 JST, NumGC: 945, PauseTotal: 51.3869ms
2025/01/13 23:24:20 LastGC: 2025-01-13 23:24:20.3460036 +0900 JST, NumGC: 995, PauseTotal: 54.0004ms
2025/01/13 23:24:21 LastGC: 2025-01-13 23:24:21.3371656 +0900 JST, NumGC: 1047, PauseTotal: 55.3437ms
2025/01/13 23:24:22 LastGC: 2025-01-13 23:24:22.344327 +0900 JST, NumGC: 1098, PauseTotal: 56.9757ms
2025/01/13 23:24:23 LastGC: 2025-01-13 23:24:23.3523815 +0900 JST, NumGC: 1147, PauseTotal: 61.8229ms
2025/01/13 23:24:24 LastGC: 2025-01-13 23:24:24.3560493 +0900 JST, NumGC: 1200, PauseTotal: 64.7476ms
2025/01/13 23:24:25 LastGC: 2025-01-13 23:24:25.3552534 +0900 JST, NumGC: 1250, PauseTotal: 67.5551ms

You can see a freeze happens between 22:23:57 and 22:24:18.

What did you expect to see?

The application doesn't freeze.

Activity

hajimehoshi

hajimehoshi commented on Jan 13, 2025

@hajimehoshi
MemberAuthor

This was originally reported at hajimehoshi/ebiten#3181 by @corfe83.

  • A Steam overlay might cause this issue but we are not sure. This freeze happens even when the Steam overlay is disabled.
  • This freeze never happens when GC is suspended by debug.SetGCPercent(-1)
  • This freeze never happens when the application is launched without Steam.
added
BugReportIssues describing a possible bug in the Go implementation.
on Jan 13, 2025
hajimehoshi

hajimehoshi commented on Jan 13, 2025

@hajimehoshi
MemberAuthor

I could reproduce this even with a console application (without -ldflags="-H=windowsgui"). runtime.LockOSThread is not needed. Replacing an exe in a Steam game is still required.

package main

import (
	"log"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}
2025/01/14 00:19:14 LastGC: 2025-01-14 00:19:14.3262459 +0900 JST, NumGC: 51, PauseTotal: 1.016ms
2025/01/14 00:19:15 LastGC: 2025-01-14 00:19:15.3519275 +0900 JST, NumGC: 100, PauseTotal: 6.9976ms
2025/01/14 00:19:16 LastGC: 2025-01-14 00:19:16.3609673 +0900 JST, NumGC: 148, PauseTotal: 11.6081ms
2025/01/14 00:19:17 LastGC: 2025-01-14 00:19:17.3715984 +0900 JST, NumGC: 197, PauseTotal: 15.5458ms
2025/01/14 00:19:18 LastGC: 2025-01-14 00:19:18.3726086 +0900 JST, NumGC: 245, PauseTotal: 17.6441ms
2025/01/14 00:19:19 LastGC: 2025-01-14 00:19:19.3673871 +0900 JST, NumGC: 293, PauseTotal: 19.8266ms
2025/01/14 00:19:20 LastGC: 2025-01-14 00:19:20.3597451 +0900 JST, NumGC: 341, PauseTotal: 25.4462ms
2025/01/14 00:19:21 LastGC: 2025-01-14 00:19:21.3536769 +0900 JST, NumGC: 388, PauseTotal: 30.0621ms
2025/01/14 00:19:37 LastGC: 2025-01-14 00:19:37.1879618 +0900 JST, NumGC: 424, PauseTotal: 30.2376ms
2025/01/14 00:19:38 LastGC: 2025-01-14 00:19:38.1708792 +0900 JST, NumGC: 468, PauseTotal: 31.8384ms
2025/01/14 00:19:39 LastGC: 2025-01-14 00:19:39.1731811 +0900 JST, NumGC: 515, PauseTotal: 33.0891ms
2025/01/14 00:19:40 LastGC: 2025-01-14 00:19:40.1890122 +0900 JST, NumGC: 562, PauseTotal: 33.7643ms
2025/01/14 00:19:41 LastGC: 2025-01-14 00:19:41.1767277 +0900 JST, NumGC: 608, PauseTotal: 34.0499ms

A freeze occurs between 00:19:21 and 00:19:37

hajimehoshi

hajimehoshi commented on Jan 13, 2025

@hajimehoshi
MemberAuthor

Apparently most of the threads got stuck at NtWaitForSingleObject when freezing, but I have no idea what was going on there...

Thread 0x4 (gameoverlayrenderer64!OverlayHookD3D3 seems a hooked function by Steam)
Image

Thread 0x5
Image

changed the title [-]a Windows application launched via Steam sometimes freezes[/-] [+]runtime: a Windows application launched via Steam sometimes freezes[/+] on Jan 13, 2025
added
NeedsInvestigationSomeone must examine and confirm this is a valid issue and not a duplicate of an existing one.
on Jan 13, 2025
added this to the Backlog milestone on Jan 13, 2025
mknyszek

mknyszek commented on Jan 13, 2025

@mknyszek
Contributor

CC @golang/runtime

qmuntal

qmuntal commented on Jan 13, 2025

@qmuntal
Member

Does this issue reproduce with Go 1.22?

hajimehoshi

hajimehoshi commented on Jan 13, 2025

@hajimehoshi
MemberAuthor

Does this issue reproduce with Go 1.22?

Yes, I could reproduce this with Go 1.22.9.

mknyszek

mknyszek commented on Jan 13, 2025

@mknyszek
Contributor

Do you have a stack trace for each thread, for example from a debugger?

My guess based on the information provided would be that the threads are stuck in some GC-related thing (spinning up GC mark workers? that only happens on the first GC if GOMAXPROCS doesn't ever change) on a note (https://cs.opensource.google/go/go/+/master:src/runtime/os_windows.go;l=676;drc=4f881115d4067bda8a236aabcae8c41cdd13b4d0). I couldn't tell you why that would be a problem, since this happens all the time.

Alternatively, the threads are going to sleep for STW, which is also based on a note (https://cs.opensource.google/go/go/+/master:src/runtime/proc.go;l=1612;drc=f025d19e7b3f0c66242760c213cc2b54cb100f69), and perhaps something is preventing the thread stopping the world from continuing promptly. Note that the thread which is stopping-the-world spins, sleeping in 100ms increments (but is awoken early when the last thread goes to sleep). So it might be that too. We could confirm/deny this by disabling the GC but calling runtime.ReadMemStats at some regular interval. This also happens all the time without issue, though, so I also don't really have a guess as to why this would be a problem running under Steam.

hajimehoshi

hajimehoshi commented on Jan 13, 2025

@hajimehoshi
MemberAuthor

Do you have a stack trace for each thread, for example from a debugger?

I'm afraid I'm not familiar with WinDbg. How can I get the stack traces as a file? I'll try tomorrow.

Also I can insert printlns in the runtime by -overlay. I'll try this later too.

10 remaining items

hajimehoshi

hajimehoshi commented on Jan 14, 2025

@hajimehoshi
MemberAuthor

addr2line requires that each line contains only an address, so you'll need to edit the stack trace file to remove everything else.

Hmm, addr2line still didn't resolve the file like this:

00000000003DE029
00000000003DC40D
00000000003DC4C1
00000000003A1B9B
hajimehoshi

hajimehoshi commented on Jan 14, 2025

@hajimehoshi
MemberAuthor

I am not sure this would be helpful, but I objdumped the binary and searched the binaries in the suspicous stack traces. I realized the addresses are 'shifted'.

Stack traces with ReleaseMutex (I newly dumped this):

                                 000000BE991FF218  00007FFDB8D9BEAD  00007FFDBB9907A4  30                System    ntdll.NtReleaseMutant+14
                                 000000BE991FF248  00007FFDAFEE1EC0  00007FFDB8D9BEAD  70                User      kernelbase.ReleaseMutex+D
                                 000000BE991FF2B8  00000000007FE029  00007FFDAFEE1EC0  170               User      gameoverlayrenderer64.OverlayHookD3D3+27380
                                 000000BE991FF428  00000000007FC40D  00000000007FE029  30                User      innovation2007_windows_amd64.00000000007FE029
                                 000000BE991FF458  00000000007FC4C1  00000000007FC40D  28                User      innovation2007_windows_amd64.00000000007FC40D
                                 000000BE991FF480  00000000007C1B9B  00000000007FC4C1  38                User      innovation2007_windows_amd64.00000000007FC4C1
                                 000000BE991FF4B8  00000000007C1C56  00000000007C1B9B  18                User      innovation2007_windows_amd64.00000000007C1B9B
                                 000000BE991FF4D0  00000000007C2592  00000000007C1C56  560               User      innovation2007_windows_amd64.00000000007C1C56
                                 000000BE991FFA30  00000000007D3D35  00000000007C2592  18                User      innovation2007_windows_amd64.00000000007C2592
                                 000000BE991FFA48  00000000007ADC96  00000000007D3D35  30                User      innovation2007_windows_amd64.00000000007D3D35
                                 000000BE991FFA78  00000000007B3825  00000000007ADC96  18                User      innovation2007_windows_amd64.00000000007ADC96
                                 000000BE991FFA90  00000000007ABC87  00000000007B3825  68                User      innovation2007_windows_amd64.00000000007B3825
                                 000000BE991FFAF8  00000000007A8407  00000000007ABC87  50                User      innovation2007_windows_amd64.00000000007ABC87
                                 000000BE991FFB48  00000000007FA869  00000000007A8407  166FD83E8         User      innovation2007_windows_amd64.00000000007A8407
                                 000000C0001D7F30  00000000007A80CC  00000000007FA869  90                User      innovation2007_windows_amd64.00000000007FA869
                                 000000C0001D7FC0  00000000007A7EA5  00000000007A80CC  18                User      innovation2007_windows_amd64.00000000007A80CC
                                 000000C0001D7FD8  00000000007FC821  00000000007A7EA5  8                 User      innovation2007_windows_amd64.00000000007A7EA5
                                 000000C0001D7FE0  0000000000000000  00000000007FC821                    User      innovation2007_windows_amd64.00000000007FC821

00000000007FE029:

TEXT runtime.asmstdcall.abi0(SB) C:/Program Files/Go/src/runtime/sys_windows_amd64.s
  sys_windows_amd64.s:20	0x46dfa0		55			PUSHQ BP			
  sys_windows_amd64.s:20	0x46dfa1		4889e5			MOVQ SP, BP			
  sys_windows_amd64.s:20	0x46dfa4		4883ec10		SUBQ $0x10, SP			
  sys_windows_amd64.s:21	0x46dfa8		4889e0			MOVQ SP, AX			
  sys_windows_amd64.s:22	0x46dfab		4883e4f0		ANDQ $-0x10, SP			
  sys_windows_amd64.s:23	0x46dfaf		4889442408		MOVQ AX, 0x8(SP)		
  sys_windows_amd64.s:24	0x46dfb4		48890c24		MOVQ CX, 0(SP)			
  sys_windows_amd64.s:26	0x46dfb8		488b01			MOVQ 0(CX), AX			
  sys_windows_amd64.s:27	0x46dfbb		488b7110		MOVQ 0x10(CX), SI		
  sys_windows_amd64.s:28	0x46dfbf		488b4908		MOVQ 0x8(CX), CX		
  sys_windows_amd64.s:31	0x46dfc3		65488b3c2530000000	MOVQ GS:0x30, DI		
  sys_windows_amd64.s:32	0x46dfcc		c7476800000000		MOVL $0x0, 0x68(DI)		
  sys_windows_amd64.s:34	0x46dfd3		4881ec50010000		SUBQ $0x150, SP			
  sys_windows_amd64.s:37	0x46dfda		83f900			CMPL CX, $0x0			
  sys_windows_amd64.s:37	0x46dfdd		7448			JE 0x46e027			
  sys_windows_amd64.s:38	0x46dfdf		83f901			CMPL CX, $0x1			
  sys_windows_amd64.s:38	0x46dfe2		743b			JE 0x46e01f			
  sys_windows_amd64.s:39	0x46dfe4		83f902			CMPL CX, $0x2			
  sys_windows_amd64.s:39	0x46dfe7		742d			JE 0x46e016			
  sys_windows_amd64.s:40	0x46dfe9		83f903			CMPL CX, $0x3			
  sys_windows_amd64.s:40	0x46dfec		741f			JE 0x46e00d			
  sys_windows_amd64.s:41	0x46dfee		83f904			CMPL CX, $0x4			
  sys_windows_amd64.s:41	0x46dff1		7411			JE 0x46e004			
  sys_windows_amd64.s:44	0x46dff3		83f92a			CMPL CX, $0x2a			
  sys_windows_amd64.s:45	0x46dff6		7e02			JLE 0x46dffa			
  sys_windows_amd64.s:46	0x46dff8		cd03			INT $0x3			
  sys_windows_amd64.s:49	0x46dffa		4889e7			MOVQ SP, DI			
  sys_windows_amd64.s:50	0x46dffd		fc			CLD				
  sys_windows_amd64.s:51	0x46dffe		f348a5			REP; MOVSQ DS:0(SI), ES:0(DI)	
  sys_windows_amd64.s:52	0x46e001		4889e6			MOVQ SP, SI			
  sys_windows_amd64.s:60	0x46e004		4c8b4e18		MOVQ 0x18(SI), R9		
  sys_windows_amd64.s:61	0x46e008		66490f6ed9		MOVQ R9, X3			
  sys_windows_amd64.s:63	0x46e00d		4c8b4610		MOVQ 0x10(SI), R8		
  sys_windows_amd64.s:64	0x46e011		66490f6ed0		MOVQ R8, X2			
  sys_windows_amd64.s:66	0x46e016		488b5608		MOVQ 0x8(SI), DX		
  sys_windows_amd64.s:67	0x46e01a		66480f6eca		MOVQ DX, X1			
  sys_windows_amd64.s:69	0x46e01f		488b0e			MOVQ 0(SI), CX			
  sys_windows_amd64.s:70	0x46e022		66480f6ec1		MOVQ CX, X0			
  sys_windows_amd64.s:74	0x46e027		ffd0			CALL AX				
  sys_windows_amd64.s:76	0x46e029		4881c450010000		ADDQ $0x150, SP			<-- Start Here
  sys_windows_amd64.s:79	0x46e030		488b0c24		MOVQ 0(SP), CX			
  sys_windows_amd64.s:80	0x46e034		488b642408		MOVQ 0x8(SP), SP		
  sys_windows_amd64.s:81	0x46e039		48894118		MOVQ AX, 0x18(CX)		
  sys_windows_amd64.s:85	0x46e03d		660fd64120		MOVQ X0, 0x20(CX)		
  sys_windows_amd64.s:88	0x46e042		65488b3c2530000000	MOVQ GS:0x30, DI		
  sys_windows_amd64.s:89	0x46e04b		8b4768			MOVL 0x68(DI), AX		
  sys_windows_amd64.s:90	0x46e04e		48894128		MOVQ AX, 0x28(CX)		
  sys_windows_amd64.s:92	0x46e052		4883c410		ADDQ $0x10, SP			
  sys_windows_amd64.s:92	0x46e056		5d			POPQ BP				
  sys_windows_amd64.s:92	0x46e057		c3			RET	<-- End Here

00000000007FC40D:

TEXT runtime.asmcgocall_landingpad.abi0(SB) C:/Program Files/Go/src/runtime/asm_amd64.s
  asm_amd64.s:858	0x46c400		55			PUSHQ BP		
  asm_amd64.s:858	0x46c401		4889e5			MOVQ SP, BP		
  asm_amd64.s:862	0x46c404		4883ec20		SUBQ $0x20, SP		
  asm_amd64.s:871	0x46c408		4889d9			MOVQ BX, CX		
  asm_amd64.s:872	0x46c40b		ffd0			CALL AX			
  asm_amd64.s:875	0x46c40d		90			NOPL			<-- Start Here
  asm_amd64.s:876	0x46c40e		4883c420		ADDQ $0x20, SP		
  asm_amd64.s:877	0x46c412		5d			POPQ BP			
  asm_amd64.s:877	0x46c413		c3			RET			<-- End Here
  asm_amd64.s:880	0x46c414		4889df			MOVQ BX, DI		
  asm_amd64.s:881	0x46c417		ffe0			JMP AX			
  :-1			0x46c419		cc			INT $0x3		
  :-1			0x46c41a		cc			INT $0x3		
  :-1			0x46c41b		cc			INT $0x3		
  :-1			0x46c41c		cc			INT $0x3		
  :-1			0x46c41d		cc			INT $0x3		
  :-1			0x46c41e		cc			INT $0x3		
  :-1			0x46c41f		cc			INT $0x3

00000000007FC4C1:

TEXT runtime.asmcgocall.abi0(SB) C:/Program Files/Go/src/runtime/asm_amd64.s
  asm_amd64.s:887	0x46c420		55			PUSHQ BP					
  asm_amd64.s:887	0x46c421		4889e5			MOVQ SP, BP					
  asm_amd64.s:888	0x46c424		488b442410		MOVQ 0x10(SP), AX				
  asm_amd64.s:889	0x46c429		488b5c2418		MOVQ 0x18(SP), BX				
  asm_amd64.s:891	0x46c42e		4889e2			MOVQ SP, DX					
  asm_amd64.s:897	0x46c431		488b0d38c81500		MOVQ runtime.tls_g(SB), CX			
  asm_amd64.s:897	0x46c438		65488b09		MOVQ GS:0(CX), CX				
  asm_amd64.s:898	0x46c43c		488b39			MOVQ 0(CX), DI					
  asm_amd64.s:899	0x46c43f		4883ff00		CMPQ DI, $0x0					
  asm_amd64.s:900	0x46c443		7462			JE 0x46c4a7					
  asm_amd64.s:901	0x46c445		4c8b4730		MOVQ 0x30(DI), R8				
  asm_amd64.s:902	0x46c449		498b7050		MOVQ 0x50(R8), SI				
  asm_amd64.s:903	0x46c44d		4839f7			CMPQ DI, SI					
  asm_amd64.s:904	0x46c450		7455			JE 0x46c4a7					
  asm_amd64.s:905	0x46c452		498b30			MOVQ 0(R8), SI					
  asm_amd64.s:906	0x46c455		4839f7			CMPQ DI, SI					
  asm_amd64.s:907	0x46c458		744d			JE 0x46c4a7					
  asm_amd64.s:912	0x46c45a		e8c1d4ffff		CALL gosave_systemstack_switch(SB)		
  asm_amd64.s:913	0x46c45f		488931			MOVQ SI, 0(CX)					
  asm_amd64.s:914	0x46c462		488b6638		MOVQ 0x38(SI), SP				
  asm_amd64.s:917	0x46c466		4883ec10		SUBQ $0x10, SP					
  asm_amd64.s:918	0x46c46a		4883e4f0		ANDQ $-0x10, SP					
  asm_amd64.s:919	0x46c46e		48897c2408		MOVQ DI, 0x8(SP)				
  asm_amd64.s:920	0x46c473		488b7f08		MOVQ 0x8(DI), DI				
  asm_amd64.s:921	0x46c477		4829d7			SUBQ DX, DI					
  asm_amd64.s:922	0x46c47a		48893c24		MOVQ DI, 0(SP)					
  asm_amd64.s:923	0x46c47e		e87dffffff		CALL runtime.asmcgocall_landingpad.abi0(SB)	
  asm_amd64.s:926	0x46c483		488b0de6c71500		MOVQ runtime.tls_g(SB), CX			
  asm_amd64.s:926	0x46c48a		65488b09		MOVQ GS:0(CX), CX				
  asm_amd64.s:927	0x46c48e		488b7c2408		MOVQ 0x8(SP), DI				
  asm_amd64.s:928	0x46c493		488b7708		MOVQ 0x8(DI), SI				
  asm_amd64.s:929	0x46c497		482b3424		SUBQ 0(SP), SI					
  asm_amd64.s:930	0x46c49b		488939			MOVQ DI, 0(CX)					
  asm_amd64.s:931	0x46c49e		4889f4			MOVQ SI, SP					
  asm_amd64.s:933	0x46c4a1		89442420		MOVL AX, 0x20(SP)				
  asm_amd64.s:934	0x46c4a5		5d			POPQ BP						
  asm_amd64.s:934	0x46c4a6		c3			RET						
  asm_amd64.s:947	0x46c4a7		4883ec10		SUBQ $0x10, SP					
  asm_amd64.s:948	0x46c4ab		4883e4f0		ANDQ $-0x10, SP					
  asm_amd64.s:949	0x46c4af		48c744240800000000	MOVQ $0x0, 0x8(SP)				
  asm_amd64.s:950	0x46c4b8		48891424		MOVQ DX, 0(SP)					
  asm_amd64.s:951	0x46c4bc		e83fffffff		CALL runtime.asmcgocall_landingpad.abi0(SB)	
  asm_amd64.s:952	0x46c4c1		488b3424		MOVQ 0(SP), SI					<-- Start Here
  asm_amd64.s:953	0x46c4c5		4889f4			MOVQ SI, SP					
  asm_amd64.s:954	0x46c4c8		89442420		MOVL AX, 0x20(SP)				
  asm_amd64.s:955	0x46c4cc		5d			POPQ BP						
  asm_amd64.s:955	0x46c4cd		c3			RET						<-- End Here
  :-1			0x46c4ce		cc			INT $0x3					
  :-1			0x46c4cf		cc			INT $0x3					
  :-1			0x46c4d0		cc			INT $0x3					
  :-1			0x46c4d1		cc			INT $0x3					
  :-1			0x46c4d2		cc			INT $0x3					
  :-1			0x46c4d3		cc			INT $0x3					
  :-1			0x46c4d4		cc			INT $0x3					
  :-1			0x46c4d5		cc			INT $0x3					
  :-1			0x46c4d6		cc			INT $0x3					
  :-1			0x46c4d7		cc			INT $0x3					
  :-1			0x46c4d8		cc			INT $0x3					
  :-1			0x46c4d9		cc			INT $0x3					
  :-1			0x46c4da		cc			INT $0x3					
  :-1			0x46c4db		cc			INT $0x3					
  :-1			0x46c4dc		cc			INT $0x3					
  :-1			0x46c4dd		cc			INT $0x3					
  :-1			0x46c4de		cc			INT $0x3					
  :-1			0x46c4df		cc			INT $0x3					

00000000007C1B9B

TEXT runtime.stdcall(SB) C:/Program Files/Go/src/runtime/os_windows.go
  os_windows.go:956	0x431b20		55			PUSHQ BP				
  os_windows.go:956	0x431b21		4889e5			MOVQ SP, BP				
  os_windows.go:956	0x431b24		4883ec28		SUBQ $0x28, SP				
  os_windows.go:957	0x431b28		498b4e30		MOVQ 0x30(R14), CX			
  os_windows.go:959	0x431b2c		488981e0020000		MOVQ AX, 0x2e0(CX)			
  os_windows.go:961	0x431b33		83b9e000000000		CMPL 0xe0(CX), $0x0			
  os_windows.go:961	0x431b3a		7438			JE 0x431b74				
  os_windows.go:961	0x431b3c		4883b91803000000	CMPQ 0x318(CX), $0x0			
  os_windows.go:961	0x431b44		752a			JNE 0x431b70				
  runtime2.go:269	0x431b46		4c89f0			MOVQ R14, AX				
  os_windows.go:963	0x431b49		90			NOPL					
  runtime2.go:269	0x431b4a		48898120030000		MOVQ AX, 0x320(CX)			
  os_windows.go:964	0x431b51		488b442430		MOVQ 0x30(SP), AX			
  os_windows.go:964	0x431b56		48898110030000		MOVQ AX, 0x310(CX)			
  os_windows.go:967	0x431b5d		488d442438		LEAQ 0x38(SP), AX			
  os_windows.go:967	0x431b62		48898118030000		MOVQ AX, 0x318(CX)			
  os_windows.go:967	0x431b69		b801000000		MOVL $0x1, AX				
  os_windows.go:968	0x431b6e		eb06			JMP 0x431b76				
  os_windows.go:968	0x431b70		31c0			XORL AX, AX				
  os_windows.go:961	0x431b72		eb02			JMP 0x431b76				
  os_windows.go:961	0x431b74		31c0			XORL AX, AX				
  os_windows.go:958	0x431b76		48894c2420		MOVQ CX, 0x20(SP)			
  os_windows.go:971	0x431b7b		8844241f		MOVB AL, 0x1f(SP)			
  os_windows.go:970	0x431b7f		488b15aa1b1500		MOVQ runtime.asmstdcallAddr(SB), DX	
  os_windows.go:970	0x431b86		48891424		MOVQ DX, 0(SP)				
  os_windows.go:970	0x431b8a		488d91e0020000		LEAQ 0x2e0(CX), DX			
  os_windows.go:970	0x431b91		4889542408		MOVQ DX, 0x8(SP)			
  os_windows.go:970	0x431b96		e885a80300		CALL runtime.asmcgocall.abi0(SB)	
  os_windows.go:970	0x431b9b		450f57ff		XORPS X15, X15				<-- Start Here
  os_windows.go:970	0x431b9f		4c8b35ca701900		MOVQ runtime.tls_g(SB), R14		
  os_windows.go:970	0x431ba6		654d8b36		MOVQ GS:0(R14), R14			
  os_windows.go:970	0x431baa		4d8b36			MOVQ 0(R14), R14			
  os_windows.go:971	0x431bad		0fb644241f		MOVZX 0x1f(SP), AX			
  os_windows.go:971	0x431bb2		84c0			TESTL AL, AL				
  os_windows.go:971	0x431bb4		7412			JE 0x431bc8				
  os_windows.go:972	0x431bb6		488b4c2420		MOVQ 0x20(SP), CX			
  os_windows.go:972	0x431bbb		48c7811803000000000000	MOVQ $0x0, 0x318(CX)			
  os_windows.go:972	0x431bc6		eb05			JMP 0x431bcd				
  os_windows.go:974	0x431bc8		488b4c2420		MOVQ 0x20(SP), CX			
  os_windows.go:974	0x431bcd		488b81f8020000		MOVQ 0x2f8(CX), AX			
  os_windows.go:974	0x431bd4		4883c428		ADDQ $0x28, SP				
  os_windows.go:974	0x431bd8		5d			POPQ BP					
  os_windows.go:974	0x431bd9		c3			RET					<-- End Here
  :-1			0x431bda		cc			INT $0x3				
  :-1			0x431bdb		cc			INT $0x3				
  :-1			0x431bdc		cc			INT $0x3				
  :-1			0x431bdd		cc			INT $0x3				
  :-1			0x431bde		cc			INT $0x3				
  :-1			0x431bdf		cc			INT $0x3
hajimehoshi

hajimehoshi commented on Jan 14, 2025

@hajimehoshi
MemberAuthor

Here is the resolved symbols in the stack trace below gameoverlayrenderer64.OverlayHookD3D3+27380:

runtime.asmstdcall
C:/Program Files/Go/src/runtime/sys_windows_amd64.s:76
runtime.asmcgocall_landingpad
C:/Program Files/Go/src/runtime/asm_amd64.s:875
runtime.asmcgocall
C:/Program Files/Go/src/runtime/asm_amd64.s:952
runtime.stdcall
C:/Program Files/Go/src/runtime/os_windows.go:970
runtime.stdcall1
C:/Program Files/Go/src/runtime/os_windows.go:991
runtime.preemptM
C:/Program Files/Go/src/runtime/os_windows.go:1351
runtime.preemptone
C:/Program Files/Go/src/runtime/proc.go:6328
runtime.(*gcControllerState).enlistWorker
C:/Program Files/Go/src/runtime/mgcpacer.go:723
runtime.(*gcWork).balance
C:/Program Files/Go/src/runtime/mgcwork.go:306
runtime.gcDrain
C:/Program Files/Go/src/runtime/mgcwork.go:235
runtime.gcBgMarkWorker.func2
C:/Program Files/Go/src/runtime/mgc.go:1459
runtime.systemstack
C:/Program Files/Go/src/runtime/asm_amd64.s:517
runtime.gcBgMarkWorker
C:/Program Files/Go/src/runtime/mgc.go:1423
runtime.gcBgMarkStartWorkers.gowrap1
C:/Program Files/Go/src/runtime/mgc.go:1279
runtime.goexit
C:/Program Files/Go/src/runtime/asm_amd64.s:1701
prattmic

prattmic commented on Jan 14, 2025

@prattmic
Member

Apologies for the bad instructions, I don't use addr2line much.

That stack is really interesting. On Windows, asynchronous preemption uses the Windows thread suspend/resume APIs. It seems that this thread is getting stuck in ResumeThread. Maybe the Steam overlay messes with this API somehow?

mknyszek

mknyszek commented on Jan 14, 2025

@mknyszek
Contributor

@hajimehoshi Can you reproduce with GODEBUG=asyncpreemptoff=1?

hajimehoshi

hajimehoshi commented on Jan 15, 2025

@hajimehoshi
MemberAuthor

@hajimehoshi Can you reproduce with GODEBUG=asyncpreemptoff=1?

I couldn't reproduce this with GODEBUG=asyncpreemptoff=1. I tried 10 times and watched each trial reached 10,000 GCs without any issues.

hajimehoshi

hajimehoshi commented on Jan 15, 2025

@hajimehoshi
MemberAuthor

By the way, is there a compile option to specify asyncpreemptoff=1 by default? //go:debug didn't recognize asyncpreemptoff=1 unfortunately. For Steam games, it is not feasible to ask users to set an environment variable GODEBUG for this, so I'd like to specify this at the compile time. The only way to do this is using -overlay to rewrite os_windows.go, but is my understanding correct?

Of course, it would be the best that the runtime is fixed to suppress this issue.

There seem multiple situations where asyncpreemptoff=1 is a workaround.

EDIT: -ldflags="-X=runtime.godebugDefault=asyncpreemptoff=1" seemed to work as intended.

mknyszek

mknyszek commented on Jan 15, 2025

@mknyszek
Contributor

//go:debug didn't recognize asyncpreemptoff=1 unfortunately.

I think we should fix that, though I'm glad you found another workaround with -ldflags. Maybe that makes more sense long-term? This issue seems to be related to the specific DLL injection (?) that Steam does (gameoverlayrenderer64.OverlayHookD3D3+27380 is pretty fishy). It would be unfortunate to disable async preemption for all users of a given package, since it does provide tangible latency benefits.

hajimehoshi

hajimehoshi commented on Jan 15, 2025

@hajimehoshi
MemberAuthor

I think we should fix that

Yeah, so would it be possible for //go:debug to accept asyncpreemptoff=1? I'm happy to file this if needed. This would be much better and less hacky than -ldflags. Also, I think this would be very helpful for various situations where this settings is required.

It would be unfortunate to disable async preemption for all users of a given package, since it does provide tangible latency benefits.

I mean, I want to disable async preemption when I build my games for Steam Windows. So I would specify asyncpreemptoff=1 for a main package (with a build tag). I don't intend to disable it for all the users of my package.

prattmic

prattmic commented on Jan 15, 2025

@prattmic
Member

Yeah, so would it be possible for //go:debug to accept asyncpreemptoff=1? I'm happy to file this if needed.

I think this is reasonable, please do file an issue.

//go:debug only has an effect in main packages anyways, so applying to all users of a package isn't a problem.

That said, I don't think this is the final resolution. Steam seems to be messing with our process in a way that breaks certain Windows API calls. I think the next step is to report this bug to Valve, as current evidence points to a bug in whatever Steam is doing.

hajimehoshi

hajimehoshi commented on Jan 15, 2025

@hajimehoshi
MemberAuthor

I think this is reasonable, please do file an issue.

Thanks! #71283

That said, I don't think this is the final resolution. Steam seems to be messing with our process in a way that breaks certain Windows API calls. I think the next step is to report this bug to Valve, as current evidence points to a bug in whatever Steam is doing.

I agree. I've already reported the current situation https://steamcommunity.com/discussions/forum/0/595138100650327297/.

I'll try to make a simple C program that invokes SuspendThread and ResumeThread and tries to cause the freeze when I have time.

I appreciate all of your quick responses!

mknyszek

mknyszek commented on Jan 15, 2025

@mknyszek
Contributor

Apologies, the //go:debug stuff is new to me. There's more than just GODEBUG setting that we should allow; perhaps we should file a new issue capturing all of them. EDIT: I'm clearly behind, #71283 already exists. 😅 Thanks!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

No one assigned

    Labels

    BugReportIssues describing a possible bug in the Go implementation.NeedsInvestigationSomeone must examine and confirm this is a valid issue and not a duplicate of an existing one.OS-Windowscompiler/runtimeIssues related to the Go compiler and/or runtime.

    Type

    Projects

    Status

    Todo

    Relationships

    None yet

      Development

      No branches or pull requests

        Participants

        @hajimehoshi@mknyszek@prattmic@qmuntal@gopherbot

        Issue actions

          runtime: a Windows application launched via Steam sometimes freezes · Issue #71242 · golang/go