Sentry: panic.go:884: runtime error: invalid memory address or nil pointer dereference (1) attached stack trace -- stack trace: | runtime.gopanic | GOROOT/src/runtime/panic.go:884 | runtime.p...

[cockroach-sentry]

This issue was auto filed by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry Link: https://cockroach-labs.sentry.io/issues/5060021413/?referrer=webhooks_plugin

Panic Message:

panic.go:884: runtime error: invalid memory address or nil pointer dereference
(1) attached stack trace
  -- stack trace:
  | runtime.gopanic
  | 	GOROOT/src/runtime/panic.go:884
  | runtime.panicmem
  | 	GOROOT/src/runtime/panic.go:260
  | runtime.sigpanic
  | 	GOROOT/src/runtime/signal_unix.go:839
  | github.com/cockroachdb/cockroach/pkg/cloud/amazon.(*s3Storage).Size
  | 	github.com/cockroachdb/cockroach/pkg/cloud/amazon/s3_storage.go:888
  | github.com/cockroachdb/cockroach/pkg/cloud/amazon.(*s3Storage).ReadFileAt
  | 	github.com/cockroachdb/cockroach/pkg/cloud/amazon/s3_storage.go:785
  | github.com/cockroachdb/cockroach/pkg/cloud/amazon.(*s3Storage).ReadFile
  | 	github.com/cockroachdb/cockroach/pkg/cloud/amazon/s3_storage.go:752
  | github.com/cockroachdb/cockroach/pkg/cloud.(*esWrapper).ReadFile
  | 	github.com/cockroachdb/cockroach/pkg/cloud/impl_registry.go:331
  | github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backupdest.containsManifest
  | 	github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backupdest/backup_destination.go:71
  | github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backupdest.ResolveDest
  | 	github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backupdest/backup_destination.go:169
  | github.com/cockroachdb/cockroach/pkg/ccl/backupccl.(*backupResumer).Resume
  | 	github.com/cockroachdb/cockroach/pkg/ccl/backupccl/backup_job.go:519
  | github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine.func2
  | 	github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1662
  | github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).stepThroughStateMachine
  | 	github.com/cockroachdb/cockroach/pkg/jobs/registry.go:1663
  | github.com/cockroachdb/cockroach/pkg/jobs.(*Registry).runJob
  | 	github.com/cockroachdb/cockroach/pkg/jobs/adopt.go:474
  | github.com/cockroachdb/cockroach/pkg/jobs.(*StartableJob).Start.func2
  | 	github.com/cockroachdb/cockroach/pkg/jobs/jobs.go:978
  | github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2
  | 	github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:470
  | runtime.goexit
  | 	src/runtime/asm_arm64.s:1172
Wraps: (2) runtime error: invalid memory address or nil pointer dereference
Error types: (1) *withstack.withStack (2) runtime.errorString
-- report composition:
runtime.errorString
panic.go:884: *withstack.withStack (top exception)

Stacktrace (expand for inline code snippets):

src/runtime/asm_arm64.s#L1171-L1173

cockroach/pkg/util/stop/stopper.go

Lines 469 to 471 in 37f7574

	sp.UpdateGoroutineIDToCurrent()
	f(ctx)
	}()

cockroach/pkg/jobs/jobs.go

Lines 977 to 979 in 37f7574

	defer cancel()
	sj.execErr = sj.registry.runJob(resumeCtx, sj.resumer, sj.Job, StatusRunning, sj.taskName())
	close(sj.execDone)

cockroach/pkg/jobs/adopt.go

Lines 473 to 475 in 37f7574

	// Run the actual job.
	err := r.stepThroughStateMachine(ctx, execCtx, resumer, job, status, finalResumeError)
	// If the context has been canceled, disregard errors for the sake of logging

cockroach/pkg/jobs/registry.go

Lines 1662 to 1664 in 37f7574

	err = resumer.Resume(resumeCtx, execCtx)
	}()

cockroach/pkg/jobs/registry.go

Lines 1661 to 1663 in 37f7574

	}()
	err = resumer.Resume(resumeCtx, execCtx)
	}()

cockroach/pkg/ccl/backupccl/backup_job.go

Lines 518 to 520 in 37f7574

	var err error
	backupDest, err = backupdest.ResolveDest(ctx, p.User(), details.Destination, details.EndTime,
	details.IncrementalFrom, p.ExecCfg())

cockroach/pkg/ccl/backupccl/backupdest/backup_destination.go

Lines 168 to 170 in 37f7574

	defer defaultStore.Close()
	exists, err := containsManifest(ctx, defaultStore)
	if err != nil {

cockroach/pkg/ccl/backupccl/backupdest/backup_destination.go

Lines 70 to 72 in 37f7574

	func containsManifest(ctx context.Context, exportStore cloud.ExternalStorage) (bool, error) {
	r, err := exportStore.ReadFile(ctx, backupbase.BackupManifestName)
	if err != nil {

cockroach/pkg/cloud/impl_registry.go

Lines 330 to 332 in 37f7574

	func (e *esWrapper) ReadFile(ctx context.Context, basename string) (ioctx.ReadCloserCtx, error) {
	r, err := e.ExternalStorage.ReadFile(ctx, basename)
	if err != nil {

cockroach/pkg/cloud/amazon/s3_storage.go

Lines 751 to 753 in 37f7574

	func (s *s3Storage) ReadFile(ctx context.Context, basename string) (ioctx.ReadCloserCtx, error) {
	reader, _, err := s.ReadFileAt(ctx, basename, 0)
	return reader, err

cockroach/pkg/cloud/amazon/s3_storage.go

Lines 784 to 786 in 37f7574

	// so try a Size() request.
	x, err := s.Size(ctx, basename)
	if err != nil {

cockroach/pkg/cloud/amazon/s3_storage.go

Lines 887 to 889 in 37f7574

	}
	return *out.ContentLength, nil
	}

GOROOT/src/runtime/signal_unix.go#L838-L840
GOROOT/src/runtime/panic.go#L259-L261
GOROOT/src/runtime/panic.go#L883-L885

src/runtime/asm_arm64.s in runtime.goexit at line 1172
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunAsyncTaskEx.func2 at line 470
pkg/jobs/jobs.go in pkg/jobs.(*StartableJob).Start.func2 at line 978
pkg/jobs/adopt.go in pkg/jobs.(*Registry).runJob at line 474
pkg/jobs/registry.go in pkg/jobs.(*Registry).stepThroughStateMachine at line 1663
pkg/jobs/registry.go in pkg/jobs.(*Registry).stepThroughStateMachine.func2 at line 1662
pkg/ccl/backupccl/backup_job.go in pkg/ccl/backupccl.(*backupResumer).Resume at line 519
pkg/ccl/backupccl/backupdest/backup_destination.go in pkg/ccl/backupccl/backupdest.ResolveDest at line 169
pkg/ccl/backupccl/backupdest/backup_destination.go in pkg/ccl/backupccl/backupdest.containsManifest at line 71
pkg/cloud/impl_registry.go in pkg/cloud.(*esWrapper).ReadFile at line 331
pkg/cloud/amazon/s3_storage.go in pkg/cloud/amazon.(*s3Storage).ReadFile at line 752
pkg/cloud/amazon/s3_storage.go in pkg/cloud/amazon.(*s3Storage).ReadFileAt at line 785
pkg/cloud/amazon/s3_storage.go in pkg/cloud/amazon.(*s3Storage).Size at line 888
GOROOT/src/runtime/signal_unix.go in runtime.sigpanic at line 839
GOROOT/src/runtime/panic.go in runtime.panicmem at line 260
GOROOT/src/runtime/panic.go in runtime.gopanic at line 884

Tags

Tag	Value
Command	server
Environment	v23.1.16
Go Version	go1.19.13
Platform	linux arm64
Distribution	CCL
Cockroach Release	v23.1.16
Cockroach SHA	37f7574
# of CPUs	4
# of Goroutines	435

Jira issue: CRDB-36659

[yuzefovich]

dup of #117227 #120389