diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 602f7de27..6d1044700 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -2336,6 +2336,9 @@ func (m *jobManager) LaunchJobForUser(req *JobRequest) (string, error) { } else { msg = fmt.Sprintf("%s - I'll send you the credentials when the cluster is ready.", msg) } + if jobHasRefs(job) { + msg = fmt.Sprintf("%s\n\nNote: your launch includes custom PR builds, which typically add 20-40 minutes to launch time. Total estimated time is up to ~90 minutes.", msg) + } return "", errors.New(msg) } return "", fmt.Errorf("%s<%s|job> started, you will be notified on completion", msg, prowJobUrl) @@ -2450,6 +2453,20 @@ func (m *jobManager) jobIsComplete(job *Job) bool { return false } +func (m *jobManager) prowJobIsStillRunning(name string) bool { + pj, err := m.prowClient.ProwJobs(m.prowNamespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + klog.Warningf("Unable to check ProwJob %q state: %v", name, err) + return false + } + switch pj.Status.State { + case prowapiv1.AbortedState, prowapiv1.ErrorState, prowapiv1.FailureState, prowapiv1.SuccessState: + return false + default: + return true + } +} + func (m *jobManager) handleJobStartup(job Job, source string) { if !m.tryJob(job.Name) { klog.Infof("Job %q already has a worker (%s)", job.Name, source) @@ -2463,6 +2480,9 @@ func (m *jobManager) handleJobStartup(job Job, source string) { } else { if strings.HasPrefix(err.Error(), "timed out waiting for your prowjob") { klog.Errorf("Job %q timed out waiting for prowjob to start (%s): %v", job.Name, source, err) + } else if strings.HasPrefix(err.Error(), "cluster never became available") && m.prowJobIsStillRunning(job.Name) { + klog.Warningf("Job %q monitoring window expired but prowjob is still running (%s): %v -- deferring to sync loop", job.Name, source, err) + return } else { klog.Errorf("Job %q failed to launch (%s): %v", job.Name, source, err) job.Failure = err.Error() diff --git a/pkg/manager/prow.go b/pkg/manager/prow.go index ed4dd6d7e..0c1a415f0 100644 --- a/pkg/manager/prow.go +++ b/pkg/manager/prow.go @@ -181,6 +181,15 @@ var ( reVersion = regexp.MustCompile(`^(\d+\.\d+)`) ) +func jobHasRefs(job *Job) bool { + for _, input := range job.Inputs { + if len(input.Refs) > 0 { + return true + } + } + return false +} + func mceReleaseName(job *Job) string { if len(job.Inputs) == 0 { return "" @@ -643,12 +652,7 @@ func (m *jobManager) newJob(job *Job) (string, error) { } sourceConfig.ReleaseTagConfiguration = nil - var hasRefs bool - for _, input := range job.Inputs { - if len(input.Refs) > 0 { - hasRefs = true - } - } + hasRefs := jobHasRefs(job) var mceReleaseVersion string if job.Mode == JobTypeMCECustomImage { mceReleaseVersion = mceReleaseName(job) @@ -1223,6 +1227,9 @@ func (m *jobManager) waitForJob(job *Job) error { } else if job.Operator.Is { setupContainerTimeout = 105 * time.Minute } + if jobHasRefs(job) { + setupContainerTimeout += 30 * time.Minute + } if job.Mode != JobTypeLaunch && job.Mode != JobTypeWorkflowLaunch { klog.Infof("Job %s will report results at %s (to %s / %s)", job.Name, job.URL, job.RequestedBy, job.RequestedChannel)