OCPCRT-490 - Improve launch notification sequence for custom builds

pkg/manager/manager.go

@@ -2336,6 +2336,9 @@ func (m *jobManager) LaunchJobForUser(req *JobRequest) (string, error) {
 		} else {
 			msg = fmt.Sprintf("%s - I'll send you the credentials when the cluster is ready.", msg)
 		}
+		if jobHasRefs(job) {
+			msg = fmt.Sprintf("%s\n\nNote: your launch includes custom PR builds, which typically add 20-40 minutes to launch time. Total estimated time is up to ~90 minutes.", msg)
+		}
 		return "", errors.New(msg)
 	}
 	return "", fmt.Errorf("%s<%s|job> started, you will be notified on completion", msg, prowJobUrl)
@@ -2450,6 +2453,20 @@ func (m *jobManager) jobIsComplete(job *Job) bool {
 	return false
 }
 
+func (m *jobManager) prowJobIsStillRunning(name string) bool {
+	pj, err := m.prowClient.ProwJobs(m.prowNamespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil {
+		klog.Warningf("Unable to check ProwJob %q state: %v", name, err)
+		return false
+	}
+	switch pj.Status.State {
+	case prowapiv1.AbortedState, prowapiv1.ErrorState, prowapiv1.FailureState, prowapiv1.SuccessState:
+		return false
+	default:
+		return true
+	}
+}
+
 func (m *jobManager) handleJobStartup(job Job, source string) {
 	if !m.tryJob(job.Name) {
 		klog.Infof("Job %q already has a worker (%s)", job.Name, source)
@@ -2463,6 +2480,9 @@ func (m *jobManager) handleJobStartup(job Job, source string) {
 		} else {
 			if strings.HasPrefix(err.Error(), "timed out waiting for your prowjob") {
 				klog.Errorf("Job %q timed out waiting for prowjob to start (%s): %v", job.Name, source, err)
+			} else if strings.HasPrefix(err.Error(), "cluster never became available") && m.prowJobIsStillRunning(job.Name) {
+				klog.Warningf("Job %q monitoring window expired but prowjob is still running (%s): %v -- deferring to sync loop", job.Name, source, err)
+				return
 			} else {
 				klog.Errorf("Job %q failed to launch (%s): %v", job.Name, source, err)
 				job.Failure = err.Error()

pkg/manager/prow.go

@@ -181,6 +181,15 @@ var (
 	reVersion = regexp.MustCompile(`^(\d+\.\d+)`)
 )
 
+func jobHasRefs(job *Job) bool {
+	for _, input := range job.Inputs {
+		if len(input.Refs) > 0 {
+			return true
+		}
+	}
+	return false
+}
+
 func mceReleaseName(job *Job) string {
 	if len(job.Inputs) == 0 {
 		return ""
@@ -643,12 +652,7 @@ func (m *jobManager) newJob(job *Job) (string, error) {
 	}
 	sourceConfig.ReleaseTagConfiguration = nil
 
-	var hasRefs bool
-	for _, input := range job.Inputs {
-		if len(input.Refs) > 0 {
-			hasRefs = true
-		}
-	}
+	hasRefs := jobHasRefs(job)
 	var mceReleaseVersion string
 	if job.Mode == JobTypeMCECustomImage {
 		mceReleaseVersion = mceReleaseName(job)
@@ -1223,6 +1227,9 @@ func (m *jobManager) waitForJob(job *Job) error {
 	} else if job.Operator.Is {
 		setupContainerTimeout = 105 * time.Minute
 	}
+	if jobHasRefs(job) {
+		setupContainerTimeout += 30 * time.Minute
+	}
 
 	if job.Mode != JobTypeLaunch && job.Mode != JobTypeWorkflowLaunch {
 		klog.Infof("Job %s will report results at %s (to %s / %s)", job.Name, job.URL, job.RequestedBy, job.RequestedChannel)