Merge pull request #521 from s22s/feature/test-max-failures

vpipkt · web-flow · commit 68ebe65b25a9 · 2020-11-05T13:27:27.000-05:00
Set spark master with spark.task.maxFailures in test
diff --git a/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala b/core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala
@@ -50,7 +50,8 @@ trait TestEnvironment extends FunSpec
     outputDir
   }
 
-  def sparkMaster: String = "local[*]"
+  // allow 2 retries, should stabilize CI builds. https://spark.apache.org/docs/2.4.7/submitting-applications.html#master-urls
+  def sparkMaster: String = "local[*, 2]"
 
   def additionalConf = new SparkConf(false)
 
diff --git a/pyrasterframes/src/main/python/tests/__init__.py b/pyrasterframes/src/main/python/tests/__init__.py
@@ -61,6 +61,7 @@ def pdir(curr):
 
 def spark_test_session():
     spark = create_rf_spark_session(**{
+        'spark.master': 'local[*, 2]',
         'spark.ui.enabled': 'false',
         'spark.app.name': app_name
     })

Original file line number	Diff line number	Diff line change
`@@ -50,7 +50,8 @@ trait TestEnvironment extends FunSpec`
`50`	`50`	`outputDir`
`51`	`51`	`}`
`52`	`52`
`53`		`- def sparkMaster: String = "local[*]"`
	`53`	`+ // allow 2 retries, should stabilize CI builds. https://spark.apache.org/docs/2.4.7/submitting-applications.html#master-urls`
	`54`	`+ def sparkMaster: String = "local[*, 2]"`
`54`	`55`
`55`	`56`	`def additionalConf = new SparkConf(false)`
`56`	`57`