Downgraded to java 1.8.

Removed the plugin module.
Added one more example.
Updated README

Author: mahen-n

README.md

@@ -8,21 +8,67 @@ A Spark bootstrap project written in Scala with gradle as build tool.
 - [Gradle](https://gradle.org/)
 - [Scala](https://www.scala-lang.org/)
 
+## Libraries Included
+
+- JavaVersion=1.8
+- sparkVersion=3.4.1
+- scalaVersion=2.12
+- deltaVersion=2.4.0
+
 ### Build
 
 `./gradlew clean build`
 
+### Test
+
+	./gradlew check
+
+### Run TestCoverage
+
+	./gradlew reportTestScoverage
+
+## Run
+
+#### Run sparkSubmit task
+
+- Runs a `spark-submit` with class `dev.template.spark.RddCollect`
+
+./gradlew sparkSubmit
+
+#### Spark Submit commands in shell
+
+> A local spark instance must be up and running: http://localhost:8080/
+
+	spark-3.4.1-bin-hadoop3/bin/spark-submit \
+		--verbose  --class dev.template.spark.Main \
+		--packages io.delta:delta-core_2.12:2.4.0 \
+		--master spark://localhost:7077 \
+		--driver-memory 1g \
+		--executor-memory 1g \
+		--executor-cores 2 \
+		build/libs/spark-scala-gradle-bootstrap-2.12.0-all.jar
+
+	spark-3.4.1-bin-hadoop3/bin/spark-submit --class dev.template.spark.RddCollect \
+		--master spark://localhost:7077 \
+		build/libs/spark-scala-gradle-bootstrap-2.12.0-all.jar
+
+	spark-3.4.1-bin-hadoop3/bin/spark-submit --class dev.template.spark.CovidDataPartitioner \
+		--packages io.delta:delta-core_2.12:2.4.0 \
+		--master spark://localhost:7077 \
+		--driver-memory 1g \
+		--executor-memory 1g \
+		--executor-cores 2 \
+		build/libs/spark-scala-gradle-bootstrap-2.12.0-all.jar \
+		src/main/resources/us-counties-recent.csv \
+		/tmp/partitioned-covid-data
+
 ### Coverage
 
 https://github.com/scoverage/gradle-scoverage
 
 ## Functional Test Examples
 
-https://github.com/scoverage/gradle-scoverage/blob/master/build.gradle
-
-## Libraries Included
-
-- Spark - 3.4.1
+https://github.com/scoverage/gradle-scoverage/blob/master/build.gradle#L59C1-L59C52
 
 ## Useful Links
 
@@ -34,11 +80,12 @@ https://github.com/scoverage/gradle-scoverage/blob/master/build.gradle
 
 ## Issues or Suggestions
 
+https://github.com/mahen-github/spark-scala-gradle-bootstrap/issues
+
 # Learn Spark
 
 https://www.databricks.com/wp-content/uploads/2021/06/Ebook_8-Steps-V2.pdf
 
 # References
 
 https://github.com/spark-examples/spark-scala-examples
-

build.gradle

@@ -1,20 +1,13 @@
-//buildscript {
-//    repositories {
-//        mavenCentral()
-//    }
-//}
 plugins {
-    id 'scala'
-    id 'application'
-    id "com.github.spotbugs-base"
-    id "com.diffplug.spotless"
-    id 'net.nemerosa.versioning'
-    id 'com.jfrog.artifactory'
-    id 'org.scoverage'
-    id 'template.spark.livyPlugin'
+	id 'scala'
+	id 'application'
+	id "com.github.spotbugs-base"
+	id "com.diffplug.spotless"
+	id 'net.nemerosa.versioning'
+	id 'com.jfrog.artifactory'
+	id 'org.scoverage'
 }
 
-//apply plugin: LivyPlugin
 apply plugin: 'java'
 apply plugin: 'scala'
 apply plugin: 'idea'
@@ -31,85 +24,81 @@ sourceCompatibility = JavaVersion.VERSION_1_8
 targetCompatibility = JavaVersion.VERSION_1_8
 
 configurations {
-    provided
+	provided
 }
 
 sourceSets {
-    main {
-        compileClasspath += configurations.provided
-    }
+	main {
+		compileClasspath += configurations.provided
+	}
 }
 
 application {
-    mainClassName = 'dev.template.spark.Main'
+	mainClassName = 'dev.template.spark.Main'
 }
 
 
 repositories {
-    mavenCentral()
+	mavenCentral()
 }
 
 dependencies {
-    implementation "org.scalameta:scalafmt-core_${scalaVersion}:${scalafmt}"
-    implementation "org.apache.spark:spark-sql_${scalaVersion}:${sparkVersion}"
-    implementation "org.apache.spark:spark-graphx_${scalaVersion}:${sparkVersion}"
-    implementation "org.apache.spark:spark-launcher_${scalaVersion}:${sparkVersion}"
-    implementation "org.apache.spark:spark-catalyst_${scalaVersion}:${sparkVersion}"
-    implementation "org.apache.spark:spark-streaming_${scalaVersion}:${sparkVersion}"
-    implementation "org.apache.spark:spark-core_${scalaVersion}:${sparkVersion}"
-    implementation "commons-io:commons-io:${commonsIO}"
-
-    implementation "org.apache.hadoop:hadoop-aws:${hadoopAWS}"
-    implementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}"
-    implementation "io.delta:delta-core_${scalaVersion}:${deltaVersion}"
-    implementation "com.google.guava:guava:31.1-jre"
-    compileOnly "org.scala-lang:scala-library:$scalaVersion"
-    compileOnly "org.scala-lang:scala-compiler:${scalaVersion}"
-
-    testImplementation "org.scalatestplus:junit-4-13_${scalaVersion}:3.2.2.0"
-    testImplementation "junit:junit:${junitVersion}"
-    testRuntimeOnly "org.scala-lang.modules:scala-xml_${scalaVersion}:1.2.0"
-    testImplementation 'org.mockito:mockito-core:5.3.1'
-
-    testImplementation "org.junit.jupiter:junit-jupiter-api:${jupiterApi}"
-    testImplementation "org.scalatest:scalatest_${scalaVersion}:${scalaTests}"
+	implementation "org.scalameta:scalafmt-core_${scalaVersion}:${scalafmt}"
+	implementation "org.apache.spark:spark-sql_${scalaVersion}:${sparkVersion}"
+	implementation "org.apache.spark:spark-graphx_${scalaVersion}:${sparkVersion}"
+	implementation "org.apache.spark:spark-launcher_${scalaVersion}:${sparkVersion}"
+	implementation "org.apache.spark:spark-catalyst_${scalaVersion}:${sparkVersion}"
+	implementation "org.apache.spark:spark-streaming_${scalaVersion}:${sparkVersion}"
+	implementation "org.apache.spark:spark-core_${scalaVersion}:${sparkVersion}"
+
+	implementation "org.apache.hadoop:hadoop-aws:${hadoopAWS}"
+	implementation "org.apache.spark:spark-hive_${scalaVersion}:${sparkVersion}"
+	implementation "io.delta:delta-core_${scalaVersion}:${deltaVersion}"
+	compileOnly "org.scala-lang:scala-library:$scalaVersion"
+	compileOnly "org.scala-lang:scala-compiler:${scalaVersion}"
+
+	testImplementation "org.scalatestplus:junit-4-13_${scalaVersion}:3.2.2.0"
+	testImplementation "junit:junit:${junitVersion}"
+	testRuntimeOnly "org.scala-lang.modules:scala-xml_${scalaVersion}:1.2.0"
+	testImplementation 'org.mockito:mockito-core:5.3.1'
+
+	testImplementation "org.junit.jupiter:junit-jupiter-api:${jupiterApi}"
+	testImplementation "org.scalatest:scalatest_${scalaVersion}:${scalaTests}"
 }
 
 
 jar {
-    classifier 'all'
-    manifest {
-        attributes 'Implementation-Title': title,
-                'Implementation-Version': archiveVersion,
-                'Main-Class': mainClassFile
-    }
-    exclude 'META-INF/*.RSA', 'META-INF/*.SF', 'META-INF/*.DSA'
-    from files(sourceSets.main.output.classesDirs)
-    zip64 true
+	classifier 'all'
+	manifest {
+		attributes 'Implementation-Title': title,
+				'Implementation-Version': archiveVersion,
+				'Main-Class': mainClassFile
+	}
+	exclude 'META-INF/*.RSA', 'META-INF/*.SF', 'META-INF/*.DSA'
+	from files(sourceSets.main.output.classesDirs)
+	zip64 true
 }
 
 
 tasks.register('scalaTest', JavaExec) {
-    dependsOn['testClasses']
-    mainClass.set("org.scalatest.tools.Runner")
-    args = ['-R', 'build/classes/scala/test', '-o']
-    classpath = sourceSets.test.runtimeClasspath
+	dependsOn['testClasses']
+	mainClass.set("org.scalatest.tools.Runner")
+	args = ['-R', 'build/classes/scala/test', '-o']
+	classpath = sourceSets.test.runtimeClasspath
 }
 
 test.dependsOn scalaTest
 
 idea {
-    module {
-        // IntelliJ does not know about the standard idiom of provided as used in managing
-        // uber/shaded jar dependencies. Make it so!
-        scopes.PROVIDED.plus += [configurations.provided]
-    }
+	module {
+		// IntelliJ does not know about the standard idiom of provided as used in managing
+		// uber/shaded jar dependencies. Make it so!
+		scopes.PROVIDED.plus += [configurations.provided]
+	}
 }
 
-def runLivy = tasks.register("runLivy", template.spark.Publish) {
-    def avscTree = project.fileTree(allAvscs)
-
-    targetFiles.from(avscTree)
-    schemaFile.value(layout.projectDirectory.file("validator/avro-schema.json"))
+task sparkSubmit(type: Exec) {
+	commandLine 'sh', '-c', "/Users/e1xx/spark-3.4.1-bin-hadoop3/bin/spark-submit " +
+			"--class dev.template.spark.RddCollect" +
+			" --master spark://localhost:7077 build/libs/spark-scala-gradle-bootstrap-2.12.0-all.jar"
 }
-

gradle.properties

@@ -1,6 +1,6 @@
 sparkVersion=3.4.1
 scalaVersion=2.12
-hadoopAWS=3.3.3
+hadoopAWS=3.3.4
 commonsIO=2.13.0
 deltaVersion=2.4.0
 #kafka
@@ -21,3 +21,5 @@ mainClassFile=dev.template.spark.Main
 title='spark-scala-gradle-bootstrap'
 group='template.spark'
 version='1.0.0.SNAPSHOT'
+#gradle run time config
+org.gradle.jvmargs=-Xmx2048m

gradle/spotless.gradle

@@ -14,7 +14,7 @@ spotless {
 		endWithNewline()
 	}
 	java {
-		googleJavaFormat('1.11.0')
+		googleJavaFormat('1.7')
 	}
 }
 

settings.gradle

@@ -1,42 +1,37 @@
 pluginManagement {
-    repositories {
-        mavenCentral()
-        maven {
-            url = uri("https://plugins.gradle.org/m2/")
-        }
-    }
-    plugins {
-        id 'com.jfrog.artifactory' version '4.29.0'
-//        id 'ch.ergon.gradle.goodies.versioning' version '1.0.0'
-        id 'com.github.johnrengelman.shadow' version "7.1.2"
-        id 'com.github.spotbugs-base' version '5.1.3'
-        id 'com.diffplug.spotless' version '6.10.0'
-        id 'net.nemerosa.versioning' version '2.15.1'
-//		id 'org.scoverage' version '4.0.0'
-        id("org.scoverage") version "8.0.3"
-
-        includeBuild("plugin")
-    }
+	repositories {
+		mavenCentral()
+		maven {
+			url = uri("https://plugins.gradle.org/m2/")
+		}
+	}
+	plugins {
+		id 'com.jfrog.artifactory' version '4.29.0'
+		id 'com.github.johnrengelman.shadow' version "7.1.2"
+		id 'com.github.spotbugs-base' version '5.1.3'
+		id 'com.diffplug.spotless' version '6.10.0'
+		id 'net.nemerosa.versioning' version '2.15.1'
+		id 'org.scoverage' version '8.0.3'
+	}
 }
 
 dependencyResolutionManagement {
-    repositories {
-        mavenCentral()
-        maven {
-            url = uri("https://plugins.gradle.org/m2/")
-        }
-    }
+	repositories {
+		mavenCentral()
+		maven {
+			url = uri("https://plugins.gradle.org/m2/")
+		}
+	}
 }
 
 buildscript {
-    repositories {
-        mavenCentral()
-        maven {
-            url = uri("https://plugins.gradle.org/m2/")
-        }
-    }
+	repositories {
+		mavenCentral()
+		maven {
+			url = uri("https://plugins.gradle.org/m2/")
+		}
+	}
 }
 
 rootProject.name = 'spark-scala-gradle-bootstrap'
-//include 'plugin'
-assert gradle.gradleVersion =~ "7..+"
+assert gradle.gradleVersion =~ "7..+"