Skip to content

Commit c80ae0c

Browse files
mgyuchtclaude
andcommitted
Fix selectSparkVersion() to use contains() instead of equals()
The current implementation of selectSparkVersion() was broken when sparkVersion is supplied. The code was requiring an exact match instead of using contains(), which never matched real Databricks Runtime version names like "12.2 LTS (includes Apache Spark 3.3.2, Scala 2.12)". This change aligns the Java SDK with the Go and Python SDK implementations, which both use contains()/in operations for sparkVersion matching. Fixes the issue described in PR #229. Added comprehensive unit tests covering: - Successful spark version matching with realistic API response data - Multiple matches with latest=true/false behavior - Integration with other selector parameters (ML, etc.) - Error cases for non-existent versions 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent 3836135 commit c80ae0c

2 files changed

Lines changed: 84 additions & 1 deletion

File tree

databricks-sdk-java/src/main/java/com/databricks/sdk/mixin/ClustersExt.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public String selectSparkVersion(SparkVersionSelector selector) throws IllegalAr
5050
matches = version.getName().contains("LTS") || version.getKey().contains("-esr-");
5151
}
5252
if (matches && selector.sparkVersion != null) {
53-
matches = ("Apache Spark " + selector.sparkVersion).equals(version.getName());
53+
matches = version.getName().contains("Apache Spark " + selector.sparkVersion);
5454
}
5555
if (matches) {
5656
versions.add(version.getKey());

databricks-sdk-java/src/test/java/com/databricks/sdk/mixin/ClustersExtTest.java

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.databricks.sdk.mixin;
22

33
import static org.junit.jupiter.api.Assertions.assertEquals;
4+
import static org.junit.jupiter.api.Assertions.assertThrows;
45

56
import com.databricks.sdk.core.ApiClient;
67
import com.databricks.sdk.core.DatabricksConfig;
@@ -118,4 +119,86 @@ void nullComparisonTest() {
118119
String nodeType = clustersExt.selectNodeType(new NodeTypeSelector().withLocalDisk());
119120
assertEquals("testId1", nodeType);
120121
}
122+
123+
private GetSparkVersionsResponse testGetSparkVersionsWithSparkVersion() {
124+
Collection<SparkVersion> versions = new ArrayList<>();
125+
// Mock realistic Databricks Runtime versions based on actual API response format
126+
versions.add(new SparkVersion()
127+
.setName("12.2 LTS (includes Apache Spark 3.3.2, Scala 2.12)")
128+
.setKey("12.2.x-scala2.12"));
129+
versions.add(new SparkVersion()
130+
.setName("13.3 LTS (includes Apache Spark 3.4.1, Scala 2.12)")
131+
.setKey("13.3.x-scala2.12"));
132+
versions.add(new SparkVersion()
133+
.setName("14.3 LTS (includes Apache Spark 3.5.0, Scala 2.12)")
134+
.setKey("14.3.x-scala2.12"));
135+
versions.add(new SparkVersion()
136+
.setName("14.2 ML (includes Apache Spark 3.5.0, Scala 2.12)")
137+
.setKey("14.2.x-cpu-ml-scala2.12"));
138+
// Add another version with same Spark version to create multiple matches
139+
versions.add(new SparkVersion()
140+
.setName("14.1 (includes Apache Spark 3.5.0, Scala 2.12)")
141+
.setKey("14.1.x-scala2.12"));
142+
return new GetSparkVersionsResponse().setVersions(versions);
143+
}
144+
145+
@Test
146+
void sparkVersionWithSparkVersionParameter() {
147+
ClustersExt clustersExt = new ClustersExt(clustersMock);
148+
Mockito.doReturn(testGetSparkVersionsWithSparkVersion()).when(clustersMock).sparkVersions();
149+
150+
// Test exact spark version match
151+
String sparkVersion = clustersExt.selectSparkVersion(
152+
new SparkVersionSelector().withSparkVersion("3.4.1"));
153+
assertEquals("13.3.x-scala2.12", sparkVersion);
154+
}
155+
156+
@Test
157+
void sparkVersionWithSparkVersionParameterMultipleMatches() {
158+
ClustersExt clustersExt = new ClustersExt(clustersMock);
159+
Mockito.doReturn(testGetSparkVersionsWithSparkVersion()).when(clustersMock).sparkVersions();
160+
161+
// Test spark version with multiple matches - should return latest when latest=true is explicitly set
162+
String sparkVersion = clustersExt.selectSparkVersion(
163+
new SparkVersionSelector().withSparkVersion("3.5.0").withLatest());
164+
// Should return the highest version (14.3.x) when latest=true and multiple 3.5.0 versions match
165+
assertEquals("14.3.x-scala2.12", sparkVersion);
166+
}
167+
168+
@Test
169+
void sparkVersionWithSparkVersionParameterAndML() {
170+
ClustersExt clustersExt = new ClustersExt(clustersMock);
171+
Mockito.doReturn(testGetSparkVersionsWithSparkVersion()).when(clustersMock).sparkVersions();
172+
173+
// Test spark version combined with ML requirement
174+
String sparkVersion = clustersExt.selectSparkVersion(
175+
new SparkVersionSelector().withSparkVersion("3.5.0").withML());
176+
assertEquals("14.2.x-cpu-ml-scala2.12", sparkVersion);
177+
}
178+
179+
@Test
180+
void sparkVersionWithSparkVersionParameterNoMatch() {
181+
ClustersExt clustersExt = new ClustersExt(clustersMock);
182+
Mockito.doReturn(testGetSparkVersionsWithSparkVersion()).when(clustersMock).sparkVersions();
183+
184+
// Test spark version that doesn't exist
185+
assertThrows(IllegalArgumentException.class, () -> {
186+
clustersExt.selectSparkVersion(
187+
new SparkVersionSelector().withSparkVersion("2.4.5"));
188+
});
189+
}
190+
191+
@Test
192+
void sparkVersionWithSparkVersionParameterMultipleMatchesLatestFalse() {
193+
ClustersExt clustersExt = new ClustersExt(clustersMock);
194+
Mockito.doReturn(testGetSparkVersionsWithSparkVersion()).when(clustersMock).sparkVersions();
195+
196+
// Test spark version with multiple matches and latest=false (default) - should throw exception
197+
SparkVersionSelector selector = new SparkVersionSelector().withSparkVersion("3.5.0");
198+
// latest defaults to false, so multiple matches should throw an exception
199+
200+
assertThrows(IllegalArgumentException.class, () -> {
201+
clustersExt.selectSparkVersion(selector);
202+
}, "Expected exception when multiple versions match with latest=false (default)");
203+
}
121204
}

0 commit comments

Comments
 (0)