Skip to content

Commit b85ac51

Browse files
nikolay-egorovnikolay-egorov
committed
Add special type classes in shared-compiler
1 parent 85ae3ff commit b85ac51

File tree

6 files changed

+93
-37
lines changed

6 files changed

+93
-37
lines changed

‎jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt

Lines changed: 40 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,48 @@ data class SerializedCompiledScriptsData(
2020
}
2121
}
2222

23+
@Serializable
24+
data class SerializableTypeInfo(val type: Type = Type.Custom, val isPrimitive: Boolean = false, val fullType: String = "") {
25+
companion object {
26+
val ignoreSet = setOf("int", "double", "boolean", "char", "float", "byte", "string", "entry")
27+
28+
val propertyNamesForNullFilter = setOf("data", "size")
29+
30+
fun makeFromSerializedVariablesState(type: String?, isContainer: Boolean?): SerializableTypeInfo {
31+
val fullType = type.orEmpty()
32+
val enumType = fullType.toTypeEnum()
33+
val isPrimitive = !(
34+
if (fullType != "Entry") (isContainer ?: false)
35+
else true
36+
)
37+
38+
return SerializableTypeInfo(enumType, isPrimitive, fullType)
39+
}
40+
}
41+
}
42+
43+
@Serializable
44+
enum class Type {
45+
Map,
46+
Entry,
47+
Array,
48+
List,
49+
Custom
50+
}
51+
52+
fun String.toTypeEnum(): Type {
53+
return when (this) {
54+
"Map" -> Type.Map
55+
"Entry" -> Type.Entry
56+
"Array" -> Type.Array
57+
"List" -> Type.List
58+
else -> Type.Custom
59+
}
60+
}
61+
2362
@Serializable
2463
data class SerializedVariablesState(
25-
val type: String = "",
64+
val type: SerializableTypeInfo = SerializableTypeInfo(),
2665
val value: String? = null,
2766
val isContainer: Boolean = false,
2867
val stateId: String = ""

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/apiImpl.kt

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,9 @@ class NotebookImpl(
133133

134134
private val history = arrayListOf<CodeCellImpl>()
135135
private var mainCellCreated = false
136-
private val unchangedVariables: MutableSet<String> = mutableSetOf()
136+
private val _unchangedVariables: MutableSet<String> = mutableSetOf()
137137

138+
val unchangedVariables: Set<String> get() = _unchangedVariables
138139
val displays = DisplayContainerImpl()
139140

140141
override fun getAllDisplays(): List<DisplayResultWithCell> {
@@ -153,16 +154,14 @@ class NotebookImpl(
153154
fun updateVariablesState(evaluator: InternalEvaluator) {
154155
variablesState += evaluator.variablesHolder
155156
currentCellVariables = evaluator.cellVariables
156-
unchangedVariables.clear()
157-
unchangedVariables.addAll(evaluator.getUnchangedVariables())
157+
_unchangedVariables.clear()
158+
_unchangedVariables.addAll(evaluator.getUnchangedVariables())
158159
}
159160

160161
fun updateVariablesState(varsStateUpdate: Map<String, VariableState>) {
161162
variablesState += varsStateUpdate
162163
}
163164

164-
fun unchangedVariables(): Set<String> = unchangedVariables
165-
166165
fun variablesReportAsHTML(): String {
167166
return generateHTMLVarsReport(variablesState)
168167
}

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/protocol.kt

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,11 @@ package org.jetbrains.kotlinx.jupyter
22

33
import ch.qos.logback.classic.Level
44
import kotlinx.serialization.json.Json
5+
import kotlinx.serialization.json.JsonElement
6+
import kotlinx.serialization.json.JsonNull
57
import kotlinx.serialization.json.JsonObject
68
import kotlinx.serialization.json.encodeToJsonElement
9+
import kotlinx.serialization.json.jsonObject
710
import org.jetbrains.annotations.TestOnly
811
import org.jetbrains.kotlinx.jupyter.LoggingManagement.disableLogging
912
import org.jetbrains.kotlinx.jupyter.LoggingManagement.mainLoggerLevel
@@ -82,7 +85,6 @@ class OkResponseWithMessage(
8285
)
8386
)
8487
}
85-
8688
socket.send(
8789
makeReplyMessage(
8890
requestMsg,
@@ -92,7 +94,7 @@ class OkResponseWithMessage(
9294
"engine" to Json.encodeToJsonElement(requestMsg.data.header?.session),
9395
"status" to Json.encodeToJsonElement("ok"),
9496
"started" to Json.encodeToJsonElement(startedTime),
95-
"eval_metadata" to Json.encodeToJsonElement(metadata),
97+
"eval_metadata" to Json.encodeToJsonElement(metadata.convertToNullIfEmpty()),
9698
),
9799
content = ExecuteReply(
98100
MessageStatus.OK,
@@ -317,7 +319,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup
317319
if (data.isEmpty()) return sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY))
318320
log.debug("Message data: $data")
319321
val messageContent = getVariablesDescriptorsFromJson(data)
320-
GlobalScope.launch(Dispatchers.Default) {
322+
connection.launchJob {
321323
repl.serializeVariables(
322324
messageContent.topLevelDescriptorName,
323325
messageContent.descriptorsState,
@@ -343,7 +345,7 @@ fun JupyterConnection.Socket.shellMessagesHandler(msg: Message, repl: ReplForJup
343345
}
344346
}
345347
is SerializationRequest -> {
346-
GlobalScope.launch(Dispatchers.Default) {
348+
connection.launchJob {
347349
if (content.topLevelDescriptorName.isNotEmpty()) {
348350
repl.serializeVariables(content.topLevelDescriptorName, content.descriptorsState, commID = content.commId, content.pathToDescriptor) { result ->
349351
sendWrapped(msg, makeReplyMessage(msg, MessageType.SERIALIZATION_REPLY, content = result))
@@ -545,3 +547,8 @@ fun JupyterConnection.evalWithIO(repl: ReplForJupyter, srcMessage: Message, body
545547
KernelStreams.setStreams(false, out, err)
546548
}
547549
}
550+
551+
fun EvaluatedSnippetMetadata?.convertToNullIfEmpty(): JsonElement? {
552+
val jsonNode = Json.encodeToJsonElement(this)
553+
return if (jsonNode is JsonNull || jsonNode?.jsonObject.isEmpty()) null else jsonNode
554+
}

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -429,9 +429,10 @@ class ReplForJupyterImpl(
429429
val newImports: List<String>
430430
val oldDeclarations: MutableMap<String, Int> = mutableMapOf()
431431
oldDeclarations.putAll(internalEvaluator.getVariablesDeclarationInfo())
432+
val jupyterId = evalData.jupyterId
432433
val result = try {
433-
log.debug("Current cell id: ${evalData.jupyterId}")
434-
executor.execute(evalData.code, evalData.displayHandler, currentCellId = evalData.jupyterId - 1) { internalId, codeToExecute ->
434+
log.debug("Current cell id: $jupyterId")
435+
executor.execute(evalData.code, evalData.displayHandler, currentCellId = jupyterId - 1) { internalId, codeToExecute ->
435436
if (evalData.storeHistory) {
436437
cell = notebook.addCell(internalId, codeToExecute, EvalData(evalData))
437438
}
@@ -460,7 +461,7 @@ class ReplForJupyterImpl(
460461
// printVars()
461462
// printUsagesInfo(jupyterId, cellVariables[jupyterId - 1])
462463
val variablesCells: Map<String, Int> = notebook.variablesState.mapValues { internalEvaluator.findVariableCell(it.key) }
463-
val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables())
464+
val serializedData = variablesSerializer.serializeVariables(jupyterId - 1, notebook.variablesState, oldDeclarations, variablesCells, notebook.unchangedVariables)
464465

465466
GlobalScope.launch(Dispatchers.Default) {
466467
variablesSerializer.tryValidateCache(jupyterId - 1, notebook.cellVariables)

‎src/main/kotlin/org/jetbrains/kotlinx/jupyter/serializationUtils.kt

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import kotlinx.serialization.json.Json
55
import kotlinx.serialization.json.JsonObject
66
import kotlinx.serialization.json.decodeFromJsonElement
77
import org.jetbrains.kotlinx.jupyter.api.VariableState
8+
import org.jetbrains.kotlinx.jupyter.compiler.util.SerializableTypeInfo
89
import org.jetbrains.kotlinx.jupyter.compiler.util.SerializedVariablesState
910
import java.lang.reflect.Field
1011
import kotlin.contracts.ExperimentalContracts
@@ -32,14 +33,14 @@ enum class PropertiesType {
3233
}
3334

3435
@Serializable
35-
data class SerializedCommMessageContent(
36+
data class VariablesStateCommMessageContent(
3637
val topLevelDescriptorName: String,
3738
val descriptorsState: Map<String, SerializedVariablesState>,
3839
val pathToDescriptor: List<String> = emptyList()
3940
)
4041

41-
fun getVariablesDescriptorsFromJson(json: JsonObject): SerializedCommMessageContent {
42-
return Json.decodeFromJsonElement<SerializedCommMessageContent>(json)
42+
fun getVariablesDescriptorsFromJson(json: JsonObject): VariablesStateCommMessageContent {
43+
return Json.decodeFromJsonElement<VariablesStateCommMessageContent>(json)
4344
}
4445

4546
class ProcessedSerializedVarsState(
@@ -216,7 +217,12 @@ class VariablesSerializer(
216217
} else {
217218
""
218219
}
219-
val serializedVersion = SerializedVariablesState(simpleTypeName, stringedValue, true, varID)
220+
val serializedVersion = SerializedVariablesState(
221+
SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, true),
222+
stringedValue,
223+
true,
224+
varID
225+
)
220226
val descriptors = serializedVersion.fieldDescriptor
221227

222228
// only for set case
@@ -700,7 +706,12 @@ class VariablesSerializer(
700706
""
701707
}
702708

703-
val serializedVariablesState = SerializedVariablesState(type, getProperString(value), isContainer, finalID)
709+
val serializedVariablesState = SerializedVariablesState(
710+
SerializableTypeInfo.makeFromSerializedVariablesState(simpleTypeName, isContainer),
711+
getProperString(value),
712+
isContainer,
713+
finalID
714+
)
704715

705716
return ProcessedSerializedVarsState(serializedVariablesState, membersProperties?.toTypedArray())
706717
}

‎src/test/kotlin/org/jetbrains/kotlinx/jupyter/test/repl/ReplTests.kt

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -797,14 +797,13 @@ class ReplVarsTest : AbstractSingleReplTest() {
797797
""".trimIndent(),
798798
jupyterId = 1
799799
)
800-
var state = repl.notebook.unchangedVariables()
801-
val res = eval(
800+
eval(
802801
"""
803802
l += 11111
804803
""".trimIndent(),
805804
jupyterId = 2
806805
).metadata.evaluatedVariablesState
807-
state = repl.notebook.unchangedVariables()
806+
val state: Set<String> = repl.notebook.unchangedVariables
808807
assertEquals(1, state.size)
809808
assertTrue(state.contains("m"))
810809
}
@@ -874,7 +873,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
874873
""".trimIndent(),
875874
jupyterId = 1
876875
)
877-
var state = repl.notebook.unchangedVariables()
876+
var state = repl.notebook.unchangedVariables
878877
assertEquals(3, state.size)
879878

880879
eval(
@@ -885,7 +884,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
885884
""".trimIndent(),
886885
jupyterId = 2
887886
)
888-
state = repl.notebook.unchangedVariables()
887+
state = repl.notebook.unchangedVariables
889888
assertEquals(0, state.size)
890889

891890
eval(
@@ -894,7 +893,7 @@ class ReplVarsTest : AbstractSingleReplTest() {
894893
""".trimIndent(),
895894
jupyterId = 3
896895
)
897-
state = repl.notebook.unchangedVariables()
896+
state = repl.notebook.unchangedVariables
898897
assertEquals(1, state.size)
899898
}
900899
}
@@ -930,7 +929,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
930929
assertEquals(listOf(1, 2, 3, 4).toString().substring(1, actualContainer.value!!.length + 1), actualContainer.value)
931930

932931
val serializer = repl.variablesSerializer
933-
val newData = serializer.doIncrementalSerialization(0, "x", "data", actualContainer)
932+
serializer.doIncrementalSerialization(0, "x", "data", actualContainer)
934933
}
935934

936935
@Test
@@ -946,7 +945,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
946945
assertEquals(2, varsData.size)
947946
assertTrue(varsData.containsKey("x"))
948947
assertTrue(varsData.containsKey("f"))
949-
var unchangedVariables = repl.notebook.unchangedVariables()
948+
var unchangedVariables = repl.notebook.unchangedVariables
950949
assertTrue(unchangedVariables.isNotEmpty())
951950

952951
eval(
@@ -955,7 +954,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
955954
""".trimIndent(),
956955
jupyterId = 1
957956
)
958-
unchangedVariables = repl.notebook.unchangedVariables()
957+
unchangedVariables = repl.notebook.unchangedVariables
959958
assertTrue(unchangedVariables.contains("x"))
960959
assertTrue(unchangedVariables.contains("f"))
961960
}
@@ -1019,7 +1018,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
10191018

10201019
val serializer = repl.variablesSerializer
10211020

1022-
val newData = serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!)
1021+
serializer.doIncrementalSerialization(0, "c", "i", descriptor["i"]!!)
10231022
}
10241023

10251024
@Test
@@ -1308,7 +1307,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13081307
""".trimIndent(),
13091308
jupyterId = 1
13101309
)
1311-
val state = repl.notebook.unchangedVariables()
1310+
val state = repl.notebook.unchangedVariables
13121311
val setOfCell = setOf("x", "f", "z")
13131312
assertTrue(state.isNotEmpty())
13141313
assertEquals(setOfCell, state)
@@ -1335,7 +1334,7 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13351334
""".trimIndent(),
13361335
jupyterId = 1
13371336
)
1338-
var state = repl.notebook.unchangedVariables()
1337+
var state = repl.notebook.unchangedVariables
13391338
val setOfCell = setOf("x", "f", "z")
13401339
assertTrue(state.isNotEmpty())
13411340
assertEquals(setOfCell, state)
@@ -1359,9 +1358,9 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13591358
""".trimIndent(),
13601359
jupyterId = 3
13611360
)
1362-
state = repl.notebook.unchangedVariables()
1363-
// assertTrue(state.isNotEmpty())
1364-
// assertEquals(state, setOfPrevCell)
1361+
state = repl.notebook.unchangedVariables
1362+
assertTrue(state.isEmpty())
1363+
// assertEquals(state, setOfPrevCell)
13651364

13661365
eval(
13671366
"""
@@ -1371,20 +1370,20 @@ class ReplVarsSerializationTest : AbstractSingleReplTest() {
13711370
""".trimIndent(),
13721371
jupyterId = 4
13731372
)
1374-
state = repl.notebook.unchangedVariables()
1373+
state = repl.notebook.unchangedVariables
13751374
assertTrue(state.isEmpty())
13761375
}
13771376

13781377
@Test
13791378
fun testSerializationClearInfo() {
1380-
var res = eval(
1379+
eval(
13811380
"""
13821381
val x = listOf(1, 2, 3, 4)
13831382
""".trimIndent(),
13841383
jupyterId = 1
13851384
).metadata.evaluatedVariablesState
1386-
var state = repl.notebook.unchangedVariables()
1387-
res = eval(
1385+
repl.notebook.unchangedVariables
1386+
eval(
13881387
"""
13891388
val x = listOf(1, 2, 3, 4)
13901389
""".trimIndent(),

0 commit comments

Comments
 (0)