diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt index 232518cda..dd48b739f 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/CodeWithConverter.kt @@ -1,7 +1,7 @@ package org.jetbrains.kotlinx.dataframe.codeGen -public typealias Code = String -public typealias VariableName = String +import org.jetbrains.kotlinx.jupyter.api.Code +import org.jetbrains.kotlinx.jupyter.api.VariableName /** * Class representing generated code declarations for a [Marker]. diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt index 01f47fcbb..63ceb4ffd 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/codeGen/ReplCodeGenerator.kt @@ -2,9 +2,9 @@ package org.jetbrains.dataframe.impl.codeGen import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.AnyRow -import org.jetbrains.kotlinx.dataframe.codeGen.Code import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.impl.codeGen.ReplCodeGeneratorImpl +import org.jetbrains.kotlinx.jupyter.api.Code import kotlin.reflect.KClass import kotlin.reflect.KProperty diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt index 0a0aca8ef..dc1d94f6a 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/CodeGeneratorImpl.kt @@ -19,7 +19,6 @@ import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.api.ColumnSelectionDsl import org.jetbrains.kotlinx.dataframe.api.DataSchemaEnum import org.jetbrains.kotlinx.dataframe.codeGen.BaseField -import org.jetbrains.kotlinx.dataframe.codeGen.Code import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod import org.jetbrains.kotlinx.dataframe.codeGen.ExtensionsCodeGenerator @@ -34,6 +33,7 @@ import org.jetbrains.kotlinx.dataframe.codeGen.toNullable import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup import org.jetbrains.kotlinx.dataframe.impl.toSnakeCase import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema +import org.jetbrains.kotlinx.jupyter.api.Code private fun renderNullability(nullable: Boolean) = if (nullable) "?" else "" diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt index aa64903c5..763ac1c87 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/ReplCodeGeneratorImpl.kt @@ -9,12 +9,12 @@ import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.DataRow import org.jetbrains.kotlinx.dataframe.annotations.DataSchema import org.jetbrains.kotlinx.dataframe.api.schema -import org.jetbrains.kotlinx.dataframe.codeGen.Code import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter import org.jetbrains.kotlinx.dataframe.codeGen.Marker import org.jetbrains.kotlinx.dataframe.codeGen.MarkerVisibility import org.jetbrains.kotlinx.dataframe.codeGen.MarkersExtractor import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema +import org.jetbrains.kotlinx.jupyter.api.Code import kotlin.reflect.KClass import kotlin.reflect.KProperty import kotlin.reflect.KType diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/SchemaReader.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/SchemaReader.kt index 026cb3df1..b5ed79216 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/SchemaReader.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/codeGen/SchemaReader.kt @@ -4,7 +4,6 @@ import org.jetbrains.dataframe.impl.codeGen.CodeGenerator import org.jetbrains.kotlinx.dataframe.AnyFrame import org.jetbrains.kotlinx.dataframe.DataFrame import org.jetbrains.kotlinx.dataframe.api.schema -import org.jetbrains.kotlinx.dataframe.codeGen.Code import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod import org.jetbrains.kotlinx.dataframe.io.SupportedCodeGenerationFormat import org.jetbrains.kotlinx.dataframe.io.SupportedDataFrameFormat @@ -13,6 +12,7 @@ import org.jetbrains.kotlinx.dataframe.io.guessFormat import org.jetbrains.kotlinx.dataframe.io.read import org.jetbrains.kotlinx.dataframe.io.readCodeForGeneration import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema +import org.jetbrains.kotlinx.jupyter.api.Code import java.net.URL /** diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/guess.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/guess.kt index 2250c9134..428254dcc 100644 --- a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/guess.kt +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/guess.kt @@ -11,8 +11,8 @@ import org.jetbrains.kotlinx.dataframe.annotations.ImportDataSchema import org.jetbrains.kotlinx.dataframe.annotations.Interpretable import org.jetbrains.kotlinx.dataframe.annotations.OptInRefine import org.jetbrains.kotlinx.dataframe.api.single -import org.jetbrains.kotlinx.dataframe.codeGen.Code import org.jetbrains.kotlinx.dataframe.codeGen.DefaultReadDfMethod +import org.jetbrains.kotlinx.jupyter.api.Code import java.io.BufferedInputStream import java.io.File import java.io.FileNotFoundException diff --git a/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt new file mode 100644 index 000000000..af4ebd902 --- /dev/null +++ b/core/generated-sources/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt @@ -0,0 +1,338 @@ +package org.jetbrains.kotlinx.dataframe.jupyter + +import org.jetbrains.dataframe.impl.codeGen.CodeGenerator +import org.jetbrains.dataframe.impl.codeGen.ReplCodeGenerator +import org.jetbrains.kotlinx.dataframe.AnyCol +import org.jetbrains.kotlinx.dataframe.AnyFrame +import org.jetbrains.kotlinx.dataframe.AnyRow +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.DataRow +import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.Convert +import org.jetbrains.kotlinx.dataframe.api.FormatClause +import org.jetbrains.kotlinx.dataframe.api.FormattedFrame +import org.jetbrains.kotlinx.dataframe.api.Gather +import org.jetbrains.kotlinx.dataframe.api.GroupBy +import org.jetbrains.kotlinx.dataframe.api.GroupClause +import org.jetbrains.kotlinx.dataframe.api.InsertClause +import org.jetbrains.kotlinx.dataframe.api.Merge +import org.jetbrains.kotlinx.dataframe.api.MoveClause +import org.jetbrains.kotlinx.dataframe.api.Pivot +import org.jetbrains.kotlinx.dataframe.api.PivotGroupBy +import org.jetbrains.kotlinx.dataframe.api.ReducedGroupBy +import org.jetbrains.kotlinx.dataframe.api.ReducedPivot +import org.jetbrains.kotlinx.dataframe.api.ReducedPivotGroupBy +import org.jetbrains.kotlinx.dataframe.api.RenameClause +import org.jetbrains.kotlinx.dataframe.api.ReplaceClause +import org.jetbrains.kotlinx.dataframe.api.Split +import org.jetbrains.kotlinx.dataframe.api.SplitWithTransform +import org.jetbrains.kotlinx.dataframe.api.Update +import org.jetbrains.kotlinx.dataframe.api.asColumnGroup +import org.jetbrains.kotlinx.dataframe.api.asDataFrame +import org.jetbrains.kotlinx.dataframe.api.columnsCount +import org.jetbrains.kotlinx.dataframe.api.isColumnGroup +import org.jetbrains.kotlinx.dataframe.api.name +import org.jetbrains.kotlinx.dataframe.codeGen.CodeWithConverter +import org.jetbrains.kotlinx.dataframe.columns.ColumnGroup +import org.jetbrains.kotlinx.dataframe.columns.ColumnReference +import org.jetbrains.kotlinx.dataframe.dataTypes.IFRAME +import org.jetbrains.kotlinx.dataframe.dataTypes.IMG +import org.jetbrains.kotlinx.dataframe.impl.codeGen.CodeGenerationReadResult +import org.jetbrains.kotlinx.dataframe.impl.codeGen.urlCodeGenReader +import org.jetbrains.kotlinx.dataframe.impl.createStarProjectedType +import org.jetbrains.kotlinx.dataframe.impl.renderType +import org.jetbrains.kotlinx.dataframe.io.DataFrameHtmlData +import org.jetbrains.kotlinx.dataframe.io.SupportedCodeGenerationFormat +import org.jetbrains.kotlinx.dataframe.io.supportedFormats +import org.jetbrains.kotlinx.jupyter.api.* +import org.jetbrains.kotlinx.jupyter.api.libraries.* +import kotlin.reflect.KClass +import kotlin.reflect.KProperty +import kotlin.reflect.KType +import kotlin.reflect.full.isSubtypeOf + +/** Users will get an error if their Kotlin Jupyter kernel is older than this version. */ +private const val MIN_KERNEL_VERSION = "0.11.0.198" + +internal val newDataSchemas = mutableListOf>() + +internal class Integration( + private val notebook: Notebook, + private val options: MutableMap, +) : JupyterIntegration() { + + val version = options["v"] + + private fun KotlinKernelHost.execute(codeWithConverter: CodeWithConverter, argument: String): VariableName? { + val code = codeWithConverter.with(argument) + return if (code.isNotBlank()) { + val result = execute(code) + if (codeWithConverter.hasConverter) { + result.name + } else null + } else null + } + + private fun KotlinKernelHost.execute( + codeWithConverter: CodeWithConverter, + property: KProperty<*>, + type: KType, + ): VariableName? { + val variableName = "(${property.name}${if (property.returnType.isMarkedNullable) "!!" else ""} as $type)" + return execute(codeWithConverter, variableName) + } + + private fun KotlinKernelHost.updateImportDataSchemaVariable( + importDataSchema: ImportDataSchema, + property: KProperty<*>, + ): VariableName? { + val formats = supportedFormats.filterIsInstance() + val name = property.name + "DataSchema" + return when ( + val codeGenResult = CodeGenerator.urlCodeGenReader(importDataSchema.url, name, formats, true) + ) { + is CodeGenerationReadResult.Success -> { + val readDfMethod = codeGenResult.getReadDfMethod(importDataSchema.url.toExternalForm()) + val code = readDfMethod.additionalImports.joinToString("\n") + + "\n" + + codeGenResult.code + + execute(code) + execute("""DISPLAY("Data schema successfully imported as ${property.name}: $name")""") + + name + } + + is CodeGenerationReadResult.Error -> { + execute("""DISPLAY("Failed to read data schema from ${importDataSchema.url}: ${codeGenResult.reason}")""") + null + } + } + } + + private fun KotlinKernelHost.updateAnyFrameVariable( + df: AnyFrame, + property: KProperty<*>, + codeGen: ReplCodeGenerator, + ): VariableName? = execute( + codeWithConverter = codeGen.process(df, property), + property = property, + type = DataFrame::class.createStarProjectedType(false), + + ) + + private fun KotlinKernelHost.updateAnyRowVariable( + row: AnyRow, + property: KProperty<*>, + codeGen: ReplCodeGenerator, + ): VariableName? = execute( + codeWithConverter = codeGen.process(row, property), + property = property, + type = DataRow::class.createStarProjectedType(false), + ) + + private fun KotlinKernelHost.updateColumnGroupVariable( + col: ColumnGroup<*>, + property: KProperty<*>, + codeGen: ReplCodeGenerator, + ): VariableName? = execute( + codeWithConverter = codeGen.process(col.asDataFrame(), property), + property = property, + type = ColumnGroup::class.createStarProjectedType(false), + ) + + private fun KotlinKernelHost.updateAnyColVariable( + col: AnyCol, + property: KProperty<*>, + codeGen: ReplCodeGenerator, + ): VariableName? = if (col.isColumnGroup()) { + val codeWithConverter = codeGen.process(col.asColumnGroup().asDataFrame(), property).let { c -> + CodeWithConverter(c.declarations) { c.converter("$it.asColumnGroup()") } + } + execute( + codeWithConverter = codeWithConverter, + property = property, + type = DataColumn::class.createStarProjectedType(false), + ) + } else { + null + } + + override fun Builder.onLoaded() { + if (version != null) { + dependencies( + "org.jetbrains.kotlinx:dataframe-excel:$version", + "org.jetbrains.kotlinx:dataframe-jdbc:$version", + "org.jetbrains.kotlinx:dataframe-arrow:$version", + "org.jetbrains.kotlinx:dataframe-openapi:$version", + ) + } + + try { + setMinimalKernelVersion(MIN_KERNEL_VERSION) + } catch (_: NoSuchMethodError) { // will be thrown when a version < 0.11.0.198 + throw IllegalStateException( + getKernelUpdateMessage(notebook.kernelVersion, MIN_KERNEL_VERSION, notebook.jupyterClientType) + ) + } + val codeGen = ReplCodeGenerator.create() + val config = JupyterConfiguration() + + if (notebook.jupyterClientType == JupyterClientType.KOTLIN_NOTEBOOK) { + config.display.isolatedOutputs = true + } + + onLoaded { + declare("dataFrameConfig" to config) + } + + resources { + if (!config.display.isolatedOutputs) { + js("DataFrame") { + if (config.display.localTesting) { + classPath("init.js") + } else { + // Update this commit when new version of init.js is pushed + val initJsSha = "3db46ccccaa1291c0627307d64133317f545e6ae" + url("https://cdn.jsdelivr.net/gh/Kotlin/dataframe@$initJsSha/core/src/main/resources/init.js") + } + } + + css("DataFrameTable") { classPath("table.css") } + } + } + + with(JupyterHtmlRenderer(config.display, this)) { + render( + { "DataRow: index = ${it.value.rowsCount()}, columnsCount = ${it.value.columnsCount()}" }, + applyRowsLimit = false + ) + + render>({ "Group" }) + render>({ "Move" }) + render>({ "Rename" }) + render>({ "Replace" }) + render>({ "Insert" }) + render>({ "Format" }) + + render { + // Our integration declares script and css definition. But in Kotlin Notebook outputs are isolated in IFrames + // That's why we include them directly in the output + if (notebook.jupyterClientType == JupyterClientType.KOTLIN_NOTEBOOK) { + it.withTableDefinitions().toJupyterHtmlData().toIFrame(notebook.currentColorScheme) + } else { + it.toJupyterHtmlData().toSimpleHtml(notebook.currentColorScheme) + } + } + + render( + { "DataRow: index = ${it.index()}, columnsCount = ${it.columnsCount()}" }, + ) + render>( + { """ColumnGroup: name = "${it.name}", rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}""" }, + ) + render( + { """DataColumn: name = "${it.name}", type = ${renderType(it.type())}, size = ${it.size()}""" }, + ) + render( + { "DataFrame: rowsCount = ${it.rowsCount()}, columnsCount = ${it.columnsCount()}" } + ) + render>( + { "DataFrame: rowsCount = ${it.df.rowsCount()}, columnsCount = ${it.df.columnsCount()}" }, + modifyConfig = { getDisplayConfiguration(it) }, + ) + render>({ "GroupBy" }) + render>({ "ReducedGroupBy" }) + render>({ "Pivot" }) + render>({ "ReducedPivot" }) + render>({ "PivotGroupBy" }) + render>({ "ReducedPivotGroupBy" }) + render>({ "Split" }) + render>({ "Split" }) + render>({ "Merge" }) + render>({ "Gather" }) + render { HTML(it.toString()) } + render