[ZEPPELIN-6733] Add AI assistant chat to notebooks (server-side)

conf/zeppelin-site.xml.template

@@ -846,4 +846,30 @@
   <description>fields to be excluded from being saved in note files, with Paragraph prefix mean the fields in Paragraph, e.g. Paragraph.results</description>
 </property>
 
+<property>
+  <name>zeppelin.assistant.enable</name>
+  <value>false</value>
+</property>
+
+<property>
+  <name>zeppelin.assistant.openai.api.key</name>
+  <value></value>
+</property>
+
+<property>
+  <name>zeppelin.assistant.openai.endpoint</name>
+  <value>https://api.openai.com/v1</value>
+</property>
+
+<property>
+  <name>zeppelin.assistant.openai.model</name>
+  <value>gpt-6-luna</value>
+</property>
+
+<property>
+  <name>zeppelin.assistant.dir</name>
+  <value>assistant</value>
+  <description>Directory for assistant conversations.</description>
+</property>
+
 </configuration>

zeppelin-common/src/main/java/org/apache/zeppelin/common/Message.java

@@ -203,7 +203,9 @@ public enum OP {
     COLLABORATIVE_MODE_STATUS,    // [s-c] collaborative mode status
     PATCH_PARAGRAPH,              // [c-s][s-c] patch editor text
     NOTE_RUNNING_STATUS,        // [s-c] sequential run status will be change
-    NOTICE                        // [s-c] Notice
+    NOTICE,                       // [s-c] Notice
+    ASSISTANT_SEND_MESSAGE,       // [c-s] send a message to the notebook assistant
+    ASSISTANT_EVENT               // [s-c] assistant run stream event (see "type" field)
   }
 
   // these messages will be ignored during the sequential run of the note

zeppelin-distribution/src/bin_license/LICENSE

@@ -1,5 +1,25 @@
 The following components are provided under Apache License.
 
+    (Apache 2.0) ClassMate (com.fasterxml:classmate:1.7.0 - https://github.com/FasterXML/java-classmate)
+    (Apache 2.0) Jackson JDK8 Datatypes (com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.18.3 - https://github.com/FasterXML/jackson-modules-java8)
+    (Apache 2.0) Jackson Java Time Datatypes (com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.3 - https://github.com/FasterXML/jackson-modules-java8)
+    (Apache 2.0) Jackson Kotlin Module (com.fasterxml.jackson.module:jackson-module-kotlin:2.18.3 - https://github.com/FasterXML/jackson-module-kotlin)
+    (Apache 2.0) JSON Schema Generator (com.github.victools:jsonschema-generator:4.38.0 - https://github.com/victools/jsonschema-generator)
+    (Apache 2.0) JSON Schema Generator (com.github.victools:jsonschema-module-jackson:4.38.0 - https://github.com/victools/jsonschema-generator)
+    (Apache 2.0) JSON Schema Generator (com.github.victools:jsonschema-module-swagger-2:4.38.0 - https://github.com/victools/jsonschema-generator)
+    (Apache 2.0) OpenAI Java SDK (com.openai:openai-java:4.69.3 - https://github.com/openai/openai-java)
+    (Apache 2.0) OpenAI Java SDK (com.openai:openai-java-client-okhttp:4.69.3 - https://github.com/openai/openai-java)
+    (Apache 2.0) OpenAI Java SDK (com.openai:openai-java-core:4.69.3 - https://github.com/openai/openai-java)
+    (Apache 2.0) OkHttp (com.squareup.okhttp3:okhttp:4.12.0 - https://github.com/square/okhttp)
+    (Apache 2.0) Okio (com.squareup.okio:okio:3.6.0 - https://github.com/square/okio)
+    (Apache 2.0) Okio (com.squareup.okio:okio-jvm:3.6.0 - https://github.com/square/okio)
+    (Apache 2.0) Swagger Annotations (io.swagger.core.v3:swagger-annotations:2.2.31 - https://github.com/swagger-api/swagger-core)
+    (Apache 2.0) Kotlin (org.jetbrains.kotlin:kotlin-reflect:1.9.10 - https://github.com/JetBrains/kotlin)
+    (Apache 2.0) Kotlin (org.jetbrains.kotlin:kotlin-stdlib:1.9.10 - https://github.com/JetBrains/kotlin)
+    (Apache 2.0) Kotlin (org.jetbrains.kotlin:kotlin-stdlib-common:1.9.10 - https://github.com/JetBrains/kotlin)
+    (Apache 2.0) Kotlin (org.jetbrains.kotlin:kotlin-stdlib-jdk7:1.9.10 - https://github.com/JetBrains/kotlin)
+    (Apache 2.0) Kotlin (org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.9.10 - https://github.com/JetBrains/kotlin)
+
     (Apache 2.0) nvd3.js v1.7.1 (http://nvd3.org/) - https://github.com/novus/nvd3/blob/v1.7.1/LICENSE.md
     (Apache 2.0) gson v2.8.9 (com.google.code.gson:gson:jar:2.8.9 - https://github.com/google/gson) - https://github.com/google/gson/blob/gson-parent-2.8.9/LICENSE
     (Apache 2.0) Amazon Web Services SDK for Java v1.11.736 (https://aws.amazon.com/sdk-for-java/) - https://raw.githubusercontent.com/aws/aws-sdk-java/1.11.736/LICENSE.txt

zeppelin-server/pom.xml

@@ -432,6 +432,71 @@
       <artifactId>gson</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>com.openai</groupId>
+      <artifactId>openai-java</artifactId>
+      <version>4.69.3</version>
+      <exclusions>
+        <!-- Declare consistent Kotlin and annotation versions below. -->
+        <exclusion>
+          <groupId>org.jetbrains.kotlin</groupId>
+          <artifactId>kotlin-stdlib-jdk8</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.jetbrains.kotlin</groupId>
+          <artifactId>kotlin-reflect</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>com.google.errorprone</groupId>
+          <artifactId>error_prone_annotations</artifactId>
+        </exclusion>
+        <exclusion>
+          <groupId>org.jetbrains</groupId>
+          <artifactId>annotations</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <!-- Align SDK transitives for server and distribution. -->
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-stdlib-jdk8</artifactId>
+      <version>1.9.10</version>
+      <exclusions>
+        <!-- Use the annotation version declared below. -->
+        <exclusion>
+          <groupId>org.jetbrains</groupId>
+          <artifactId>annotations</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-reflect</artifactId>
+      <version>1.9.10</version>
+      <scope>runtime</scope>
+      <exclusions>
+        <!-- Use the annotation version declared below. -->
+        <exclusion>
+          <groupId>org.jetbrains</groupId>
+          <artifactId>annotations</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+
+    <dependency>
+      <groupId>org.jetbrains</groupId>
+      <artifactId>annotations</artifactId>
+      <version>15.0</version>
+    </dependency>
+
+    <dependency>
+      <groupId>com.google.errorprone</groupId>
+      <artifactId>error_prone_annotations</artifactId>
+      <version>2.18.0</version>
+    </dependency>
+
     <dependency>
       <groupId>com.nimbusds</groupId>
       <artifactId>nimbus-jose-jwt</artifactId>

zeppelin-server/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java

@@ -888,6 +888,26 @@ public boolean isZeppelinSearchSemanticEnable() {
     return getBoolean(ConfVars.ZEPPELIN_SEARCH_SEMANTIC_ENABLE);
   }
 
+  public boolean isAssistantEnabled() {
+    return getBoolean(ConfVars.ZEPPELIN_ASSISTANT_ENABLE);
+  }
+
+  public String getAssistantApiKey() {
+    return getString(ConfVars.ZEPPELIN_ASSISTANT_OPENAI_API_KEY);
+  }
+
+  public String getAssistantEndpoint() {
+    return getString(ConfVars.ZEPPELIN_ASSISTANT_OPENAI_ENDPOINT);
+  }
+
+  public String getAssistantModel() {
+    return getString(ConfVars.ZEPPELIN_ASSISTANT_OPENAI_MODEL);
+  }
+
+  public String getAssistantDir() {
+    return getAbsoluteDir(ConfVars.ZEPPELIN_ASSISTANT_DIR);
+  }
+
   public boolean isOnlyYarnCluster() {
     return getBoolean(ConfVars.ZEPPELIN_SPARK_ONLY_YARN_CLUSTER);
   }
@@ -1193,7 +1213,16 @@ public enum ConfVars {
     ZEPPELIN_SPARK_ONLY_YARN_CLUSTER("zeppelin.spark.only_yarn_cluster", false),
     ZEPPELIN_SESSION_CHECK_INTERVAL("zeppelin.session.check_interval", 60 * 10 * 1000),
     ZEPPELIN_NOTE_CACHE_THRESHOLD("zeppelin.note.cache.threshold", 50),
-    ZEPPELIN_NOTE_FILE_EXCLUDE_FIELDS("zeppelin.note.file.exclude.fields", "");
+    ZEPPELIN_NOTE_FILE_EXCLUDE_FIELDS("zeppelin.note.file.exclude.fields", ""),
+    ZEPPELIN_ASSISTANT_ENABLE("zeppelin.assistant.enable", false),
+    ZEPPELIN_ASSISTANT_DIR("zeppelin.assistant.dir", "assistant"),
+    ZEPPELIN_ASSISTANT_OPENAI_API_KEY("zeppelin.assistant.openai.api.key", ""),
+    ZEPPELIN_ASSISTANT_OPENAI_ENDPOINT(
+        "zeppelin.assistant.openai.endpoint", "https://api.openai.com/v1"
+    ),
+    ZEPPELIN_ASSISTANT_OPENAI_MODEL(
+        "zeppelin.assistant.openai.model", "gpt-6-luna"
+    );
 
     private String varName;
     private Class<?> varClass;

zeppelin-server/src/main/java/org/apache/zeppelin/rest/AssistantConversationRestApi.java

@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.rest;
+
+import com.google.gson.JsonSyntaxException;
+
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+import jakarta.ws.rs.BadRequestException;
+import jakarta.ws.rs.DELETE;
+import jakarta.ws.rs.GET;
+import jakarta.ws.rs.PATCH;
+import jakarta.ws.rs.POST;
+import jakarta.ws.rs.Path;
+import jakarta.ws.rs.PathParam;
+import jakarta.ws.rs.Produces;
+import jakarta.ws.rs.core.Response;
+
+import java.io.IOException;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.apache.zeppelin.annotation.ZeppelinApi;
+import org.apache.zeppelin.rest.message.ConversationRequest;
+import org.apache.zeppelin.rest.message.ConversationMetadata;
+import org.apache.zeppelin.rest.message.ConversationResponse;
+import org.apache.zeppelin.server.JsonResponse;
+import org.apache.zeppelin.service.AuthenticationService;
+import org.apache.zeppelin.service.assistant.Assistant;
+
+@Path("/notes/{noteId}/conversations")
+@Produces("application/json")
+@Singleton
+public class AssistantConversationRestApi extends AbstractRestApi {
+
+  private final Assistant assistant;
+
+  @Inject
+  public AssistantConversationRestApi(
+      AuthenticationService authenticationService,
+      Assistant assistant
+  ) {
+    super(authenticationService);
+    this.assistant = assistant;
+  }
+
+  private static ConversationRequest parseBody(String body) {
+    try {
+      return GSON.fromJson(body, ConversationRequest.class);
+    } catch (JsonSyntaxException e) {
+      throw new BadRequestException();
+    }
+  }
+
+  @GET
+  @ZeppelinApi
+  public Response list(@PathParam("noteId") String noteId) throws IOException {
+    var context = getServiceContext();
+    return new JsonResponse<>(
+        Response.Status.OK,
+        "",
+        assistant.listConversations(noteId, context.getUserAndRoles()).stream()
+            .map(c -> ConversationMetadata.of(c, context.getAutheInfo().getUser()))
+            .collect(Collectors.toList())
+    ).build();
+  }
+
+  @GET
+  @Path("/{conversationId}")
+  @ZeppelinApi
+  public Response get(
+      @PathParam("noteId") String noteId,
+      @PathParam("conversationId") String conversationId
+  ) throws IOException {
+    var context = getServiceContext();
+    var conversation = assistant.getConversation(
+        noteId,
+        conversationId,
+        context.getUserAndRoles()
+    );
+    return new JsonResponse<>(
+        Response.Status.OK,
+        "",
+        ConversationResponse.of(conversation, context.getAutheInfo().getUser())
+    ).build();
+  }
+
+  @POST
+  @ZeppelinApi
+  public Response create(
+      @PathParam("noteId") String noteId,
+      String body
+  ) throws IOException {
+    String title = Optional.ofNullable(parseBody(body))
+        .map(ConversationRequest::getTitle)
+        .orElse(null);
+    var context = getServiceContext();
+    var conversation = assistant.createConversation(
+        noteId, title, context.getAutheInfo(), context.getUserAndRoles()
+    );
+    return new JsonResponse<>(
+        Response.Status.CREATED,
+        "",
+        ConversationMetadata.of(conversation, context.getAutheInfo().getUser())
+    ).build();
+  }
+
+  @PATCH
+  @Path("/{conversationId}")
+  @ZeppelinApi
+  public Response updateTitle(
+      @PathParam("noteId") String noteId,
+      @PathParam("conversationId") String conversationId,
+      String body
+  ) throws IOException {
+    String title = Optional.ofNullable(parseBody(body))
+        .map(ConversationRequest::getTitle)
+        .filter(t -> !t.isBlank())
+        .orElseThrow(() -> new BadRequestException("title is required"));
+    var context = getServiceContext();
+    var conversation = assistant.updateTitle(
+        noteId, conversationId, title, context.getAutheInfo().getUser(), context.getUserAndRoles());
+    return new JsonResponse<>(
+        Response.Status.OK,
+        "",
+        ConversationMetadata.of(conversation, context.getAutheInfo().getUser())
+    ).build();
+  }
+
+  @DELETE
+  @Path("/{conversationId}")
+  @ZeppelinApi
+  public Response delete(
+      @PathParam("noteId") String noteId,
+      @PathParam("conversationId") String conversationId
+  ) throws IOException {
+    var context = getServiceContext();
+    assistant.deleteConversation(
+        noteId, conversationId, context.getAutheInfo().getUser(), context.getUserAndRoles());
+    return Response.noContent().build();
+  }
+}