diff --git a/changelog/unreleased/migrate-packageapi-to-jax-rs.yml b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml new file mode 100644 index 000000000000..62157390e348 --- /dev/null +++ b/changelog/unreleased/migrate-packageapi-to-jax-rs.yml @@ -0,0 +1,7 @@ +title: Migrate PackageAPI to JAX-RS. PackageAPI now has OpenAPI and SolrJ support. +type: changed +authors: + - name: Eric Pugh +links: +- name: PR#4178 + url: https://github.com/apache/solr/pull/4178 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java new file mode 100644 index 000000000000..9aeb40119d76 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/PackageApis.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.DELETE; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.QueryParam; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; + +/** V2 API definitions for managing Solr packages. */ +@Path("/cluster/package") +public interface PackageApis { + + @GET + @Operation( + summary = "List all packages registered in this Solr cluster.", + tags = {"package"}) + PackagesResponse listPackages( + @Parameter(description = "If provided, the named package is refreshed on this node.") + @QueryParam("refreshPackage") + String refreshPackage, + @Parameter( + description = + "If provided, the node waits until its package data matches this ZooKeeper version.") + @QueryParam("expectedVersion") + Integer expectedVersion); + + @GET + @Path("/{packageName}") + @Operation( + summary = "Get information about a specific package in this Solr cluster.", + tags = {"package"}) + PackagesResponse getPackage( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName); + + @POST + @Path("/{packageName}/versions") + @Operation( + summary = "Add a version of a package to this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse addPackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @RequestBody(description = "Details of the package version to add.", required = true) + AddPackageVersionRequestBody requestBody); + + @DELETE + @Path("/{packageName}/versions/{version}") + @Operation( + summary = "Delete a specific version of a package from this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse deletePackageVersion( + @Parameter(description = "The name of the package.", required = true) + @PathParam("packageName") + String packageName, + @Parameter(description = "The version of the package to delete.", required = true) + @PathParam("version") + String version); + + @POST + @Path("/{packageName}/refresh") + @Operation( + summary = "Refresh a package on all nodes in this Solr cluster.", + tags = {"package"}) + SolrJerseyResponse refreshPackage( + @Parameter(description = "The name of the package to refresh.", required = true) + @PathParam("packageName") + String packageName); +} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java similarity index 50% rename from solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java rename to solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java index 831d87d809f7..3b077c110d3b 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/beans/PackagePayload.java +++ b/solr/api/src/java/org/apache/solr/client/api/model/AddPackageVersionRequestBody.java @@ -14,34 +14,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package org.apache.solr.client.api.model; -package org.apache.solr.client.solrj.request.beans; - +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; import java.util.List; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.util.ReflectMapWriter; - -/** Just a container class for POJOs used in Package APIs */ -public class PackagePayload { - public static class AddVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; - @JsonProperty(required = true) - public String version; +/** Request body for adding a version of a package. */ +public class AddPackageVersionRequestBody { - @JsonProperty(required = true) - public List files; + @JsonProperty("version") + @Schema(description = "The version string for this package version.", required = true) + public String version; - @JsonProperty public String manifest; - @JsonProperty public String manifestSHA512; - } + @JsonProperty("files") + @Schema( + description = "File paths from the file store to include in this version.", + required = true) + public List files; - public static class DelVersion implements ReflectMapWriter { - @JsonProperty(value = "package", required = true) - public String pkg; + @JsonProperty("manifest") + @Schema(description = "Optional path to a manifest file in the file store.") + public String manifest; - @JsonProperty(required = true) - public String version; - } + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest file.") + public String manifestSHA512; } diff --git a/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java new file mode 100644 index 000000000000..0dce425c807b --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/PackagesResponse.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; +import java.util.Map; + +/** Response for the package listing API. */ +public class PackagesResponse extends SolrJerseyResponse { + + @JsonProperty("result") + @Schema(description = "The package data including znode version and package definitions.") + public PackageData result; + + /** Package data returned by the package API. */ + public static class PackageData { + @JsonProperty("znodeVersion") + @Schema(description = "The ZooKeeper version of the packages.json node.") + public int znodeVersion; + + @JsonProperty("packages") + @Schema(description = "Map from package name to list of package versions.") + public Map> packages; + } + + /** Describes a single version of a package. */ + public static class PackageVersion { + @JsonProperty("package") + @Schema(description = "The package name.") + public String pkg; + + @JsonProperty("version") + @Schema(description = "The version string.") + public String version; + + @JsonProperty("files") + @Schema(description = "List of file paths from the file store included in this version.") + public List files; + + @JsonProperty("manifest") + @Schema(description = "Optional manifest reference.") + public String manifest; + + @JsonProperty("manifestSHA512") + @Schema(description = "Optional SHA-512 hash of the manifest.") + public String manifestSHA512; + } +} diff --git a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java index 20cd2440bbe3..f58cb95fbcb5 100644 --- a/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java +++ b/solr/core/src/java/org/apache/solr/api/ContainerPluginsRegistry.java @@ -335,7 +335,7 @@ public ApiInfo(PluginMetaHolder infoHolder, List errs) { coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); if (ver.isEmpty()) { // may be we are a bit early. Do a refresh and try again - coreContainer.getPackageLoader().getPackageAPI().refreshPackages(null); + coreContainer.getPackageLoader().getPackageStore().refreshPackages(null); ver = coreContainer.getPackageLoader().getPackageVersion(pkg, info.version); } if (ver.isEmpty()) { diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index f6bf1dfb36ab..66daf22de020 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -136,6 +136,7 @@ import org.apache.solr.metrics.SolrMetricProducer; import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.metrics.otel.OtelUnit; +import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; @@ -841,8 +842,7 @@ private void loadInternal() { registerV2ApiIfEnabled(ClusterFileStore.class); packageLoader = new SolrPackageLoader(this); - registerV2ApiIfEnabled(packageLoader.getPackageAPI().editAPI); - registerV2ApiIfEnabled(packageLoader.getPackageAPI().readAPI); + registerV2ApiIfEnabled(PackageAPI.class); registerV2ApiIfEnabled(ZookeeperRead.class); } diff --git a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java index c3b76dca4bba..bdda47d55f7d 100644 --- a/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java +++ b/solr/core/src/java/org/apache/solr/filestore/ClusterFileStore.java @@ -47,7 +47,7 @@ import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.jersey.PermissionName; -import org.apache.solr.pkg.PackageAPI; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.security.PermissionNameProvider; @@ -86,8 +86,8 @@ public ClusterFileStore( public UploadToFileStoreResponse uploadFile( String filePath, List sig, InputStream requestBody) { final var response = instantiateJerseyResponse(UploadToFileStoreResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } try { coreContainer @@ -302,12 +302,12 @@ private void doDelete(String filePath, Boolean localDelete) { @PermissionName(PermissionNameProvider.Name.FILESTORE_WRITE_PERM) public SolrJerseyResponse deleteFile(String filePath, Boolean localDelete) { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); - if (!coreContainer.getPackageLoader().getPackageAPI().isEnabled()) { - throw new RuntimeException(PackageAPI.ERR_MSG); + if (!coreContainer.getPackageLoader().getPackageStore().isEnabled()) { + throw new RuntimeException(PackageStore.ERR_MSG); } validateName(filePath, true); - if (coreContainer.getPackageLoader().getPackageAPI().isJarInuse(filePath)) { + if (coreContainer.getPackageLoader().getPackageStore().isJarInuse(filePath)) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "jar in use, can't delete"); } doDelete(filePath, localDelete); diff --git a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java index cba47bad98d2..e7a7f9f98f4d 100644 --- a/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java @@ -88,8 +88,8 @@ import org.apache.solr.handler.admin.api.GetConfigAPI; import org.apache.solr.handler.admin.api.ModifyConfigComponentAPI; import org.apache.solr.handler.admin.api.ModifyParamSetAPI; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; @@ -303,7 +303,7 @@ private void handleGET() { List listeners = req.getCore().getPackageListeners().getListeners(); for (PackageListeners.Listener listener : listeners) { - Map infos = listener.packageDetails(); + Map infos = listener.packageDetails(); if (infos == null || infos.isEmpty()) continue; infos.forEach( (s, mapWriter) -> { diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java index d3bb6216b8ae..54b434eca793 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java @@ -62,8 +62,8 @@ import org.apache.solr.core.SolrCore; import org.apache.solr.handler.RequestHandlerBase; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -202,7 +202,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Collections.emptyMap(); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 943c9248a62b..4dc807b44e2e 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -51,10 +51,8 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.beans.PluginMeta; -import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; @@ -152,20 +150,9 @@ public void uninstall(String packageName, String version) // Delete the package by calling the Package API and remove the Jar printGreen("Executing Package API to remove this package..."); - PackagePayload.DelVersion del = new PackagePayload.DelVersion(); - del.version = version; - del.pkg = packageName; - - V2Request req = - new V2Request.Builder(PackageUtils.PACKAGE_PATH) - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("delete", del)) - .build(); - try { - V2Response resp = req.process(solrClient); - printGreen("Response: " + resp.jsonStr()); + new PackageApi.DeletePackageVersion(packageName, version).process(solrClient); + printGreen("Package version deleted from Package API."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } @@ -467,10 +454,7 @@ private Pair, List> deployCollectionPackage( // If updating, refresh the package version for this to take effect if (isUpdate || pegToLatest) { try { - SolrCLI.postJsonToSolr( - solrClient, - PackageUtils.PACKAGE_PATH, - "{\"refresh\": \"" + packageInstance.name + "\"}"); + new PackageApi.RefreshPackage(packageInstance.name).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } @@ -1082,8 +1066,7 @@ public void undeploy( solrClient, PackageUtils.getCollectionParamsPath(collection), "{set: {PKG_VERSIONS: {" + packageName + ": null}}}"); - SolrCLI.postJsonToSolr( - solrClient, PackageUtils.PACKAGE_PATH, "{\"refresh\": \"" + packageName + "\"}"); + new PackageApi.RefreshPackage(packageName).process(solrClient); } catch (Exception ex) { throw new SolrException(ErrorCode.SERVER_ERROR, ex); } diff --git a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java index 013d3639e55b..3a99673b1531 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/RepositoryManager.java @@ -40,25 +40,19 @@ import org.apache.solr.cli.SolrCLI; import org.apache.solr.client.api.util.SolrVersion; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.FileStoreApi; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.GenericV2SolrRequest; -import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.SystemInfoRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.SystemInfoResponse; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.Utils; import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.packagemanager.SolrPackage.Artifact; import org.apache.solr.packagemanager.SolrPackage.SolrPackageRelease; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -227,10 +221,9 @@ private boolean installPackage(String packageName, String version) throws SolrEx // Call Package API to add this version of the package printGreen("Executing Package API to register this package..."); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = version; - add.pkg = packageName; - add.files = + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion(packageName); + addRequest.setVersion(version); + addRequest.setFiles( downloaded.stream() .map( file -> @@ -240,21 +233,12 @@ private boolean installPackage(String packageName, String version) throws SolrEx packageName, version, file.getFileName().toString())) - .collect(Collectors.toList()); - add.manifest = "/package/" + packageName + "/" + version + "/manifest.json"; - add.manifestSHA512 = manifestSHA512; - - GenericSolrRequest request = - new GenericV2SolrRequest(SolrRequest.METHOD.POST, PackageUtils.PACKAGE_PATH) { - @Override - public RequestWriter.ContentWriter getContentWriter(String expectedType) { - return new RequestWriter.StringPayloadContentWriter( - "{add:" + add.jsonStr() + "}", "application/json"); - } - }; + .collect(Collectors.toList())); + addRequest.setManifest("/package/" + packageName + "/" + version + "/manifest.json"); + addRequest.setManifestSHA512(manifestSHA512); try { - NamedList resp = solrClient.request(request); - printGreen("Response: " + resp.jsonStr()); + addRequest.process(solrClient); + printGreen("Package version registered successfully."); } catch (SolrServerException | IOException e) { throw new SolrException(ErrorCode.BAD_REQUEST, e); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java index 24ea22cd3874..0e0318bd0505 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageAPI.java @@ -14,426 +14,291 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.solr.pkg; import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_EDIT_PERM; import static org.apache.solr.security.PermissionNameProvider.Name.PACKAGE_READ_PERM; -import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.inject.Inject; import java.io.IOException; -import java.io.StringWriter; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import org.apache.solr.api.Command; -import org.apache.solr.api.EndPoint; -import org.apache.solr.api.PayloadObj; +import java.util.stream.Collectors; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.PackageApis; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.JavaBinResponseParser; import org.apache.solr.common.SolrException; -import org.apache.solr.common.annotation.JsonProperty; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZooKeeperException; import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.common.util.EnvUtils; -import org.apache.solr.common.util.ReflectMapWriter; import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.filestore.FileStoreUtils; +import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; -import org.apache.solr.util.SolrJacksonAnnotationInspector; import org.apache.zookeeper.KeeperException; -import org.apache.zookeeper.WatchedEvent; -import org.apache.zookeeper.Watcher; -import org.apache.zookeeper.data.Stat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** This implements the public end points (/api/cluster/package) of package API. */ -public class PackageAPI { - public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); +/** + * JAX-RS implementation of the package management API ({@code /api/cluster/package}). + * + * @see PackageApis + */ +public class PackageAPI extends JerseyResource implements PackageApis { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final String ERR_MSG = - "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + private static final int SYNC_MAX_RETRIES = 10; + private static final long SYNC_SLEEP_MS = 10L; - final CoreContainer coreContainer; - private final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); - private final SolrPackageLoader packageLoader; - Packages pkgs; + private final CoreContainer coreContainer; + private final SolrQueryRequest solrQueryRequest; + private final SolrQueryResponse solrQueryResponse; - public final Edit editAPI = new Edit(); - public final Read readAPI = new Read(); - - public PackageAPI(CoreContainer coreContainer, SolrPackageLoader loader) { + @Inject + public PackageAPI( + CoreContainer coreContainer, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { this.coreContainer = coreContainer; - this.packageLoader = loader; - pkgs = new Packages(); - SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - pkgs = new Packages(); - // ignore + this.solrQueryRequest = solrQueryRequest; + this.solrQueryResponse = solrQueryResponse; + } + + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse listPackages(String refreshPackage, Integer expectedVersion) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + + if (refreshPackage != null) { + packageStore.packageLoader.notifyListeners(refreshPackage); + return instantiateJerseyResponse(PackagesResponse.class); } - try { - registerListener(zkClient); - } catch (KeeperException | InterruptedException e) { - SolrZkClient.checkInterrupted(e); + + if (expectedVersion != null) { + syncToVersion(packageStore, expectedVersion); } - } - private void registerListener(SolrZkClient zkClient) - throws KeeperException, InterruptedException { - zkClient.exists( - SOLR_PKGS_PATH, - new Watcher() { - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher - if (Event.EventType.None.equals(event.getType())) { - return; - } - synchronized (this) { - log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); - // remake watch - final Watcher thisWatch = this; - refreshPackages(thisWatch); - } - } - }); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageStore.pkgs); + return response; } - public void refreshPackages(Watcher watcher) { - final Stat stat = new Stat(); - try { - final byte[] data = - coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); - pkgs = readPkgsFromZk(data, stat); - packageLoader.refreshPackageConf(); - } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); - } catch (KeeperException e) { - log.error("A ZK error has occurred", e); - throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); - } catch (InterruptedException e) { - // Restore the interrupted status - Thread.currentThread().interrupt(); - log.warn("Interrupted", e); + @Override + @PermissionName(PACKAGE_READ_PERM) + public PackagesResponse getPackage(String packageName) { + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); + final var response = instantiateJerseyResponse(PackagesResponse.class); + response.result = toPackageData(packageStore.pkgs); + // Filter to only the requested package + if (response.result != null && response.result.packages != null) { + final var pkgVersions = response.result.packages.get(packageName); + response.result.packages = Collections.singletonMap(packageName, pkgVersions); } + return response; } - private Packages readPkgsFromZk(byte[] data, Stat stat) - throws KeeperException, InterruptedException { + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse addPackageVersion( + String packageName, AddPackageVersionRequestBody requestBody) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - if (data == null || stat == null) { - stat = new Stat(); - data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); } - Packages packages = null; - if (data == null || data.length == 0) { - packages = new Packages(); - } else { - try { - packages = mapper.readValue(data, Packages.class); - packages.znodeVersion = stat.getVersion(); - } catch (IOException e) { - // invalid data in packages - // TODO handle properly; - return new Packages(); - } + if (requestBody == null || requestBody.files == null || requestBody.files.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No files specified"); } - return packages; - } - - public static class Packages implements ReflectMapWriter { - @JsonProperty public int znodeVersion = -1; - @JsonProperty public Map> packages = new LinkedHashMap<>(); + final List errors = new ArrayList<>(); + FileStoreUtils.validateFiles( + coreContainer.getFileStore(), requestBody.files, true, errors::add); + if (!errors.isEmpty()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.join("; ", errors)); + } - public Packages copy() { - Packages p = new Packages(); - p.znodeVersion = this.znodeVersion; - p.packages = new LinkedHashMap<>(); - packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); - return p; + final PackageStore.Packages[] finalState = new PackageStore.Packages[1]; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = + bytes == null + ? new PackageStore.Packages() + : packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + log.error("Error deserializing packages.json", e); + packages = new PackageStore.Packages(); + } + List list = + packages.packages.computeIfAbsent(packageName, o -> new ArrayList<>()); + for (PackageStore.PkgVersion pkgVersion : list) { + if (Objects.equals(pkgVersion.version, requestBody.version)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Version '" + requestBody.version + "' exists already"); + } + } + list.add(new PackageStore.PkgVersion(packageName, requestBody)); + packages.znodeVersion = stat.getVersion() + 1; + finalState[0] = packages; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + finalState[0] = null; + packageStore.handleZkErr(e); } - } - public static class PkgVersion implements ReflectMapWriter { + if (finalState[0] != null) { + packageStore.pkgs = finalState[0]; + notifyAllNodesToSync(packageStore.pkgs.znodeVersion); + coreContainer.getPackageLoader().refreshPackageConf(); + } - @JsonProperty("package") - public String pkg; + return response; + } - @JsonProperty public String version; + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse deletePackageVersion(String packageName, String version) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - @JsonProperty public List files; + if (!packageStore.isEnabled()) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, PackageStore.ERR_MSG); + } - @JsonProperty public String manifest; + try { + coreContainer + .getZkController() + .getZkClient() + .atomicUpdate( + SOLR_PKGS_PATH, + (stat, bytes) -> { + PackageStore.Packages packages; + try { + packages = packageStore.mapper.readValue(bytes, PackageStore.Packages.class); + packages = packages.copy(); + } catch (IOException e) { + packages = new PackageStore.Packages(); + } + + List versions = packages.packages.get(packageName); + if (versions == null || versions.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); + } + int idxToRemove = -1; + for (int i = 0; i < versions.size(); i++) { + if (Objects.equals(versions.get(i).version, version)) { + idxToRemove = i; + break; + } + } + if (idxToRemove == -1) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such version: " + version); + } + versions.remove(idxToRemove); + packages.znodeVersion = stat.getVersion() + 1; + return Utils.toJSON(packages); + }); + } catch (KeeperException | InterruptedException e) { + packageStore.handleZkErr(e); + } - @JsonProperty public String manifestSHA512; + return response; + } - public PkgVersion() {} + @Override + @PermissionName(PACKAGE_EDIT_PERM) + public SolrJerseyResponse refreshPackage(String packageName) { + final var response = instantiateJerseyResponse(SolrJerseyResponse.class); + PackageStore packageStore = coreContainer.getPackageLoader().getPackageStore(); - public PkgVersion(PackagePayload.AddVersion addVersion) { - this.pkg = addVersion.pkg; - this.version = addVersion.version; - this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); - this.manifest = addVersion.manifest; - this.manifestSHA512 = addVersion.manifestSHA512; + SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(packageName); + if (pkg == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "No such package: " + packageName); } + // first refresh on the current node + packageStore.packageLoader.notifyListeners(packageName); - @Override - public boolean equals(Object obj) { - if (obj instanceof PkgVersion that) { - return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); - } - return false; - } + final var solrParams = new ModifiableSolrParams(); + solrParams.add("omitHeader", "true"); + solrParams.add("refreshPackage", packageName); - @Override - public int hashCode() { - return Objects.hash(version); - } + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); + request.setResponseParser(new JavaBinResponseParser()); - @Override - public String toString() { + for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { + final var baseUrl = + coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); try { - return Utils.writeJson(this, new StringWriter(), false).toString(); - } catch (IOException e) { - throw new RuntimeException(e); + var solrClient = coreContainer.getDefaultHttpSolrClient(); + solrClient.requestWithBaseUrl(baseUrl, request::process); + } catch (SolrServerException | IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Failed to refresh package on node: " + liveNode, + e); } } - public PkgVersion copy() { - PkgVersion result = new PkgVersion(); - result.pkg = this.pkg; - result.version = this.version; - result.files = this.files; - result.manifest = this.manifest; - result.manifestSHA512 = this.manifestSHA512; - return result; - } + return response; } - @EndPoint( - method = SolrRequest.METHOD.POST, - path = "/cluster/package", - permission = PACKAGE_EDIT_PERM) - public class Edit { - - @Command(name = "refresh") - public void refresh(PayloadObj payload) { - String p = payload.get(); - if (p == null) { - payload.addError("Package null"); - return; + private void syncToVersion(PackageStore packageStore, int expectedVersion) { + int origVersion = packageStore.pkgs.znodeVersion; + for (int i = 0; i < SYNC_MAX_RETRIES; i++) { + if (log.isDebugEnabled()) { + log.debug( + "my version is {} , and expected version {}", + packageStore.pkgs.znodeVersion, + expectedVersion); } - SolrPackageLoader.SolrPackage pkg = coreContainer.getPackageLoader().getPackage(p); - if (pkg == null) { - payload.addError("No such package: " + p); - return; - } - // first refresh my own - packageLoader.notifyListeners(p); - - final var solrParams = new ModifiableSolrParams(); - solrParams.add("omitHeader", "true"); - solrParams.add("refreshPackage", p); - - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); - request.setResponseParser(new JavaBinResponseParser()); - - for (String liveNode : FileStoreUtils.fetchAndShuffleRemoteLiveNodes(coreContainer)) { - final var baseUrl = - coreContainer.getZkController().zkStateReader.getBaseUrlV2ForNodeName(liveNode); - try { - var solrClient = coreContainer.getDefaultHttpSolrClient(); - solrClient.requestWithBaseUrl(baseUrl, request::process); - } catch (SolrServerException | IOException e) { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Failed to refresh package on node: " + liveNode, - e); + if (packageStore.pkgs.znodeVersion >= expectedVersion) { + if (origVersion < packageStore.pkgs.znodeVersion) { + coreContainer.getPackageLoader().refreshPackageConf(); } - } - } - - @Command(name = "add") - public void add(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.AddVersion add = payload.get(); - if (add.files.isEmpty()) { - payload.addError("No files specified"); return; } - FileStoreUtils.validateFiles( - coreContainer.getFileStore(), add.files, true, s -> payload.addError(s)); - if (payload.hasError()) return; - Packages[] finalState = new Packages[1]; try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = - bytes == null ? new Packages() : mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - log.error("Error deserializing packages.json", e); - packages = new Packages(); - } - List list = - packages.packages.computeIfAbsent(add.pkg, o -> new ArrayList<>()); - for (PkgVersion version : list) { - if (Objects.equals(version.version, add.version)) { - payload.addError("Version '" + add.version + "' exists already"); - return null; - } - } - list.add(new PkgVersion(add)); - packages.znodeVersion = stat.getVersion() + 1; - finalState[0] = packages; - return Utils.toJSON(packages); - }); - } catch (KeeperException | InterruptedException e) { - finalState[0] = null; - handleZkErr(e); - } - if (finalState[0] != null) { - // succeeded in updating - pkgs = finalState[0]; - notifyAllNodesToSync(pkgs.znodeVersion); - packageLoader.refreshPackageConf(); + Thread.sleep(SYNC_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); } - } - - @Command(name = "delete") - public void del(PayloadObj payload) { - if (!checkEnabled(payload)) return; - PackagePayload.DelVersion delVersion = payload.get(); try { - coreContainer - .getZkController() - .getZkClient() - .atomicUpdate( - SOLR_PKGS_PATH, - (stat, bytes) -> { - Packages packages = null; - try { - packages = mapper.readValue(bytes, Packages.class); - packages = packages.copy(); - } catch (IOException e) { - packages = new Packages(); - } - - List versions = packages.packages.get(delVersion.pkg); - if (versions == null || versions.isEmpty()) { - payload.addError("No such package: " + delVersion.pkg); - return null; // no change - } - int idxToremove = -1; - for (int i = 0; i < versions.size(); i++) { - if (Objects.equals(versions.get(i).version, delVersion.version)) { - idxToremove = i; - break; - } - } - if (idxToremove == -1) { - payload.addError("No such version: " + delVersion.version); - return null; - } - versions.remove(idxToremove); - packages.znodeVersion = stat.getVersion() + 1; - return Utils.toJSON(packages); - }); + packageStore.pkgs = packageStore.readPkgsFromZk(null, null); } catch (KeeperException | InterruptedException e) { - handleZkErr(e); + packageStore.handleZkErr(e); } } } - public boolean isEnabled() { - return enablePackages; - } - - private boolean checkEnabled(CommandOperation payload) { - if (!enablePackages) { - payload.addError(ERR_MSG); - return false; - } - return true; - } - - public class Read { - @EndPoint( - method = SolrRequest.METHOD.GET, - path = {"/cluster/package/", "/cluster/package/{name}"}, - permission = PACKAGE_READ_PERM) - public void get(SolrQueryRequest req, SolrQueryResponse rsp) { - String refresh = req.getParams().get("refreshPackage"); - if (refresh != null) { - packageLoader.notifyListeners(refresh); - return; - } - - int expectedVersion = req.getParams().getInt("expectedVersion", -1); - if (expectedVersion != -1) { - syncToVersion(expectedVersion); - } - String name = req.getPathTemplateValues().get("name"); - if (name == null) { - rsp.add("result", pkgs); - } else { - rsp.add("result", Collections.singletonMap(name, pkgs.packages.get(name))); - } - } - - private void syncToVersion(int expectedVersion) { - int origVersion = pkgs.znodeVersion; - for (int i = 0; i < 10; i++) { - log.debug("my version is {} , and expected version {}", pkgs.znodeVersion, expectedVersion); - if (pkgs.znodeVersion >= expectedVersion) { - if (origVersion < pkgs.znodeVersion) { - packageLoader.refreshPackageConf(); - } - return; - } - try { - Thread.sleep(10); - } catch (InterruptedException e) { - } - try { - pkgs = readPkgsFromZk(null, null); - } catch (KeeperException | InterruptedException e) { - handleZkErr(e); - } - } - } - } - - void notifyAllNodesToSync(int expected) { - + private void notifyAllNodesToSync(int expectedVersion) { final var solrParams = new ModifiableSolrParams(); solrParams.add("omitHeader", "true"); - solrParams.add("expectedVersion", String.valueOf(expected)); + solrParams.add("expectedVersion", String.valueOf(expectedVersion)); final var request = new GenericSolrRequest(SolrRequest.METHOD.GET, "/cluster/package", solrParams); @@ -447,32 +312,41 @@ void notifyAllNodesToSync(int expected) { } catch (SolrServerException | IOException e) { throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, - "Failed to notify node: " + liveNode + " to sync expected package version: " + expected, + "Failed to notify node: " + + liveNode + + " to sync expected package version: " + + expectedVersion, e); } } } - public void handleZkErr(Exception e) { - log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); + private static PackagesResponse.PackageData toPackageData(PackageStore.Packages packages) { + if (packages == null) { + return null; + } + final var data = new PackagesResponse.PackageData(); + data.znodeVersion = packages.znodeVersion; + data.packages = + packages.packages.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + e -> + e.getValue().stream() + .map(PackageAPI::toPkgVersionResponse) + .collect(Collectors.toList()))); + return data; } - public boolean isJarInuse(String path) { - Packages pkg = null; - try { - pkg = readPkgsFromZk(null, null); - } catch (KeeperException.NoNodeException nne) { - return false; - } catch (InterruptedException | KeeperException e) { - throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); - } - for (List vers : pkg.packages.values()) { - for (PkgVersion ver : vers) { - if (ver.files.contains(path)) { - return true; - } - } - } - return false; + private static PackagesResponse.PackageVersion toPkgVersionResponse( + PackageStore.PkgVersion pkgVersion) { + final var v = new PackagesResponse.PackageVersion(); + v.pkg = pkgVersion.pkg; + v.version = pkgVersion.version; + v.files = pkgVersion.files; + v.manifest = pkgVersion.manifest; + v.manifestSHA512 = pkgVersion.manifestSHA512; + return v; } } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java index 7535bb2c7fe9..b892ae4a9561 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeners.java @@ -110,7 +110,7 @@ public interface Listener { String packageName(); /** fetch the package versions of class names */ - Map packageDetails(); + Map packageDetails(); /** A callback when the package is updated */ void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx); diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java index 272de544b6e7..8c9fa0a97889 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackageListeningClassLoader.java @@ -42,7 +42,7 @@ public class PackageListeningClassLoader implements SolrClassLoader, PackageList private final Function pkgVersionSupplier; /** package name and the versions that we are tracking */ - private Map packageVersions = new ConcurrentHashMap<>(1); + private Map packageVersions = new ConcurrentHashMap<>(1); private Map classNameVsPackageName = new ConcurrentHashMap<>(); private final Runnable reloadAction; @@ -99,7 +99,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( p.getLatest(pkgVersionSupplier.apply(cName.pkg)); if (registerListener) { classNameVsPackageName.put(cName.original, cName.pkg); - PackageAPI.PkgVersion pkgVersion = theVersion.getPkgVersion(); + PackageStore.PkgVersion pkgVersion = theVersion.getPkgVersion(); if (pkgVersion != null) packageVersions.put(cName.pkg, pkgVersion); } return theVersion; @@ -111,7 +111,7 @@ public SolrPackageLoader.SolrPackage.Version findPackageVersion( @Override public MapWriter getPackageVersion(PluginInfo.ClassName cName) { if (cName.pkg == null) return null; - PackageAPI.PkgVersion p = packageVersions.get(cName.pkg); + PackageStore.PkgVersion p = packageVersions.get(cName.pkg); return p == null ? null : p::writeMap; } @@ -162,15 +162,15 @@ public String packageName() { } @Override - public Map packageDetails() { - Map result = new LinkedHashMap<>(); + public Map packageDetails() { + Map result = new LinkedHashMap<>(); classNameVsPackageName.forEach((k, v) -> result.put(k, packageVersions.get(v))); return result; } @Override public void changed(SolrPackageLoader.SolrPackage pkg, Ctx ctx) { - PackageAPI.PkgVersion currVer = packageVersions.get(pkg.name); + PackageStore.PkgVersion currVer = packageVersions.get(pkg.name); if (currVer == null) { // not watching this return; diff --git a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java index b0731d05095f..2ac38f3b7ab3 100644 --- a/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java +++ b/solr/core/src/java/org/apache/solr/pkg/PackagePluginHolder.java @@ -59,7 +59,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return Collections.singletonMap(info.cName.original, pkgVersion.getPkgVersion()); } diff --git a/solr/core/src/java/org/apache/solr/pkg/PackageStore.java b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java new file mode 100644 index 000000000000..77c32899c392 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/pkg/PackageStore.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.pkg; + +import static org.apache.solr.common.cloud.ZkStateReader.SOLR_PKGS_PATH; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.StringWriter; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.annotation.JsonProperty; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.ZooKeeperException; +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.ReflectMapWriter; +import org.apache.solr.common.util.Utils; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.util.SolrJacksonAnnotationInspector; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Holds package data loaded from ZooKeeper ({@code /solr/packages.json}) and manages ZK watchers. + */ +public class PackageStore { + public final boolean enablePackages = EnvUtils.getPropertyAsBool("solr.packages.enabled", false); + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public static final String ERR_MSG = + "Package loading is not enabled , Start your nodes with -Dsolr.packages.enabled=true"; + + final CoreContainer coreContainer; + final ObjectMapper mapper = SolrJacksonAnnotationInspector.createObjectMapper(); + final SolrPackageLoader packageLoader; + Packages pkgs; + + public PackageStore(CoreContainer coreContainer, SolrPackageLoader loader) { + this.coreContainer = coreContainer; + this.packageLoader = loader; + pkgs = new Packages(); + SolrZkClient zkClient = coreContainer.getZkController().getZkClient(); + try { + pkgs = readPkgsFromZk(null, null); + } catch (KeeperException | InterruptedException e) { + pkgs = new Packages(); + // ignore + } + try { + registerListener(zkClient); + } catch (KeeperException | InterruptedException e) { + SolrZkClient.checkInterrupted(e); + } + } + + private void registerListener(SolrZkClient zkClient) + throws KeeperException, InterruptedException { + zkClient.exists( + SOLR_PKGS_PATH, + new Watcher() { + + @Override + public void process(WatchedEvent event) { + // session events are not change events, and do not remove the watcher + if (Event.EventType.None.equals(event.getType())) { + return; + } + synchronized (this) { + log.debug("Updating [{}] ... ", SOLR_PKGS_PATH); + // remake watch + final Watcher thisWatch = this; + refreshPackages(thisWatch); + } + } + }); + } + + public void refreshPackages(Watcher watcher) { + final Stat stat = new Stat(); + try { + final byte[] data = + coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, watcher, stat); + pkgs = readPkgsFromZk(data, stat); + packageLoader.refreshPackageConf(); + } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { + log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); + } catch (KeeperException e) { + log.error("A ZK error has occurred", e); + throw new ZooKeeperException(SolrException.ErrorCode.SERVER_ERROR, "", e); + } catch (InterruptedException e) { + // Restore the interrupted status + Thread.currentThread().interrupt(); + log.warn("Interrupted", e); + } + } + + Packages readPkgsFromZk(byte[] data, Stat stat) throws KeeperException, InterruptedException { + + if (data == null || stat == null) { + stat = new Stat(); + data = coreContainer.getZkController().getZkClient().getData(SOLR_PKGS_PATH, null, stat); + } + Packages packages = null; + if (data == null || data.length == 0) { + packages = new Packages(); + } else { + try { + packages = mapper.readValue(data, Packages.class); + packages.znodeVersion = stat.getVersion(); + } catch (IOException e) { + // invalid data in packages + // TODO handle properly; + return new Packages(); + } + } + return packages; + } + + public static class Packages implements ReflectMapWriter { + @JsonProperty public int znodeVersion = -1; + + @JsonProperty public Map> packages = new LinkedHashMap<>(); + + public Packages copy() { + Packages p = new Packages(); + p.znodeVersion = this.znodeVersion; + p.packages = new LinkedHashMap<>(); + packages.forEach((s, versions) -> p.packages.put(s, new ArrayList<>(versions))); + return p; + } + } + + public static class PkgVersion implements ReflectMapWriter { + + @JsonProperty("package") + public String pkg; + + @JsonProperty public String version; + + @JsonProperty public List files; + + @JsonProperty public String manifest; + + @JsonProperty public String manifestSHA512; + + public PkgVersion() {} + + public PkgVersion(String packageName, AddPackageVersionRequestBody addVersion) { + this.pkg = packageName; + this.version = addVersion.version; + this.files = addVersion.files == null ? null : Collections.unmodifiableList(addVersion.files); + this.manifest = addVersion.manifest; + this.manifestSHA512 = addVersion.manifestSHA512; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof PkgVersion that) { + return Objects.equals(this.version, that.version) && Objects.equals(this.files, that.files); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(version); + } + + @Override + public String toString() { + try { + return Utils.writeJson(this, new StringWriter(), false).toString(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public PkgVersion copy() { + PkgVersion result = new PkgVersion(); + result.pkg = this.pkg; + result.version = this.version; + result.files = this.files; + result.manifest = this.manifest; + result.manifestSHA512 = this.manifestSHA512; + return result; + } + } + + public boolean isEnabled() { + return enablePackages; + } + + public void handleZkErr(Exception e) { + log.error("Error reading package config from zookeeper", SolrZkClient.checkInterrupted(e)); + } + + public boolean isJarInuse(String path) { + Packages pkg = null; + try { + pkg = readPkgsFromZk(null, null); + } catch (KeeperException.NoNodeException nne) { + return false; + } catch (InterruptedException | KeeperException e) { + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e); + } + for (List vers : pkg.packages.values()) { + for (PkgVersion ver : vers) { + if (ver.files.contains(path)) { + return true; + } + } + } + return false; + } +} diff --git a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java index f3a86dd55d7b..98593a355a25 100644 --- a/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java +++ b/solr/core/src/java/org/apache/solr/pkg/SolrPackageLoader.java @@ -53,9 +53,9 @@ public class SolrPackageLoader implements Closeable { private final CoreContainer coreContainer; private final Map packageClassLoaders = new ConcurrentHashMap<>(); - private PackageAPI.Packages myCopy = new PackageAPI.Packages(); + private PackageStore.Packages myCopy = new PackageStore.Packages(); - private PackageAPI packageAPI; + private PackageStore packageStore; public Optional getPackageVersion(String pkg, String version) { SolrPackage p = packageClassLoaders.get(pkg); @@ -65,12 +65,12 @@ public Optional getPackageVersion(String pkg, String versio public SolrPackageLoader(CoreContainer coreContainer) { this.coreContainer = coreContainer; - packageAPI = new PackageAPI(coreContainer, this); + packageStore = new PackageStore(coreContainer, this); refreshPackageConf(); } - public PackageAPI getPackageAPI() { - return packageAPI; + public PackageStore getPackageStore() { + return packageStore; } public SolrPackage getPackage(String key) { @@ -83,12 +83,12 @@ public Map getPackages() { public void refreshPackageConf() { log.debug( - "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageAPI.pkgs.znodeVersion); + "{} updated to version {}", ZkStateReader.SOLR_PKGS_PATH, packageStore.pkgs.znodeVersion); List updated = new ArrayList<>(); - Map> modified = getModified(myCopy, packageAPI.pkgs); + Map> modified = getModified(myCopy, packageStore.pkgs); - for (Map.Entry> e : modified.entrySet()) { + for (Map.Entry> e : modified.entrySet()) { if (e.getValue() != null) { SolrPackage p = packageClassLoaders.get(e.getKey()); if (e.getValue() != null && p == null) { @@ -109,14 +109,14 @@ public void refreshPackageConf() { for (SolrCore core : coreContainer.getCores()) { core.getPackageListeners().packagesUpdated(updated); } - myCopy = packageAPI.pkgs; + myCopy = packageStore.pkgs; } - public Map> getModified( - PackageAPI.Packages old, PackageAPI.Packages newPkgs) { - Map> changed = new HashMap<>(); - for (Map.Entry> e : newPkgs.packages.entrySet()) { - List versions = old.packages.get(e.getKey()); + public Map> getModified( + PackageStore.Packages old, PackageStore.Packages newPkgs) { + Map> changed = new HashMap<>(); + for (Map.Entry> e : newPkgs.packages.entrySet()) { + List versions = old.packages.get(e.getKey()); if (versions != null) { if (!Objects.equals(e.getValue(), versions)) { if (log.isInfoEnabled()) { @@ -172,8 +172,8 @@ public Set allVersions() { return myVersions.keySet(); } - private synchronized void updateVersions(List modified) { - for (PackageAPI.PkgVersion v : modified) { + private synchronized void updateVersions(List modified) { + for (PackageStore.PkgVersion v : modified) { Version version = myVersions.get(v.version); if (version == null) { log.info( @@ -194,7 +194,7 @@ private synchronized void updateVersions(List modified) { } Set newVersions = new HashSet<>(); - for (PackageAPI.PkgVersion v : modified) { + for (PackageStore.PkgVersion v : modified) { newVersions.add(v.version); } for (String s : new HashSet<>(myVersions.keySet())) { @@ -258,7 +258,7 @@ public class Version implements MapWriter, Closeable { private final SolrPackage parent; private SolrResourceLoader loader; - private final PackageAPI.PkgVersion version; + private final PackageStore.PkgVersion version; @Override public void writeMap(EntryWriter ew) throws IOException { @@ -266,7 +266,7 @@ public void writeMap(EntryWriter ew) throws IOException { version.writeMap(ew); } - Version(SolrPackage parent, PackageAPI.PkgVersion v) { + Version(SolrPackage parent, PackageStore.PkgVersion v) { this.parent = parent; this.version = v; List paths = new ArrayList<>(); @@ -293,7 +293,7 @@ public String getVersion() { return version.version; } - public PackageAPI.PkgVersion getPkgVersion() { + public PackageStore.PkgVersion getPkgVersion() { return version.copy(); } diff --git a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java index 06593f4b3ab9..3b312785e175 100644 --- a/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java +++ b/solr/core/src/test/org/apache/solr/handler/TestContainerPlugin.java @@ -42,8 +42,8 @@ import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.request.beans.PluginMeta; import org.apache.solr.client.solrj.response.V2Response; import org.apache.solr.cloud.ClusterSingleton; @@ -58,8 +58,8 @@ import org.apache.solr.filestore.ClusterFileStore; import org.apache.solr.filestore.TestDistribFileStore; import org.apache.solr.filestore.TestDistribFileStore.Fetcher; -import org.apache.solr.pkg.PackageAPI; import org.apache.solr.pkg.PackageListeners; +import org.apache.solr.pkg.PackageStore; import org.apache.solr.pkg.SolrPackageLoader; import org.apache.solr.pkg.TestPackages; import org.apache.solr.request.SolrQueryRequest; @@ -93,7 +93,7 @@ public String packageName() { } @Override - public Map packageDetails() { + public Map packageDetails() { return null; // only used to print meta information } @@ -309,16 +309,9 @@ public void testApiFromPackage() throws Exception { // We have two versions of the plugin in 2 different jar files. they are already uploaded to // the package store listener.reset(); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); - add.version = "1.0"; - add.pkg = "mypkg"; - add.files = singletonList(FILE1); - V2Request addPkgVersionReq = - new V2Request.Builder("/cluster/package") - .forceV2(forceV2) - .POST() - .withPayload(singletonMap("add", add)) - .build(); + PackageApi.AddPackageVersion addPkgVersionReq = new PackageApi.AddPackageVersion("mypkg"); + addPkgVersionReq.setVersion("1.0"); + addPkgVersionReq.setFiles(singletonList(FILE1)); addPkgVersionReq.process(cluster.getSolrClient()); assertTrue( "core package listeners did not notify", @@ -336,7 +329,7 @@ public void testApiFromPackage() throws Exception { PluginMeta plugin = new PluginMeta(); plugin.name = "myplugin"; plugin.klass = "mypkg:org.apache.solr.handler.MyPlugin"; - plugin.version = add.version; + plugin.version = "1.0"; final V2Request addPluginReq = postPlugin(singletonMap("add", plugin)); addPluginReq.process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); @@ -352,12 +345,12 @@ public void testApiFromPackage() throws Exception { TestDistribFileStore.assertResponseValues(invokePlugin, Map.of("/myplugin.version", "1.0")); // now let's upload the jar file for version 2.0 of the plugin - add.version = "2.0"; - add.files = singletonList(FILE2); + addPkgVersionReq.setVersion("2.0"); + addPkgVersionReq.setFiles(singletonList(FILE2)); addPkgVersionReq.process(cluster.getSolrClient()); // here the plugin version is updated - plugin.version = add.version; + plugin.version = "2.0"; postPlugin(singletonMap("update", plugin)).process(cluster.getSolrClient()); version = phaser.awaitAdvanceInterruptibly(version, 10, TimeUnit.SECONDS); diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java new file mode 100644 index 000000000000..b1a29c819464 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/pkg/PackageAPITest.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.pkg; + +import static org.apache.solr.filestore.TestDistribFileStore.uploadKey; + +import java.util.List; +import org.apache.solr.client.api.model.PackagesResponse; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.PackageApi; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.filestore.ClusterFileStore; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Integration tests for the JAX-RS-based {@link PackageAPI}. + * + *

Note: SolrJettyTestRule cannot be used here because the Package API requires ZooKeeper for its + * cluster-level operations. A one-node SolrCloud cluster is used instead. + */ +public class PackageAPITest extends SolrCloudTestCase { + + @BeforeClass + public static void setupCluster() throws Exception { + System.setProperty("solr.packages.enabled", "true"); + configureCluster(1) + .withJettyConfig(jetty -> jetty.enableV2(true)) + .addConfig("conf", configset("cloud-minimal")) + .configure(); + } + + @Test + public void testListPackagesReturnsResult() throws Exception { + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + PackagesResponse response = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null response from GET /cluster/package", response); + assertNotNull("Expected 'result' field in GET /cluster/package response", response.result); + } + } + + @Test + public void testAddAndDeletePackageVersion() throws Exception { + String FILE1 = "/pkgapitestpkg/runtimelibs.jar"; + + // Upload a key and a signed jar file to the filestore + byte[] derFile = + org.apache.solr.filestore.TestDistribFileStore.readFile("cryptokeys/pub_key512.der"); + uploadKey(derFile, ClusterFileStore.KEYS_DIR + "/pub_key512.der", cluster); + TestPackages.postFileAndWait( + cluster, + "runtimecode/runtimelibs.jar.bin", + FILE1, + "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); + + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + // Add a package version via POST /cluster/package/{name}/versions + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("pkgapitestpkg"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of(FILE1)); + addRequest.process(client); + + // Verify the package was added via GET /cluster/package + PackagesResponse listResponse = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response", listResponse); + assertNotNull("Expected non-null result", listResponse.result); + assertNotNull( + "Expected pkgapitestpkg in packages", listResponse.result.packages.get("pkgapitestpkg")); + assertFalse( + "Expected at least one version", + listResponse.result.packages.get("pkgapitestpkg").isEmpty()); + + // Verify GET /cluster/package/{name} returns only this package + PackagesResponse getByNameResponse = + new PackageApi.GetPackage("pkgapitestpkg").process(client); + assertNotNull("Expected non-null get-by-name response", getByNameResponse); + assertNotNull("Expected non-null result from get-by-name", getByNameResponse.result); + assertNotNull( + "Expected pkgapitestpkg in get-by-name response", + getByNameResponse.result.packages.get("pkgapitestpkg")); + + // Delete the package version via DELETE /cluster/package/{name}/versions/{version} + new PackageApi.DeletePackageVersion("pkgapitestpkg", "1.0").process(client); + + // Verify it's deleted + PackagesResponse listAfterDelete = new PackageApi.ListPackages().process(client); + assertNotNull("Expected non-null list response after delete", listAfterDelete); + assertNotNull("Expected non-null result after delete", listAfterDelete.result); + // After deleting the only version, the package entry should be empty or absent + List versionsAfterDelete = listAfterDelete.result.packages.get("pkgapitestpkg"); + assertTrue( + "Expected no versions after delete", + versionsAfterDelete == null || versionsAfterDelete.isEmpty()); + } + } + + @Test + public void testAddPackageVersionValidatesFiles() throws Exception { + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + // Try to add a package version with a non-existent file. + // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; + // instead, the error is in the response body's 'error' field. + PackageApi.AddPackageVersion addRequest = new PackageApi.AddPackageVersion("testpkg_invalid"); + addRequest.setVersion("1.0"); + addRequest.setFiles(List.of("/nonexistent/file.jar")); + + var response = addRequest.process(client); + assertNotNull("Expected error in response for non-existent file", response.error); + assertEquals("Expected 400 for non-existent file", 400, (int) response.error.code); + assertTrue( + "Expected error message to mention the file", + response.error.msg.contains("No such file")); + } + } + + @Test + public void testRefreshNonExistentPackage() throws Exception { + try (HttpSolrClient client = + new HttpSolrClient.Builder(cluster.getJettySolrRunner(0).getBaseUrl().toString()).build()) { + // Try to refresh a non-existent package. + // Note: JacksonDataBindResponseParser doesn't expose errors as RemoteSolrException; + // instead, the error is in the response body's 'error' field. + var response = new PackageApi.RefreshPackage("nonexistentpkg_test").process(client); + assertNotNull("Expected error in response for non-existent package", response.error); + assertEquals("Expected 400 for non-existent package", 400, (int) response.error.code); + assertTrue( + "Expected error message to mention the package", + response.error.msg.contains("No such package")); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java index 8faac9b68f9e..25f0e481cd72 100644 --- a/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/PackageStoreSchemaPluginsTest.java @@ -27,10 +27,10 @@ import java.security.Signature; import java.util.Base64; import java.util.List; -import java.util.Map; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.PackageApi; import org.apache.solr.client.solrj.request.V2Request; import org.apache.solr.client.solrj.response.SolrResponseBase; import org.apache.solr.cloud.SolrCloudTestCase; @@ -133,22 +133,10 @@ private void uploadPluginJar(String version, Path jarPath) throws Exception { } private void registerPackage(String version) throws Exception { - var packageRequest = - new V2Request.Builder("/cluster/package") - .POST() - .forceV2(true) - .withPayload( - Map.of( - "add", - Map.of( - "package", - "mypkg", - "version", - version, - "files", - List.of("/my-plugin/plugin-" + version + ".jar")))) - .build(); - processRequest(client, packageRequest); + var addRequest = new PackageApi.AddPackageVersion("mypkg"); + addRequest.setVersion(version); + addRequest.setFiles(List.of("/my-plugin/plugin-" + version + ".jar")); + addRequest.process(client); } private void createCollection() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java index 96d5f86b28a9..685c133c3e56 100644 --- a/solr/core/src/test/org/apache/solr/pkg/TestPackages.java +++ b/solr/core/src/test/org/apache/solr/pkg/TestPackages.java @@ -42,6 +42,7 @@ import org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory; import org.apache.lucene.util.ResourceLoader; import org.apache.lucene.util.ResourceLoaderAware; +import org.apache.solr.client.api.model.AddPackageVersionRequestBody; import org.apache.solr.client.solrj.RemoteSolrException; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; @@ -54,7 +55,6 @@ import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.request.V2Request; -import org.apache.solr.client.solrj.request.beans.PackagePayload; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.util.ClientUtils; import org.apache.solr.cloud.MiniSolrCloudCluster; @@ -130,15 +130,14 @@ public void testCoreReloadingPlugin() throws Exception { FILE1, "L3q/qIGs4NaF6JiO0ZkMUFa88j0OmYc+I6O7BOdNuMct/xoZ4h73aZHZGc0+nmI1f/U3bOlMPINlSOM6LK3JpQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "mypkg"; add.files = Arrays.asList(new String[] {FILE1}); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); req.process(cluster.getSolrClient()); @@ -160,7 +159,7 @@ public void testCoreReloadingPlugin() throws Exception { cluster.waitForActiveCollection(COLLECTION_NAME, 2, 4); verifyComponent( - cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", add.pkg, add.version); + cluster.getSolrClient(), COLLECTION_NAME, "query", "filterCache", "mypkg", add.version); add.version = "2.0"; req.process(cluster.getSolrClient()); @@ -211,15 +210,14 @@ public void testPluginLoading() throws Exception { EXPR1, "ZOT11arAiPmPZYOHzqodiNnxO9pRyRozWZEBX8XGjU1/HJptFnZK+DI7eXnUtbNaMcbXE2Ze8hh4M/eGyhY8BQ=="); - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "mypkg"; add.files = Arrays.asList(new String[] {FILE1, URP1, EXPR1}); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); req.process(cluster.getSolrClient()); @@ -402,16 +400,11 @@ public void testPluginLoading() throws Exception { assertEquals("Version 2", result.getResults().get(0).getFieldValue("TestVersionedURP.Ver_s")); - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.pkg = "mypkg"; - delVersion.version = "1.0"; - V2Request delete = - new V2Request.Builder("/cluster/package") - .withMethod(SolrRequest.METHOD.POST) - .forceV2(true) - .withPayload(Collections.singletonMap("delete", delVersion)) - .build(); - delete.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/mypkg/versions/1.0") + .withMethod(SolrRequest.METHOD.DELETE) + .forceV2(true) + .build() + .process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "2.1"); @@ -422,9 +415,12 @@ public void testPluginLoading() throws Exception { verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "requestHandler", "/runtime", "mypkg", "2.1"); - // now remove the hughest version. So, it will roll back to the next highest one - delVersion.version = "2.1"; - delete.process(cluster.getSolrClient()); + // now remove the highest version. So, it will roll back to the next highest one + new V2Request.Builder("/cluster/package/mypkg/versions/2.1") + .withMethod(SolrRequest.METHOD.DELETE) + .forceV2(true) + .build() + .process(cluster.getSolrClient()); verifyComponent( cluster.getSolrClient(), COLLECTION_NAME, "queryResponseWriter", "json1", "mypkg", "1.1"); @@ -473,9 +469,8 @@ public RequestWriter.ContentWriter getContentWriter(String expectedType) { // now, let's force every collection using 'mypkg' to refresh // so that it uses version 2.1 - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/mypkg/refresh") .withMethod(SolrRequest.METHOD.POST) - .withPayload("{refresh : mypkg}") .forceV2(true) .build() .process(cluster.getSolrClient()); @@ -578,20 +573,19 @@ private void verifyComponent( @Test @SuppressWarnings("unchecked") public void testAPI() throws Exception { - String errPath = "/details[0]/errorMessages[0]"; + String errPath = "/msg"; String FILE1 = "/mypkg/v.0.12/jar_a.jar"; String FILE2 = "/mypkg/v.0.12/jar_b.jar"; String FILE3 = "/mypkg/v.0.13/jar_a.jar"; - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "0.12"; - add.pkg = "test_pkg"; add.files = List.of(FILE1, FILE2); V2Request req = - new V2Request.Builder("/cluster/package") + new V2Request.Builder("/cluster/package/test_pkg/versions") .forceV2(true) .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) + .withPayload(add) .build(); // the files are not yet there. The command should fail with error saying "No such file" @@ -640,7 +634,6 @@ public void testAPI() throws Exception { // this time we are adding the second version of the package (0.13) add.version = "0.13"; - add.pkg = "test_pkg"; add.files = Collections.singletonList(FILE3); // this request should succeed @@ -654,21 +647,21 @@ public void testAPI() throws Exception { Map.of(":packages:test_pkg[1]:version", "0.13", ":packages:test_pkg[1]:files[0]", FILE3)); // Now we will just delete one version - PackagePayload.DelVersion delVersion = new PackagePayload.DelVersion(); - delVersion.version = "0.1"; // this version does not exist - delVersion.pkg = "test_pkg"; - req = - new V2Request.Builder("/cluster/package") + V2Request deleteReq = + new V2Request.Builder("/cluster/package/test_pkg/versions/0.1") .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("delete", delVersion)) + .withMethod(SolrRequest.METHOD.DELETE) .build(); // we are expecting an error - expectError(req, cluster.getSolrClient(), errPath, "No such version:"); + expectError(deleteReq, cluster.getSolrClient(), errPath, "No such version:"); - delVersion.version = "0.12"; // correct version. Should succeed - req.process(cluster.getSolrClient()); + new V2Request.Builder( + "/cluster/package/test_pkg/versions/0.12") // correct version. Should succeed + .forceV2(true) + .withMethod(SolrRequest.METHOD.DELETE) + .build() + .process(cluster.getSolrClient()); // Verify with ZK that the data is correct TestDistribFileStore.assertResponseValues( 1, @@ -762,17 +755,15 @@ public void testSchemaPlugins() throws Exception { "gI6vYUDmSXSXmpNEeK1cwqrp4qTeVQgizGQkd8A4Prx2K8k7c5QlXbcs4lxFAAbbdXz9F4esBqTCiLMjVDHJ5Q=="); // upload package v1.0 - PackagePayload.AddVersion add = new PackagePayload.AddVersion(); + AddPackageVersionRequestBody add = new AddPackageVersionRequestBody(); add.version = "1.0"; - add.pkg = "schemapkg"; add.files = Arrays.asList(FILE1, FILE2); - V2Request req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) - .build(); - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/schemapkg/versions") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(add) + .build() + .process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, @@ -804,17 +795,15 @@ public void testSchemaPlugins() throws Exception { coreProvider.withCore(core -> schemas[0] = core.getLatestSchema()); // upload package v2.0 - add = new PackagePayload.AddVersion(); + add = new AddPackageVersionRequestBody(); add.version = "2.0"; - add.pkg = "schemapkg"; add.files = Arrays.asList(FILE1, FILE2); - req = - new V2Request.Builder("/cluster/package") - .forceV2(true) - .withMethod(SolrRequest.METHOD.POST) - .withPayload(Collections.singletonMap("add", add)) - .build(); - req.process(cluster.getSolrClient()); + new V2Request.Builder("/cluster/package/schemapkg/versions") + .forceV2(true) + .withMethod(SolrRequest.METHOD.POST) + .withPayload(add) + .build() + .process(cluster.getSolrClient()); TestDistribFileStore.assertResponseValues( 10, diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc index 323aa9a3bfe3..4814ebb929a4 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/package-manager-internals.adoc @@ -178,18 +178,19 @@ For example: == API Endpoints -* `GET /api/cluster/package` Get the list of packages -* `POST /api/cluster/package` edit packages -** `add` command: add a version of a package -** `delete` command: delete a version of a package +* `GET /api/cluster/package` Get the list of all packages +* `GET /api/cluster/package/{name}` Get the details of a specific package +* `POST /api/cluster/package/{name}/versions` Add a version of a package +* `DELETE /api/cluster/package/{name}/versions/{version}` Delete a specific version of a package +* `POST /api/cluster/package/{name}/refresh` Refresh a package on all nodes in the cluster === How to Upgrade? -Use the `add` command to add a version that is higher than the current version. +Use the add version endpoint to add a version that is higher than the current version. === How to Downgrade? -Use the `delete` command to delete the highest version and choose the next highest version. +Use the delete version endpoint to delete the highest version and choose the next highest version. === Using Multiple Versions in Parallel @@ -244,11 +245,9 @@ The plugins loaded from packages cannot depend on core level classes. + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"1.0", - "files" :["/mypkg/1.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"1.0","files":["/mypkg/1.0/myplugins.jar"]}' ---- . Verify the created package: @@ -350,11 +349,9 @@ curl http://localhost:8983/api/cluster/filestore/metadata/mypkg/2.0?omitHeader=t + [source,bash] ---- -curl http://localhost:8983/api/cluster/package -H 'Content-type:application/json' -d ' -{"add": { - "package" : "mypkg", - "version":"2.0", - "files" :["/mypkg/2.0/myplugins.jar"]}}' +curl -X POST http://localhost:8983/api/cluster/package/mypkg/versions \ + -H 'Content-type:application/json' \ + -d '{"version":"2.0","files":["/mypkg/2.0/myplugins.jar"]}' ---- . Verify the plugin to see if the correct version of the package is being used: