Skip to content

HIVE-29698: Replace ESRI geometry library with JTS in geospatial plugin #6581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ayushtkn wants to merge 1 commit into
base: master
Choose a base branch
from
+2,569 −1,431
Open

HIVE-29698: Replace ESRI geometry library with JTS in geospatial plugin #6581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@
<druid.version>0.17.1</druid.version>
<esri.version>2.2.4</esri.version>
<flatbuffers.version>1.12.0</flatbuffers.version>
<jts.version>1.20.0</jts.version>
<guava.version>22.0</guava.version>
<groovy.version>2.4.21</groovy.version>
<h2database.version>2.2.220</h2database.version>
Expand Down
14 changes: 10 additions & 4 deletions ql/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,10 +861,16 @@
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>com.esri.geometry</groupId>
<artifactId>esri-geometry-api</artifactId>
<groupId>org.locationtech.jts</groupId>
<artifactId>jts-core</artifactId>
<version>${jts.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.locationtech.jts.io</groupId>
<artifactId>jts-io-common</artifactId>
<version>${jts.version}</version>
<scope>compile</scope>
<version>${esri.version}</version>
</dependency>
<dependency>
<groupId>org.roaringbitmap</groupId>
Expand Down Expand Up @@ -1104,10 +1110,10 @@
<include>org.apache.datasketches:*</include>
<include>org.apache.calcite:*</include>
<include>org.apache.calcite.avatica:*</include>
<include>com.esri.geometry:esri-geometry-api</include>
<include>org.roaringbitmap:RoaringBitmap</include>
<include>com.github.luben:zstd-jni</include>
<include>org.locationtech.jts:jts-core</include>
<include>org.locationtech.jts.io:jts-io-common</include>
<include>org.apache.tez:tez-protobuf-history-plugin</include>
</includes>
</artifactSet>
Expand Down
4 changes: 2 additions & 2 deletions ql/src/java/org/apache/hadoop/hive/ql/io/esriJson/EsriFeature.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hive.ql.io.esriJson;

import com.esri.core.geometry.Geometry;
import org.locationtech.jts.geom.Geometry;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
Expand All @@ -34,7 +34,7 @@ public class EsriFeature {
public Map<String, Object> attributes;

/**
* Geometry associated with this feature
* JTS Geometry associated with this feature
*/
public Geometry geometry;

Expand Down
21 changes: 15 additions & 6 deletions ql/src/java/org/apache/hadoop/hive/ql/io/esriJson/EsriFeatureClass.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,17 @@
*/
package org.apache.hadoop.hive.ql.io.esriJson;

import com.esri.core.geometry.Geometry;
import com.esri.core.geometry.SpatialReference;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import org.apache.hadoop.hive.serde2.esriJson.deserializer.GeometryTypeJsonDeserializer;
import org.apache.hadoop.hive.serde2.esriJson.deserializer.SpatialReferenceJsonDeserializer;
import org.apache.hadoop.hive.serde2.esriJson.serializer.GeometryTypeJsonSerializer;
import org.apache.hadoop.hive.serde2.esriJson.serializer.SpatialReferenceJsonSerializer;

import java.io.IOException;
import java.io.InputStream;
Expand All @@ -39,14 +43,19 @@ public class EsriFeatureClass {
public Map<String, Object> fieldAliases;

/**
* Esri geometry type (Polygon, Point, ...)
* Esri geometry type string (e.g. "esriGeometryPoint", "esriGeometryPolygon").
*/
public Geometry.Type geometryType;
@JsonDeserialize(using = GeometryTypeJsonDeserializer.class)
@JsonSerialize(using = GeometryTypeJsonSerializer.class)
public String geometryType;

/**
* Spatial reference for the feature class (null, if undefined)
* Spatial reference WKID for the feature class (0 if undefined).
* The JSON form is {"wkid": N}; deserialized to a plain int.
*/
public SpatialReference spatialReference;
@JsonDeserialize(using = SpatialReferenceJsonDeserializer.class)
@JsonSerialize(using = SpatialReferenceJsonSerializer.class)
public int spatialReference;

/**
* Array of field definitions (name, type, alias, ...)
Expand Down
17 changes: 5 additions & 12 deletions ql/src/java/org/apache/hadoop/hive/ql/io/esriJson/EsriJsonFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hive.ql.io.esriJson;

import com.esri.core.geometry.Geometry;
import com.esri.core.geometry.SpatialReference;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
Expand All @@ -29,11 +27,8 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.apache.hadoop.hive.serde2.esriJson.deserializer.GeometryJsonDeserializer;
import org.apache.hadoop.hive.serde2.esriJson.deserializer.GeometryTypeJsonDeserializer;
import org.apache.hadoop.hive.serde2.esriJson.deserializer.SpatialReferenceJsonDeserializer;
import org.apache.hadoop.hive.serde2.esriJson.serializer.GeometryJsonSerializer;
import org.apache.hadoop.hive.serde2.esriJson.serializer.GeometryTypeJsonSerializer;
import org.apache.hadoop.hive.serde2.esriJson.serializer.SpatialReferenceJsonSerializer;
import org.locationtech.jts.geom.Geometry;

import java.io.IOException;
import java.io.InputStream;
Expand All @@ -48,14 +43,12 @@ public class EsriJsonFactory {

SimpleModule module = new SimpleModule("EsriJsonModule", new Version(1, 0, 0, null));

// add deserializers and serializers for types that can't be mapped field for field from the JSON
// Geometry: JTS Geometry <-> Esri JSON, handled module-wide because Geometry
// is a well-defined type with no ambiguity.
// geometryType (String) and spatialReference (int) use field-level @JsonDeserialize
// annotations on EsriFeatureClass to avoid intercepting all Strings/ints.
module.addDeserializer(Geometry.class, new GeometryJsonDeserializer());
module.addDeserializer(SpatialReference.class, new SpatialReferenceJsonDeserializer());
module.addDeserializer(Geometry.Type.class, new GeometryTypeJsonDeserializer());

module.addSerializer(Geometry.class, new GeometryJsonSerializer());
module.addSerializer(Geometry.Type.class, new GeometryTypeJsonSerializer());
module.addSerializer(SpatialReference.class, new SpatialReferenceJsonSerializer());

jsonObjectMapper.registerModule(module);
}
Expand Down
18 changes: 9 additions & 9 deletions ql/src/java/org/apache/hadoop/hive/ql/udf/esri/BinUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hive.ql.udf.esri;

import com.esri.core.geometry.Envelope;
import org.locationtech.jts.geom.Envelope;

public class BinUtils {
final long numCols;
Expand Down Expand Up @@ -58,9 +58,9 @@ public long getId(double x, double y) {
* Gets the envelope for the bin ID.
*
* @param binId
* @param envelope
* @return envelope
*/
public void queryEnvelope(long binId, Envelope envelope) {
public Envelope queryEnvelope(long binId) {
long down = binId / numCols;
long over = binId % numCols;

Expand All @@ -69,25 +69,25 @@ public void queryEnvelope(long binId, Envelope envelope) {
double ymax = extentMax - (down * binSize);
double ymin = ymax - binSize;

envelope.setCoords(xmin, ymin, xmax, ymax);
return new Envelope(xmin, xmax, ymin, ymax);
}

/**
* Gets the envelope for the bin that contains the x,y coords.
*
* @param x
* @param y
* @param envelope
* @return envelope
*/
public void queryEnvelope(double x, double y, Envelope envelope) {
public Envelope queryEnvelope(double x, double y) {
double down = (extentMax - y) / binSize;
double over = (x - extentMin) / binSize;

double xmin = extentMin + (over * binSize);
double xmin = extentMin + ((long) over * binSize);
double xmax = xmin + binSize;
double ymax = extentMax - (down * binSize);
double ymax = extentMax - ((long) down * binSize);
double ymin = ymax - binSize;

envelope.setCoords(xmin, ymin, xmax, ymax);
return new Envelope(xmin, xmax, ymin, ymax);
}
}
Loading
Loading