Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,7 @@ package org.apache.texera.amber.core.tuple

import com.fasterxml.jackson.databind.JsonNode
import com.fasterxml.jackson.databind.node.ObjectNode
import org.apache.texera.amber.core.tuple.AttributeTypeUtils.{inferSchemaFromRows, parseField}
import org.apache.texera.amber.util.JSONUtils
import org.apache.texera.amber.util.JSONUtils.{JSONToMap, objectMapper}

import scala.collection.mutable.ArrayBuffer

object TupleUtils {

Expand All @@ -39,53 +35,4 @@ object TupleUtils {
objectNode
}

def json2tuple(json: String): Tuple = {
var fieldNames = Set[String]()

val allFields: ArrayBuffer[Map[String, String]] = ArrayBuffer()

// Parse and flatten once; reused for schema inference and value extraction.
val root: JsonNode = objectMapper.readTree(json)
val data: Map[String, String] = JSONToMap(root)
if (root.isObject) {
fieldNames = fieldNames.++(data.keySet)
allFields += data
}

val sortedFieldNames = fieldNames.toList

val attributeTypes = inferSchemaFromRows(allFields.iterator.map(fields => {
val result = ArrayBuffer[Object]()
for (fieldName <- sortedFieldNames) {
if (fields.contains(fieldName)) {
result += fields(fieldName)
} else {
result += null
}
}
result.toArray
}))

val schema = Schema(
sortedFieldNames.indices
.map(i => new Attribute(sortedFieldNames(i), attributeTypes(i)))
.toList
)

try {
val fields = scala.collection.mutable.ArrayBuffer.empty[Any]

for (fieldName <- schema.getAttributeNames) {
if (data.contains(fieldName)) {
fields += parseField(data(fieldName), schema.getAttribute(fieldName).getType)
} else {
fields += null
}
}
Tuple.builder(schema).addSequentially(fields.toArray).build()
} catch {
case e: Exception => throw e
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

package org.apache.texera.amber.core.tuple

import org.apache.texera.amber.core.tuple.TupleUtils.{json2tuple, tuple2json}
import org.scalatest.flatspec.AnyFlatSpec

import java.sql.Timestamp
Expand Down Expand Up @@ -104,22 +103,6 @@ class TupleSpec extends AnyFlatSpec {
assert(outputTuple.length == 2);
}

it should "produce identical strings" in {
val inputSchema =
Schema().add(stringAttribute).add(integerAttribute).add(boolAttribute)
val inputTuple = Tuple
.builder(inputSchema)
.add(integerAttribute, 1)
.add(stringAttribute, "string-attr")
.add(boolAttribute, true)
.build()

val line = tuple2json(inputTuple.schema, inputTuple.fieldVals).toString
val newTuple = json2tuple(line)
assert(line == tuple2json(newTuple.schema, newTuple.fieldVals).toString)

}

it should "calculate hash" in {
val inputSchema =
Schema()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,48 +63,4 @@ class TupleUtilsSpec extends AnyFlatSpec {
val node = TupleUtils.tuple2json(new Schema(), Array.empty[Any])
assert(node.size() == 0)
}

// --- json2tuple ------------------------------------------------------------

"TupleUtils.json2tuple" should "infer a schema from a flat JSON object's keys and types" in {
val tuple = TupleUtils.json2tuple("""{"name": "bob", "age": 30}""")
val names = tuple.getSchema.getAttributeNames.toSet
assert(names == Set("name", "age"))
assert(tuple.getField[Any]("name") == "bob")
// age is parsed via inferSchemaFromRows; the inferred type for "30" is
// a numeric type — assert we can read the field rather than locking in
// the precise inferred AttributeType.
assert(tuple.getField[Any]("age").toString == "30")
}

it should "round-trip a schema-and-values through tuple2json → json2tuple" in {
val schema = new Schema(
new Attribute("city", AttributeType.STRING),
new Attribute("score", AttributeType.INTEGER)
)
val original = TupleUtils.tuple2json(schema, Array[Any]("Irvine", Int.box(42))).toString
val parsed = TupleUtils.json2tuple(original)
val reSerialized =
TupleUtils.tuple2json(parsed.getSchema, parsed.getFields.toArray.asInstanceOf[Array[Any]])
// The exact column order isn't part of the json2tuple contract (it builds
// schemaFieldNames from a Set), so compare by JSON-tree equality.
val mapper = org.apache.texera.amber.util.JSONUtils.objectMapper
assert(mapper.readTree(reSerialized.toString) == mapper.readTree(original))
}

it should "drop non-object roots (e.g. a JSON array) into an empty tuple" in {
// The implementation only collects fields when the root `isObject`. A
// non-object root leaves `fieldNames` empty, so the result is a tuple
// over an empty schema with no fields — observed contract is no-throw,
// empty result.
val tuple = TupleUtils.json2tuple("""[1, 2, 3]""")
assert(tuple.getSchema.getAttributes.isEmpty)
assert(tuple.getFields.isEmpty)
}

it should "throw when given malformed JSON" in {
intercept[Exception] {
TupleUtils.json2tuple("{ this is not json }")
}
}
}
Loading