diff --git a/core/src/main/java/de/bwaldvogel/mongo/backend/Utils.java b/core/src/main/java/de/bwaldvogel/mongo/backend/Utils.java index b4676c5f..ead8d799 100644 --- a/core/src/main/java/de/bwaldvogel/mongo/backend/Utils.java +++ b/core/src/main/java/de/bwaldvogel/mongo/backend/Utils.java @@ -476,6 +476,8 @@ public static String describeType(Class type) { return "long"; } else if (Double.class.isAssignableFrom(type)) { return "double"; + } else if (Boolean.class.isAssignableFrom(type)) { + return "bool"; } else if (ObjectId.class.isAssignableFrom(type)) { return "objectId"; } else if (Instant.class.isAssignableFrom(type)) { diff --git a/core/src/main/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStage.java b/core/src/main/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStage.java index aec891a3..01b5dc2a 100644 --- a/core/src/main/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStage.java +++ b/core/src/main/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStage.java @@ -1,17 +1,24 @@ package de.bwaldvogel.mongo.backend.aggregation.stage; -import static java.util.stream.Collectors.toList; +import static de.bwaldvogel.mongo.backend.Constants.ID_FIELD; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; +import java.util.Deque; import java.util.List; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Stream; import de.bwaldvogel.mongo.MongoCollection; import de.bwaldvogel.mongo.MongoDatabase; +import de.bwaldvogel.mongo.backend.Missing; import de.bwaldvogel.mongo.backend.Utils; +import de.bwaldvogel.mongo.backend.ValueComparator; +import de.bwaldvogel.mongo.backend.aggregation.Expression; import de.bwaldvogel.mongo.bson.Document; -import de.bwaldvogel.mongo.exception.FailedToParseException; +import de.bwaldvogel.mongo.exception.MongoServerError; public class GraphLookupStage extends AbstractLookupStage { @@ -24,6 +31,7 @@ public class GraphLookupStage extends AbstractLookupStage { private static final Set CONFIGURATION_KEYS = Set.of( FROM, START_WITH, CONNECT_FROM_FIELD, CONNECT_TO_FIELD, AS, MAX_DEPTH, DEPTH_FIELD); + private final Object startWith; private final String connectFromField; private final String connectToField; private final String asField; @@ -35,8 +43,10 @@ public class GraphLookupStage extends AbstractLookupStage { public GraphLookupStage(Document configuration, MongoDatabase mongoDatabase) { String from = readStringConfigurationProperty(configuration, FROM); collection = mongoDatabase.resolveCollection(from, false); - readStringConfigurationProperty(configuration, FROM); - readVariableConfigurationProperty(configuration, START_WITH); + if (!configuration.containsKey(START_WITH)) { + throw new MongoServerError(40105, "$graphLookup requires 'from', 'as', 'startWith', 'connectFromField', and 'connectToField' to be specified."); + } + startWith = configuration.get(START_WITH); connectFromField = readStringConfigurationProperty(configuration, CONNECT_FROM_FIELD); connectToField = readStringConfigurationProperty(configuration, CONNECT_TO_FIELD); asField = readStringConfigurationProperty(configuration, AS); @@ -50,37 +60,22 @@ public String name() { return "$graphLookup"; } - Integer readOptionalIntegerConfigurationProperty(Document configuration, String name) { + private Integer readOptionalIntegerConfigurationProperty(Document configuration, String name) { Object value = configuration.get(name); - if (value == null) { - return null; - } - if (value instanceof Integer integer) { - return integer; - } - throw new FailedToParseException("'" + name + "' option to \" + stageName + \" must be a integer, but was type " + Utils.describeType(value)); + return switch (value) { + case null -> null; + case Integer integer -> integer; + default -> throw new MongoServerError(40100, name + " must be numeric, found type: " + Utils.describeType(value)); + }; } - String readOptionalStringConfigurationProperty(Document configuration, String name) { + private String readOptionalStringConfigurationProperty(Document configuration, String name) { Object value = configuration.get(name); - if (value == null) { - return null; - } - if (value instanceof String string) { - return string; - } - throw new FailedToParseException("'" + name + "' option to \" + stageName + \" must be a string, but was type " + Utils.describeType(value)); - } - - void readVariableConfigurationProperty(Document configuration, String name) { - Object value = configuration.get(name); - if (value == null) { - throw new FailedToParseException("missing '" + name + "' option to $graphLookup stage specification: " + configuration); - } - if (value instanceof String) { - return; - } - throw new FailedToParseException("'" + name + "' option to $graphLookup must be a string, but was type " + Utils.describeType(value)); + return switch (value) { + case null -> null; + case String string -> string; + default -> throw new MongoServerError(40103, "expected string as argument for " + name + ", found: " + Utils.describeType(value)); + }; } @Override @@ -89,37 +84,54 @@ public Stream apply(Stream stream) { } private Document connectRemoteDocument(Document document) { - Object value = document.get(connectFromField); - List documentList = new ArrayList<>(); - findLinkedDocuments(0, documentList, value); + Object value = Expression.evaluateDocument(startWith, document); Document result = document.clone(); - result.put(asField, documentList); + result.put(asField, findLinkedDocuments(value)); return result; } - private List findLinkedDocuments(long depth, final List linked, Object value) { + private List findLinkedDocuments(Object value) { + List linked = new ArrayList<>(); + Deque pending = new ArrayDeque<>(); + enqueueValues(pending, value, 0); + Set visitedIds = new TreeSet<>(ValueComparator.asc()); - if (maxDepth != null && depth > maxDepth) { - return linked; + while (!pending.isEmpty()) { + SearchValue current = pending.removeFirst(); + if (maxDepth != null && current.depth() > maxDepth) { + continue; + } + for (Document newDocument : lookupDocuments(current.value())) { + if (!visitedIds.add(newDocument.get(ID_FIELD))) { + continue; + } + Object newValue = Utils.getSubdocumentValueCollectionAware(newDocument, connectFromField); + enqueueValues(pending, newValue, current.depth() + 1); + Document result = newDocument.clone(); + if (depthField != null) { + result.put(depthField, current.depth()); + } + linked.add(result); + } } + return linked; + } - if (value instanceof List list) { - return list.stream() - .flatMap(item -> findLinkedDocuments(depth + 1, linked, item).stream()) - .collect(toList()); - } - Document query = new Document(connectToField, value); + private List lookupDocuments(Object value) { + // Unlike a plain equality query, a null graph link must not match missing fields. + Object condition = value == null ? new Document("$eq", null).append("$exists", true) : value; + return collection.handleQueryAsStream(new Document(connectToField, condition)).toList(); + } - List newlyLinkedDocuments = collection.handleQueryAsStream(query).collect(toList()); - for (Document newDocument : newlyLinkedDocuments) { - Object newValue = newDocument.get(connectFromField); - if (depthField != null) { - newDocument.put(depthField, depth); + private static void enqueueValues(Deque pending, Object value, long depth) { + List values = value instanceof List list ? list : Collections.singletonList(value); + for (Object item : values) { + if (!(item instanceof Missing)) { + pending.addLast(new SearchValue(item, depth)); } - linked.add(0, newDocument); - findLinkedDocuments(depth + 1, linked, newValue); } - return linked; } + private record SearchValue(Object value, long depth) {} + } diff --git a/core/src/test/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStageTest.java b/core/src/test/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStageTest.java index 52be57cc..c13efe99 100644 --- a/core/src/test/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStageTest.java +++ b/core/src/test/java/de/bwaldvogel/mongo/backend/aggregation/stage/GraphLookupStageTest.java @@ -2,6 +2,7 @@ import static de.bwaldvogel.mongo.TestUtils.json; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.list; import static org.mockito.Mockito.when; import java.util.stream.Stream; @@ -33,13 +34,37 @@ void testGraphLookupObjectThatHasLinkedDocuments() { configureEmployeesCollection("name: 'Mike'", "_id: 1, name: 'Mike'"); Document document = json("name: 'Bob', manager: 'Dave'"); - Stream result = graphLookupStage.apply(Stream.of(document)); + Document result = graphLookupStage.apply(Stream.of(document)).findFirst().orElseThrow(); - assertThat(result).containsExactly( - json("name: 'Bob', manager: 'Dave', managers: [{_id: 1, name: 'Mike', 'order': NumberLong(1)}, {_id: 2, name: 'Dave', 'manager': 'Mike', 'order': NumberLong(0)}]") + assertThat(result).containsEntry("name", "Bob").containsEntry("manager", "Dave"); + assertThat(result.get("managers")).asInstanceOf(list(Document.class)).containsExactlyInAnyOrder( + json("_id: 1, name: 'Mike', order: NumberLong(1)"), + json("_id: 2, name: 'Dave', manager: 'Mike', order: NumberLong(0)") ); } + @Test + void testGraphLookupDoesNotModifyInputOrLinkedDocuments() { + GraphLookupStage graphLookupStage = buildGraphLookupStage(""" + from: 'employees', + startWith: '$manager', + connectFromField: 'manager', + connectToField: 'name', + as: 'managers', + depthField: 'order' + """); + Document employee = json("_id: 2, name: 'Dave', order: 42"); + when(employeesCollection.handleQueryAsStream(json("name: 'Dave'"))) + .thenAnswer(invocation -> Stream.of(employee)); + Document input = json("name: 'Bob', manager: 'Dave', managers: ['previous value']"); + + assertThat(graphLookupStage.apply(Stream.of(input))).containsExactly( + json("name: 'Bob', manager: 'Dave', managers: [{_id: 2, name: 'Dave', order: NumberLong(0)}]") + ); + assertThat(input).isEqualTo(json("name: 'Bob', manager: 'Dave', managers: ['previous value']")); + assertThat(employee).isEqualTo(json("_id: 2, name: 'Dave', order: 42")); + } + private GraphLookupStage buildGraphLookupStage(String jsonDocument) { return new GraphLookupStage(json(jsonDocument), database); } diff --git a/test-common/src/main/java/de/bwaldvogel/mongo/backend/AbstractAggregationTest.java b/test-common/src/main/java/de/bwaldvogel/mongo/backend/AbstractAggregationTest.java index d5e5c0a6..5c9640e2 100644 --- a/test-common/src/main/java/de/bwaldvogel/mongo/backend/AbstractAggregationTest.java +++ b/test-common/src/main/java/de/bwaldvogel/mongo/backend/AbstractAggregationTest.java @@ -23,6 +23,7 @@ import org.bson.types.ObjectId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -35,6 +36,8 @@ public abstract class AbstractAggregationTest extends AbstractTest { + private static final int TEST_TIMEOUT_SECONDS = 30; + @BeforeEach void dropAllDatabasesBeforeEachTest() { dropAllDatabases(); @@ -571,6 +574,17 @@ void testAggregateWithProjection() { ); } + // https://github.com/bwaldvogel/mongo-java-server/issues/218 + @Test + void testAggregateWithProjectionOfUserIdsFromArray() { + collection.insertOne(json("_id: 1, user: [{_id: 10, name: 'Alice'}, {_id: 20, name: 'Bob'}], other: 'excluded'")); + + List pipeline = jsonList("$project: {user: '$user._id'}"); + + assertThat(collection.aggregate(pipeline)) + .containsExactly(json("_id: 1, user: [10, 20]")); + } + // https://github.com/bwaldvogel/mongo-java-server/issues/121 @Test void testAggregateWithNestedExclusiveProjection() { @@ -1704,6 +1718,269 @@ void testAggregateWithGraphLookup() { ); } + @Test + void testAggregateWithGraphLookupRequiresStartWith() { + List pipeline = jsonList(""" + $graphLookup: { + from: 'testcoll', + connectFromField: 'manager', + connectToField: 'name', + as: 'managers' + } + """); + + assertThatExceptionOfType(MongoCommandException.class) + .isThrownBy(() -> collection.aggregate(pipeline).first()) + .withMessageContaining("Command execution failed on MongoDB server with error 40105 (Location40105): " + + "'$graphLookup requires 'from', 'as', 'startWith', 'connectFromField', and 'connectToField' to be specified.'"); + } + + private static Stream graphLookupInvalidOptionalConfigurations() { + return Stream.of( + Arguments.of("maxDepth", true, "40100 (Location40100): 'maxDepth must be numeric, found type: bool'"), + Arguments.of("depthField", 42, "40103 (Location40103): 'expected string as argument for depthField, found: int'") + ); + } + + @ParameterizedTest + @MethodSource("graphLookupInvalidOptionalConfigurations") + void testAggregateWithGraphLookupRejectsInvalidOptionalConfiguration(String option, Object value, String expectedMessage) { + Document graphLookup = json(""" + from: 'testcoll', + startWith: '$manager', + connectFromField: 'manager', + connectToField: 'name', + as: 'managers' + """); + graphLookup.put(option, value); + List pipeline = List.of(new Document("$graphLookup", graphLookup)); + + assertThatExceptionOfType(MongoCommandException.class) + .isThrownBy(() -> collection.aggregate(pipeline).first()) + .withMessageContaining("Command execution failed on MongoDB server with error " + expectedMessage); + } + + // https://github.com/bwaldvogel/mongo-java-server/issues/218 + @Test + void testAggregateWithGraphLookupThroughNestedArrayField() { + collection.insertOne(json("_id: 1, employeeId: 'john', directReports: [{employeeId: 'paul'}, {employeeId: 'george'}]")); + collection.insertOne(json("_id: 2, employeeId: 'paul', directReports: [{employeeId: 'ringo'}]")); + collection.insertOne(json("_id: 3, employeeId: 'george', directReports: []")); + collection.insertOne(json("_id: 4, employeeId: 'ringo', directReports: []")); + collection.insertOne(json("_id: 5, employeeId: 'unrelated', directReports: []")); + + List pipeline = jsonList("$match: {employeeId: 'john'}", + """ + $graphLookup: { + from: 'testcoll', + startWith: '$directReports.employeeId', + connectFromField: 'directReports.employeeId', + connectToField: 'employeeId', + as: 'allReports' + } + """); + + Document result = collection.aggregate(pipeline).first(); + + assertThat(result).isNotNull(); + assertThat(result.getList("allReports", Document.class)).containsExactlyInAnyOrder( + json("_id: 2, employeeId: 'paul', directReports: [{employeeId: 'ringo'}]"), + json("_id: 3, employeeId: 'george', directReports: []"), + json("_id: 4, employeeId: 'ringo', directReports: []") + ); + } + + private static Stream graphLookupMaxDepths() { + return Stream.of(null, 0, 1, 2); + } + + @ParameterizedTest + @MethodSource("graphLookupMaxDepths") + @Timeout(TEST_TIMEOUT_SECONDS) + void testAggregateWithGraphLookupStartWithExpressionAndCycles(Integer maxDepth) { + collection.insertOne(json("_id: 1, employeeId: 'john', initialReports: [{employeeId: 'paul'}, {employeeId: 'george'}]")); + collection.insertOne(json("_id: 2, employeeId: 'paul', directReports: [{employeeId: 'ringo'}, {employeeId: 'george'}]")); + collection.insertOne(json("_id: 3, employeeId: 'george', directReports: [{employeeId: 'ringo'}]")); + collection.insertOne(json("_id: 4, employeeId: 'ringo', directReports: [{employeeId: 'paul'}]")); + collection.insertOne(json("_id: 5, employeeId: 'unrelated', directReports: []")); + + Document graphLookup = json(""" + from: 'testcoll', + startWith: {$ifNull: ['$initialReports.employeeId', []]}, + connectFromField: 'directReports.employeeId', + connectToField: 'employeeId', + as: 'allReports', + depthField: 'depth' + """); + if (maxDepth != null) { + graphLookup.put("maxDepth", maxDepth); + } + List pipeline = List.of( + json("$match: {employeeId: 'john'}"), + new Document("$graphLookup", graphLookup) + ); + + List expectedReports = new ArrayList<>(); + expectedReports.add(json("_id: 2, employeeId: 'paul', directReports: [{employeeId: 'ringo'}, {employeeId: 'george'}], depth: NumberLong(0)")); + expectedReports.add(json("_id: 3, employeeId: 'george', directReports: [{employeeId: 'ringo'}], depth: NumberLong(0)")); + if (maxDepth == null || maxDepth >= 1) { + expectedReports.add(json("_id: 4, employeeId: 'ringo', directReports: [{employeeId: 'paul'}], depth: NumberLong(1)")); + } + + Document result = collection.aggregate(pipeline).first(); + + assertThat(result).isNotNull(); + assertThat(result.getList("allReports", Document.class)) + .containsExactlyInAnyOrderElementsOf(expectedReports); + } + + private static Stream graphLookupEmptyStartDocuments() { + return Stream.of( + "_id: 1, employeeId: 'john'", + "_id: 1, employeeId: 'john', directReports: []", + "_id: 1, employeeId: 'john', directReports: [{name: 'no employeeId'}]", + "_id: 1, employeeId: 'john', directReports: [{employeeId: 'unknown'}]" + ); + } + + @ParameterizedTest + @MethodSource("graphLookupEmptyStartDocuments") + void testAggregateWithGraphLookupWithoutMatchingStartValues(String input) { + Document employee = json(input); + collection.insertOne(employee); + collection.insertOne(json("_id: 2, employeeId: null")); + collection.insertOne(json("_id: 3, name: 'missing employeeId'")); + + List pipeline = jsonList("$match: {_id: 1}", """ + $graphLookup: { + from: 'testcoll', + startWith: '$directReports.employeeId', + connectFromField: 'directReports.employeeId', + connectToField: 'employeeId', + as: 'allReports' + } + """); + + assertThat(collection.aggregate(pipeline)) + .containsExactly(new Document(employee).append("allReports", List.of())); + } + + private static Stream graphLookupStartExpressions() { + return Stream.of( + Arguments.of(2, 3), + Arguments.of(List.of(2, 2), 3), + Arguments.of(json("$literal: 2"), 3), + Arguments.of(json("$add: [1, 1]"), 3), + Arguments.of(json("$literal: '$literalValue'"), 4), + Arguments.of(null, 2) + ); + } + + @ParameterizedTest + @MethodSource("graphLookupStartExpressions") + void testAggregateWithGraphLookupLiteralAndComputedStartValues(Object startWith, int expectedId) { + collection.insertOne(json("_id: 1, key: 'root'")); + collection.insertOne(json("_id: 2, key: null, next: []")); + collection.insertOne(json("_id: 3, key: 2, next: []")); + collection.insertOne(json("_id: 4, key: '$literalValue', next: []")); + collection.insertOne(json("_id: 5, next: []")); + + Document graphLookup = json(""" + from: 'testcoll', + connectFromField: 'next', + connectToField: 'key', + as: 'matches' + """); + graphLookup.put("startWith", startWith); + List pipeline = List.of(json("$match: {_id: 1}"), new Document("$graphLookup", graphLookup)); + + Document result = collection.aggregate(pipeline).first(); + + assertThat(result).isNotNull(); + assertThat(result.getList("matches", Document.class)) + .containsExactly(collection.find(new Document("_id", expectedId)).first()); + } + + @Test + void testAggregateWithGraphLookupThroughSparseArrays() { + collection.insertOne(json("_id: 1, employeeId: 'john', directReports: [{}, null, 17, {employeeId: 'paul'}]")); + collection.insertOne(json("_id: 2, employeeId: 'paul', directReports: [{}, {employeeId: 'george'}, null]")); + collection.insertOne(json("_id: 3, employeeId: 'george'")); + collection.insertOne(json("_id: 4, employeeId: null")); + collection.insertOne(json("_id: 5, name: 'missing employeeId'")); + + List pipeline = jsonList("$match: {_id: 1}", """ + $graphLookup: { + from: 'testcoll', + startWith: '$directReports.employeeId', + connectFromField: 'directReports.employeeId', + connectToField: 'employeeId', + as: 'allReports' + } + """); + + Document result = collection.aggregate(pipeline).first(); + + assertThat(result).isNotNull(); + assertThat(result.getList("allReports", Document.class)).containsExactlyInAnyOrder( + json("_id: 2, employeeId: 'paul', directReports: [{}, {employeeId: 'george'}, null]"), + json("_id: 3, employeeId: 'george'") + ); + } + + @Test + void testAggregateWithGraphLookupTracksDepthSeparatelyForEachInput() { + collection.insertOne(json("_id: 1, start: 10")); + collection.insertOne(json("_id: 2, start: 20")); + collection.insertOne(json("_id: 10, next: []")); + collection.insertOne(json("_id: 20, next: 10")); + + List pipeline = jsonList("$match: {_id: {$in: [1, 2]}}", "$sort: {_id: 1}", """ + $graphLookup: { + from: 'testcoll', + startWith: '$start', + connectFromField: 'next', + connectToField: '_id', + as: 'matches', + depthField: 'depth' + } + """); + + assertThat(collection.aggregate(pipeline).map(withSortedDocuments("matches"))) + .containsExactly( + json("_id: 1, start: 10, matches: [{_id: 10, next: [], depth: NumberLong(0)}]"), + json("_id: 2, start: 20, matches: [{_id: 10, next: [], depth: NumberLong(1)}, {_id: 20, next: 10, depth: NumberLong(0)}]") + ); + assertThat(collection.find(json("_id: {$in: [10, 20]}"))) + .containsExactlyInAnyOrder(json("_id: 10, next: []"), json("_id: 20, next: 10")); + } + + @Test + @Timeout(TEST_TIMEOUT_SECONDS) + void testAggregateWithGraphLookupSelfLoopAndSharedConnectToValue() { + collection.insertOne(json("_id: 1, key: 'self', next: 'self'")); + collection.insertOne(json("_id: 2, key: 'self', next: 'self'")); + + List pipeline = jsonList("$match: {_id: 1}", """ + $graphLookup: { + from: 'testcoll', + startWith: '$next', + connectFromField: 'next', + connectToField: 'key', + as: 'matches', + depthField: 'depth' + } + """); + + Document result = collection.aggregate(pipeline).first(); + + assertThat(result).isNotNull(); + assertThat(result.getList("matches", Document.class)).containsExactlyInAnyOrder( + json("_id: 1, key: 'self', next: 'self', depth: NumberLong(0)"), + json("_id: 2, key: 'self', next: 'self', depth: NumberLong(0)") + ); + } + @Test void testObjectToArrayExpression() { List pipeline = jsonList("$project: {_id: 1, a: {$objectToArray: '$value'}}");