Search in sources :

Example 1 with HARD_MAX_LIMIT

use of org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.HARD_MAX_LIMIT in project janusgraph by JanusGraph.

the class JanusGraphTest method testGraphCentricQueryProfilingWithLimitAdjusting.

@Test
public void testGraphCentricQueryProfilingWithLimitAdjusting() throws BackendException {
    Runnable dataLoader = () -> {
        final PropertyKey name = makeKey("name", String.class);
        final JanusGraphIndex compositeNameIndex = mgmt.buildIndex("nameIdx", Vertex.class).addKey(name).buildCompositeIndex();
        finishSchema();
        newTx();
        for (int i = 0; i < 3000; i++) {
            tx.addVertex("name", "bob");
        }
        tx.commit();
    };
    clopen(option(ADJUST_LIMIT), false, option(HARD_MAX_LIMIT), 100000);
    dataLoader.run();
    newTx();
    Metrics mCompSingle = tx.traversal().V().has("name", "bob").profile().next().getMetrics(0);
    assertEquals(2, mCompSingle.getNested().size());
    Metrics nested = (Metrics) mCompSingle.getNested().toArray()[1];
    Map<String, String> nameIdxAnnotations = new HashMap() {

        {
            put("condition", "(name = bob)");
            put("orders", "[]");
            put("isFitted", "true");
            put("isOrdered", "true");
            // 100000 is HARD_MAX_LIMIT
            put("query", "multiKSQ[1]@100000");
            put("index", "nameIdx");
        }
    };
    assertEquals(nameIdxAnnotations, nested.getAnnotations());
    List<Metrics> backendQueryMetrics = nested.getNested().stream().map(m -> (Metrics) m).collect(Collectors.toList());
    assertEquals(1, backendQueryMetrics.size());
    Map<String, String> backendAnnotations = new HashMap() {

        {
            put("query", "nameIdx:multiKSQ[1]@100000");
            put("limit", 100000);
        }
    };
    assertEquals(backendAnnotations, backendQueryMetrics.get(0).getAnnotations());
    assertTrue(backendQueryMetrics.get(0).getDuration(TimeUnit.MICROSECONDS) > 0);
    close();
    JanusGraphFactory.drop(graph);
    clopen(option(ADJUST_LIMIT), false, option(HARD_MAX_LIMIT), Integer.MAX_VALUE);
    dataLoader.run();
    newTx();
    mCompSingle = tx.traversal().V().has("name", "bob").profile().next().getMetrics(0);
    assertEquals(2, mCompSingle.getNested().size());
    nested = (Metrics) mCompSingle.getNested().toArray()[1];
    nameIdxAnnotations = new HashMap() {

        {
            put("condition", "(name = bob)");
            put("orders", "[]");
            put("isFitted", "true");
            put("isOrdered", "true");
            put("query", "multiKSQ[1]");
            put("index", "nameIdx");
        }
    };
    assertEquals(nameIdxAnnotations, nested.getAnnotations());
    backendQueryMetrics = nested.getNested().stream().map(m -> (Metrics) m).collect(Collectors.toList());
    assertEquals(1, backendQueryMetrics.size());
    backendAnnotations = new HashMap() {

        {
            put("query", "nameIdx:multiKSQ[1]");
        }
    };
    assertEquals(backendAnnotations, backendQueryMetrics.get(0).getAnnotations());
    assertTrue(backendQueryMetrics.get(0).getDuration(TimeUnit.MICROSECONDS) > 0);
    close();
    JanusGraphFactory.drop(graph);
    clopen(option(ADJUST_LIMIT), true);
    dataLoader.run();
    newTx();
    mCompSingle = tx.traversal().V().has("name", "bob").profile().next().getMetrics(0);
    assertEquals("JanusGraphStep([],[name.eq(bob)])", mCompSingle.getName());
    assertTrue(mCompSingle.getDuration(TimeUnit.MICROSECONDS) > 0);
    assertEquals(2, mCompSingle.getNested().size());
    nested = (Metrics) mCompSingle.getNested().toArray()[0];
    assertEquals(QueryProfiler.CONSTRUCT_GRAPH_CENTRIC_QUERY, nested.getName());
    assertTrue(nested.getDuration(TimeUnit.MICROSECONDS) > 0);
    nested = (Metrics) mCompSingle.getNested().toArray()[1];
    assertEquals(QueryProfiler.GRAPH_CENTRIC_QUERY, nested.getName());
    assertTrue(nested.getDuration(TimeUnit.MICROSECONDS) > 0);
    nameIdxAnnotations = new HashMap() {

        {
            put("condition", "(name = bob)");
            put("orders", "[]");
            put("isFitted", "true");
            put("isOrdered", "true");
            put("query", "multiKSQ[1]@4000");
            put("index", "nameIdx");
        }
    };
    assertEquals(nameIdxAnnotations, nested.getAnnotations());
    backendQueryMetrics = nested.getNested().stream().map(m -> (Metrics) m).collect(Collectors.toList());
    assertEquals(3, backendQueryMetrics.size());
    int limit = 1000;
    // due to LimitAdjustingIterator, there are three backend queries with limits 1000, 2000, and 4000, respectively.
    for (Metrics backendQueryMetric : backendQueryMetrics) {
        int queryLimit = limit;
        backendAnnotations = new HashMap() {

            {
                put("query", "nameIdx:multiKSQ[1]@" + queryLimit);
                put("limit", queryLimit);
            }
        };
        assertEquals(backendAnnotations, backendQueryMetric.getAnnotations());
        assertTrue(backendQueryMetric.getDuration(TimeUnit.MICROSECONDS) > 0);
        limit = limit * 2;
    }
}
Also used : ManagementUtil(org.janusgraph.core.util.ManagementUtil) PredicateCondition(org.janusgraph.graphdb.query.condition.PredicateCondition) CacheVertex(org.janusgraph.graphdb.vertices.CacheVertex) BasicVertexCentricQueryBuilder(org.janusgraph.graphdb.query.vertex.BasicVertexCentricQueryBuilder) DisableDefaultSchemaMaker(org.janusgraph.core.schema.DisableDefaultSchemaMaker) JanusGraphAssert.getStepMetrics(org.janusgraph.testutil.JanusGraphAssert.getStepMetrics) Cardinality(org.janusgraph.core.Cardinality) Serializer(org.janusgraph.graphdb.database.serialize.Serializer) JanusGraphVertexStep(org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphVertexStep) Duration(java.time.Duration) Map(java.util.Map) JanusGraphAssert.assertNotContains(org.janusgraph.testutil.JanusGraphAssert.assertNotContains) Path(java.nio.file.Path) Metrics(org.apache.tinkerpop.gremlin.process.traversal.util.Metrics) EnumSet(java.util.EnumSet) IndexRepairJob(org.janusgraph.graphdb.olap.job.IndexRepairJob) PropertyKey(org.janusgraph.core.PropertyKey) OUT(org.apache.tinkerpop.gremlin.structure.Direction.OUT) JanusGraphPredicateUtils(org.janusgraph.graphdb.query.JanusGraphPredicateUtils) EdgeSerializer(org.janusgraph.graphdb.database.EdgeSerializer) Arguments(org.junit.jupiter.params.provider.Arguments) VERBOSE_TX_RECOVERY(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.VERBOSE_TX_RECOVERY) CountDownLatch(java.util.concurrent.CountDownLatch) Stream(java.util.stream.Stream) LogProcessorFramework(org.janusgraph.core.log.LogProcessorFramework) ADJUST_LIMIT(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.ADJUST_LIMIT) RELATION(org.janusgraph.graphdb.internal.RelationCategory.RELATION) SCHEMA_CONSTRAINTS(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.SCHEMA_CONSTRAINTS) Assertions.fail(org.junit.jupiter.api.Assertions.fail) Assertions.assertNotNull(org.junit.jupiter.api.Assertions.assertNotNull) MultiCondition(org.janusgraph.graphdb.query.condition.MultiCondition) FORCE_INDEX_USAGE(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.FORCE_INDEX_USAGE) CUSTOM_ATTRIBUTE_CLASS(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.CUSTOM_ATTRIBUTE_CLASS) JanusGraphAssert.assertCount(org.janusgraph.testutil.JanusGraphAssert.assertCount) TestGraphConfigs(org.janusgraph.testutil.TestGraphConfigs) Supplier(java.util.function.Supplier) Lists(com.google.common.collect.Lists) Change(org.janusgraph.core.log.Change) GhostVertexRemover(org.janusgraph.graphdb.olap.job.GhostVertexRemover) StreamSupport(java.util.stream.StreamSupport) QueryUtil(org.janusgraph.graphdb.query.QueryUtil) LOG_SEND_DELAY(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.LOG_SEND_DELAY) ALLOW_STALE_CONFIG(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.ALLOW_STALE_CONFIG) T(org.apache.tinkerpop.gremlin.structure.T) ExecutionException(java.util.concurrent.ExecutionException) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) IDS_STORE_NAME(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.IDS_STORE_NAME) JanusGraphVertexQuery(org.janusgraph.core.JanusGraphVertexQuery) Preconditions(com.google.common.base.Preconditions) VertexLabel(org.janusgraph.core.VertexLabel) JanusGraphConfigurationException(org.janusgraph.core.JanusGraphConfigurationException) Log(org.janusgraph.diskstorage.log.Log) Date(java.util.Date) Assertions.assertNotEquals(org.junit.jupiter.api.Assertions.assertNotEquals) Random(java.util.Random) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SchemaStatus(org.janusgraph.core.schema.SchemaStatus) NotImplementedException(org.apache.commons.lang.NotImplementedException) AbstractEdge(org.janusgraph.graphdb.relations.AbstractEdge) TextP(org.apache.tinkerpop.gremlin.process.traversal.TextP) MethodSource(org.junit.jupiter.params.provider.MethodSource) Multiplicity(org.janusgraph.core.Multiplicity) RelationType(org.janusgraph.core.RelationType) ElementLifeCycle(org.janusgraph.graphdb.internal.ElementLifeCycle) FeatureFlag(org.janusgraph.testutil.FeatureFlag) Collection(java.util.Collection) LogTxMeta(org.janusgraph.graphdb.database.log.LogTxMeta) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) JanusGraphAssert.assertEmpty(org.janusgraph.testutil.JanusGraphAssert.assertEmpty) TestCategory(org.janusgraph.TestCategory) Test(org.junit.jupiter.api.Test) Objects(java.util.Objects) RelationIdentifier(org.janusgraph.graphdb.relations.RelationIdentifier) ImplicitKey(org.janusgraph.graphdb.types.system.ImplicitKey) InternalRelationType(org.janusgraph.graphdb.internal.InternalRelationType) STORAGE_BATCH(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_BATCH) Order.desc(org.apache.tinkerpop.gremlin.process.traversal.Order.desc) STORAGE_BACKEND(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_BACKEND) JanusGraphFeature(org.janusgraph.testutil.JanusGraphFeature) JanusGraphQuery(org.janusgraph.core.JanusGraphQuery) HashSet(java.util.HashSet) VertexProperty(org.apache.tinkerpop.gremlin.structure.VertexProperty) ImmutableList(com.google.common.collect.ImmutableList) Message(org.janusgraph.diskstorage.log.Message) TRANSACTION_LOG(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.TRANSACTION_LOG) RelationCategory(org.janusgraph.graphdb.internal.RelationCategory) Arguments.arguments(org.junit.jupiter.params.provider.Arguments.arguments) GraphOfTheGodsFactory(org.janusgraph.example.GraphOfTheGodsFactory) Logger(org.slf4j.Logger) BaseVertexLabel(org.janusgraph.graphdb.types.system.BaseVertexLabel) VertexList(org.janusgraph.core.VertexList) GraphTraversal(org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal) Contain(org.janusgraph.core.attribute.Contain) JanusGraphVertex(org.janusgraph.core.JanusGraphVertex) Arrays(java.util.Arrays) JanusGraphAssert.assertNotEmpty(org.janusgraph.testutil.JanusGraphAssert.assertNotEmpty) MessageReader(org.janusgraph.diskstorage.log.MessageReader) Geoshape(org.janusgraph.core.attribute.Geoshape) EDGE(org.janusgraph.graphdb.internal.RelationCategory.EDGE) JanusGraphTransaction(org.janusgraph.core.JanusGraphTransaction) WriteConfiguration(org.janusgraph.diskstorage.configuration.WriteConfiguration) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) Tag(org.junit.jupiter.api.Tag) LogTxStatus(org.janusgraph.graphdb.database.log.LogTxStatus) USE_MULTIQUERY(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.USE_MULTIQUERY) JanusGraphSchemaType(org.janusgraph.core.schema.JanusGraphSchemaType) JanusGraphFactory(org.janusgraph.core.JanusGraphFactory) EnumMap(java.util.EnumMap) TimestampProvider(org.janusgraph.diskstorage.util.time.TimestampProvider) StandardJanusGraph(org.janusgraph.graphdb.database.StandardJanusGraph) Set(java.util.Set) Vertex(org.apache.tinkerpop.gremlin.structure.Vertex) EdgeLabel(org.janusgraph.core.EdgeLabel) JanusGraphVertexProperty(org.janusgraph.core.JanusGraphVertexProperty) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) JanusGraphEdgeVertexStep(org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphEdgeVertexStep) INITIAL_JANUSGRAPH_VERSION(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.INITIAL_JANUSGRAPH_VERSION) RelationTypeIndex(org.janusgraph.core.schema.RelationTypeIndex) GraphTraversalSource(org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Iterables(com.google.common.collect.Iterables) Assertions.assertNull(org.junit.jupiter.api.Assertions.assertNull) SimpleQueryProfiler(org.janusgraph.graphdb.query.profile.SimpleQueryProfiler) PermanentLockingException(org.janusgraph.diskstorage.locking.PermanentLockingException) ArrayList(java.util.ArrayList) StandardTransactionLogProcessor(org.janusgraph.graphdb.log.StandardTransactionLogProcessor) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) Cmp(org.janusgraph.core.attribute.Cmp) ConsistencyModifier(org.janusgraph.core.schema.ConsistencyModifier) JanusGraphException(org.janusgraph.core.JanusGraphException) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) StandardEdgeLabelMaker(org.janusgraph.graphdb.types.StandardEdgeLabelMaker) MAX_COMMIT_TIME(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.MAX_COMMIT_TIME) JanusGraphElement(org.janusgraph.core.JanusGraphElement) TransactionLogHeader(org.janusgraph.graphdb.database.log.TransactionLogHeader) ValueSource(org.junit.jupiter.params.provider.ValueSource) TX_CACHE_SIZE(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.TX_CACHE_SIZE) File(java.io.File) Assertions.assertArrayEquals(org.junit.jupiter.api.Assertions.assertArrayEquals) ScanJobFuture(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJobFuture) Direction(org.apache.tinkerpop.gremlin.structure.Direction) SYSTEM_LOG_TRANSACTIONS(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.SYSTEM_LOG_TRANSACTIONS) ChronoUnit(java.time.temporal.ChronoUnit) HARD_MAX_LIMIT(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.HARD_MAX_LIMIT) Traversal(org.apache.tinkerpop.gremlin.process.traversal.Traversal) GraphDatabaseConfiguration(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration) TraversalMetrics(org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMetrics) SpecialIntSerializer(org.janusgraph.graphdb.serializer.SpecialIntSerializer) StandardJanusGraphTx(org.janusgraph.graphdb.transaction.StandardJanusGraphTx) LoggerFactory(org.slf4j.LoggerFactory) ConfigOption(org.janusgraph.diskstorage.configuration.ConfigOption) JanusGraphEdge(org.janusgraph.core.JanusGraphEdge) SchemaAction(org.janusgraph.core.schema.SchemaAction) Order.asc(org.apache.tinkerpop.gremlin.process.traversal.Order.asc) OrderList(org.janusgraph.graphdb.internal.OrderList) BATCH_PROPERTY_PREFETCHING(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.BATCH_PROPERTY_PREFETCHING) Assertions.assertFalse(org.junit.jupiter.api.Assertions.assertFalse) JanusGraphManagement(org.janusgraph.core.schema.JanusGraphManagement) JanusGraphIndex(org.janusgraph.core.schema.JanusGraphIndex) IndexSelectionUtil(org.janusgraph.graphdb.query.index.IndexSelectionUtil) P(org.apache.tinkerpop.gremlin.process.traversal.P) BOTH(org.apache.tinkerpop.gremlin.structure.Direction.BOTH) TransactionRecovery(org.janusgraph.core.log.TransactionRecovery) USER_LOG(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.USER_LOG) Property(org.apache.tinkerpop.gremlin.structure.Property) SchemaViolationException(org.janusgraph.core.SchemaViolationException) Mapping(org.janusgraph.core.schema.Mapping) ImmutableMap(com.google.common.collect.ImmutableMap) IndexRemoveJob(org.janusgraph.graphdb.olap.job.IndexRemoveJob) StandardPropertyKeyMaker(org.janusgraph.graphdb.types.StandardPropertyKeyMaker) Sets(com.google.common.collect.Sets) LOG_READ_INTERVAL(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.LOG_READ_INTERVAL) List(java.util.List) TempDir(org.junit.jupiter.api.io.TempDir) JanusGraphDefaultSchemaMaker(org.janusgraph.core.schema.JanusGraphDefaultSchemaMaker) GraphCentricQueryBuilder(org.janusgraph.graphdb.query.graph.GraphCentricQueryBuilder) CUSTOM_SERIALIZER_CLASS(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.CUSTOM_SERIALIZER_CLASS) AUTO_TYPE(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.AUTO_TYPE) HashMap(java.util.HashMap) LIMIT_BATCH_SIZE(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.LIMIT_BATCH_SIZE) JanusGraphAssert.assertContains(org.janusgraph.testutil.JanusGraphAssert.assertContains) Backend(org.janusgraph.diskstorage.Backend) Iterators(com.google.common.collect.Iterators) SpecialInt(org.janusgraph.graphdb.serializer.SpecialInt) ConfigElement(org.janusgraph.diskstorage.configuration.ConfigElement) DB_CACHE(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.DB_CACHE) LOG_BACKEND(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.LOG_BACKEND) IgnorePropertySchemaMaker(org.janusgraph.core.schema.IgnorePropertySchemaMaker) Edge(org.apache.tinkerpop.gremlin.structure.Edge) BackendException(org.janusgraph.diskstorage.BackendException) MANAGEMENT_LOG(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.MANAGEMENT_LOG) QueryProfiler(org.janusgraph.graphdb.query.profile.QueryProfiler) KCVSLog(org.janusgraph.diskstorage.log.kcvs.KCVSLog) Iterator(java.util.Iterator) org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__(org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__) STORAGE_READONLY(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.STORAGE_READONLY) JanusGraph(org.janusgraph.core.JanusGraph) IN(org.apache.tinkerpop.gremlin.structure.Direction.IN) ElementCategory(org.janusgraph.graphdb.internal.ElementCategory) Order(org.janusgraph.graphdb.internal.Order) PROPERTY(org.janusgraph.graphdb.internal.RelationCategory.PROPERTY) TimeUnit(java.util.concurrent.TimeUnit) JanusGraphAssert.queryProfilerAnnotationIsPresent(org.janusgraph.testutil.JanusGraphAssert.queryProfilerAnnotationIsPresent) ManagementSystem(org.janusgraph.graphdb.database.management.ManagementSystem) ReadMarker(org.janusgraph.diskstorage.log.ReadMarker) Collections(java.util.Collections) CacheVertex(org.janusgraph.graphdb.vertices.CacheVertex) JanusGraphVertex(org.janusgraph.core.JanusGraphVertex) Vertex(org.apache.tinkerpop.gremlin.structure.Vertex) JanusGraphAssert.getStepMetrics(org.janusgraph.testutil.JanusGraphAssert.getStepMetrics) Metrics(org.apache.tinkerpop.gremlin.process.traversal.util.Metrics) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) TraversalMetrics(org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMetrics) HashMap(java.util.HashMap) JanusGraphIndex(org.janusgraph.core.schema.JanusGraphIndex) PropertyKey(org.janusgraph.core.PropertyKey) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Test(org.junit.jupiter.api.Test)

Aggregations

Preconditions (com.google.common.base.Preconditions)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 Iterables (com.google.common.collect.Iterables)1 Iterators (com.google.common.collect.Iterators)1 Lists (com.google.common.collect.Lists)1 Sets (com.google.common.collect.Sets)1 File (java.io.File)1 Path (java.nio.file.Path)1 Duration (java.time.Duration)1 Instant (java.time.Instant)1 ChronoUnit (java.time.temporal.ChronoUnit)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 Date (java.util.Date)1 EnumMap (java.util.EnumMap)1 EnumSet (java.util.EnumSet)1 HashMap (java.util.HashMap)1