Search in sources :

Example 1 with ScanMetrics

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics in project janusgraph by JanusGraph.

the class AbstractIndexManagementIT method testRepairRelationIndex.

@Test
public void testRepairRelationIndex() throws InterruptedException, BackendException, ExecutionException {
    tx.commit();
    mgmt.commit();
    // Load the "Graph of the Gods" sample data (WITHOUT mixed index coverage)
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
    // Create and enable a relation index on lives edges by reason
    JanusGraphManagement m = graph.openManagement();
    PropertyKey reason = m.getPropertyKey("reason");
    EdgeLabel lives = m.getEdgeLabel("lives");
    m.buildEdgeIndex(lives, "livesByReason", Direction.BOTH, Order.decr, reason);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to REGISTERED
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "livesByReason", "lives").status(SchemaStatus.REGISTERED).call().getSucceeded());
    m = graph.openManagement();
    RelationTypeIndex index = m.getRelationIndex(m.getRelationType("lives"), "livesByReason");
    m.updateIndex(index, SchemaAction.ENABLE_INDEX);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to ENABLED
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "livesByReason", "lives").status(SchemaStatus.ENABLED).call().getSucceeded());
    // Run a query that hits the index but erroneously returns nothing because we haven't repaired yet
    // assertFalse(graph.query().has("reason", "no fear of death").edges().iterator().hasNext());
    // Repair
    MapReduceIndexManagement mri = new MapReduceIndexManagement(graph);
    m = graph.openManagement();
    index = m.getRelationIndex(m.getRelationType("lives"), "livesByReason");
    ScanMetrics metrics = mri.updateIndex(index, SchemaAction.REINDEX).get();
    assertEquals(8, metrics.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
}
Also used : JanusGraphManagement(org.janusgraph.core.schema.JanusGraphManagement) EdgeLabel(org.janusgraph.core.EdgeLabel) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) RelationTypeIndex(org.janusgraph.core.schema.RelationTypeIndex) PropertyKey(org.janusgraph.core.PropertyKey) Test(org.junit.Test) JanusGraphBaseTest(org.janusgraph.graphdb.JanusGraphBaseTest)

Example 2 with ScanMetrics

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics in project janusgraph by JanusGraph.

the class AbstractIndexManagementIT method testRepairGraphIndex.

@Test
public void testRepairGraphIndex() throws InterruptedException, BackendException, ExecutionException {
    tx.commit();
    mgmt.commit();
    // Load the "Graph of the Gods" sample data (WITHOUT mixed index coverage)
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
    // Create and enable a graph index on age
    JanusGraphManagement m = graph.openManagement();
    PropertyKey age = m.getPropertyKey("age");
    m.buildIndex("verticesByAge", Vertex.class).addKey(age).buildCompositeIndex();
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to REGISTERED
    assertTrue(ManagementSystem.awaitGraphIndexStatus(graph, "verticesByAge").status(SchemaStatus.REGISTERED).call().getSucceeded());
    m = graph.openManagement();
    JanusGraphIndex index = m.getGraphIndex("verticesByAge");
    m.updateIndex(index, SchemaAction.ENABLE_INDEX);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to ENABLED
    assertTrue(ManagementSystem.awaitGraphIndexStatus(graph, "verticesByAge").status(SchemaStatus.ENABLED).call().getSucceeded());
    // Run a query that hits the index but erroneously returns nothing because we haven't repaired yet
    assertFalse(graph.query().has("age", 10000).vertices().iterator().hasNext());
    // Repair
    MapReduceIndexManagement mri = new MapReduceIndexManagement(graph);
    m = graph.openManagement();
    index = m.getGraphIndex("verticesByAge");
    ScanMetrics metrics = mri.updateIndex(index, SchemaAction.REINDEX).get();
    assertEquals(6, metrics.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    // Test the index
    Iterable<JanusGraphVertex> hits = graph.query().has("age", 4500).vertices();
    assertNotNull(hits);
    assertEquals(1, Iterables.size(hits));
    JanusGraphVertex v = Iterables.getOnlyElement(hits);
    assertNotNull(v);
    assertEquals("neptune", v.value("name"));
}
Also used : JanusGraphManagement(org.janusgraph.core.schema.JanusGraphManagement) JanusGraphVertex(org.janusgraph.core.JanusGraphVertex) JanusGraphIndex(org.janusgraph.core.schema.JanusGraphIndex) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) PropertyKey(org.janusgraph.core.PropertyKey) Test(org.junit.Test) JanusGraphBaseTest(org.janusgraph.graphdb.JanusGraphBaseTest)

Example 3 with ScanMetrics

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics in project janusgraph by JanusGraph.

the class AbstractIndexManagementIT method testRemoveRelationIndex.

@Test
public void testRemoveRelationIndex() throws InterruptedException, BackendException, ExecutionException {
    tx.commit();
    mgmt.commit();
    // Load the "Graph of the Gods" sample data
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
    // Disable the "battlesByTime" index
    JanusGraphManagement m = graph.openManagement();
    RelationType battled = m.getRelationType("battled");
    RelationTypeIndex battlesByTime = m.getRelationIndex(battled, "battlesByTime");
    m.updateIndex(battlesByTime, SchemaAction.DISABLE_INDEX);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to DISABLED
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "battlesByTime", "battled").status(SchemaStatus.DISABLED).call().getSucceeded());
    // Remove index
    MapReduceIndexManagement mri = new MapReduceIndexManagement(graph);
    m = graph.openManagement();
    battled = m.getRelationType("battled");
    battlesByTime = m.getRelationIndex(battled, "battlesByTime");
    ScanMetrics metrics = mri.updateIndex(battlesByTime, SchemaAction.REMOVE_INDEX).get();
    assertEquals(6, metrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}
Also used : JanusGraphManagement(org.janusgraph.core.schema.JanusGraphManagement) RelationType(org.janusgraph.core.RelationType) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) RelationTypeIndex(org.janusgraph.core.schema.RelationTypeIndex) Test(org.junit.Test) JanusGraphBaseTest(org.janusgraph.graphdb.JanusGraphBaseTest)

Example 4 with ScanMetrics

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics in project janusgraph by JanusGraph.

the class FulgoraGraphComputer method submit.

@Override
public Future<ComputerResult> submit() {
    if (executed)
        throw Exceptions.computerHasAlreadyBeenSubmittedAVertexProgram();
    else
        executed = true;
    // it is not possible execute a computer if it has no vertex program nor map-reducers
    if (null == vertexProgram && mapReduces.isEmpty())
        throw GraphComputer.Exceptions.computerHasNoVertexProgramNorMapReducers();
    // it is possible to run map-reducers without a vertex program
    if (null != vertexProgram) {
        GraphComputerHelper.validateProgramOnComputer(this, vertexProgram);
        this.mapReduces.addAll(this.vertexProgram.getMapReducers());
    }
    // if the user didn't set desired persistence/resultgraph, then get from vertex program or else, no persistence
    this.persistMode = GraphComputerHelper.getPersistState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.persistMode));
    this.resultGraphMode = GraphComputerHelper.getResultGraphState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.resultGraphMode));
    // determine the legality persistence and result graph options
    if (!this.features().supportsResultGraphPersistCombination(this.resultGraphMode, this.persistMode))
        throw GraphComputer.Exceptions.resultGraphPersistCombinationNotSupported(this.resultGraphMode, this.persistMode);
    // ensure requested workers are not larger than supported workers
    if (this.numThreads > this.features().getMaxWorkers())
        throw GraphComputer.Exceptions.computerRequiresMoreWorkersThanSupported(this.numThreads, this.features().getMaxWorkers());
    memory = new FulgoraMemory(vertexProgram, mapReduces);
    return CompletableFuture.supplyAsync(() -> {
        final long time = System.currentTimeMillis();
        if (null != vertexProgram) {
            // ##### Execute vertex program
            vertexMemory = new FulgoraVertexMemory(expectedNumVertices, graph.getIDManager(), vertexProgram);
            // execute the vertex program
            vertexProgram.setup(memory);
            try (VertexProgramScanJob.Executor job = VertexProgramScanJob.getVertexProgramScanJob(graph, memory, vertexMemory, vertexProgram)) {
                for (int iteration = 1; ; iteration++) {
                    memory.completeSubRound();
                    vertexMemory.nextIteration(vertexProgram.getMessageScopes(memory));
                    jobId = name + "#" + iteration;
                    StandardScanner.Builder scanBuilder = graph.getBackend().buildEdgeScanJob();
                    scanBuilder.setJobId(jobId);
                    scanBuilder.setNumProcessingThreads(numThreads);
                    scanBuilder.setWorkBlockSize(readBatchSize);
                    scanBuilder.setJob(job);
                    PartitionedVertexProgramExecutor programExecutor = new PartitionedVertexProgramExecutor(graph, memory, vertexMemory, vertexProgram);
                    try {
                        // Iterates over all vertices and computes the vertex program on all non-partitioned vertices. For partitioned ones, the data is aggregated
                        ScanMetrics jobResult = scanBuilder.execute().get();
                        long failures = jobResult.get(ScanMetrics.Metric.FAILURE);
                        if (failures > 0) {
                            throw new JanusGraphException("Failed to process [" + failures + "] vertices in vertex program iteration [" + iteration + "]. Computer is aborting.");
                        }
                        // Runs the vertex program on all aggregated, partitioned vertices.
                        programExecutor.run(numThreads, jobResult);
                        failures = jobResult.getCustom(PartitionedVertexProgramExecutor.PARTITION_VERTEX_POSTFAIL);
                        if (failures > 0) {
                            throw new JanusGraphException("Failed to process [" + failures + "] partitioned vertices in vertex program iteration [" + iteration + "]. Computer is aborting.");
                        }
                    } catch (Exception e) {
                        throw new JanusGraphException(e);
                    }
                    vertexMemory.completeIteration();
                    memory.completeSubRound();
                    try {
                        if (this.vertexProgram.terminate(this.memory)) {
                            break;
                        }
                    } finally {
                        memory.incrIteration();
                    }
                }
            }
        }
        // ##### Execute map-reduce jobs
        // Collect map jobs
        Map<MapReduce, FulgoraMapEmitter> mapJobs = new HashMap<>(mapReduces.size());
        for (MapReduce mapReduce : mapReduces) {
            if (mapReduce.doStage(MapReduce.Stage.MAP)) {
                FulgoraMapEmitter mapEmitter = new FulgoraMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
                mapJobs.put(mapReduce, mapEmitter);
            }
        }
        // Execute map jobs
        jobId = name + "#map";
        try (VertexMapJob.Executor job = VertexMapJob.getVertexMapJob(graph, vertexMemory, mapJobs)) {
            StandardScanner.Builder scanBuilder = graph.getBackend().buildEdgeScanJob();
            scanBuilder.setJobId(jobId);
            scanBuilder.setNumProcessingThreads(numThreads);
            scanBuilder.setWorkBlockSize(readBatchSize);
            scanBuilder.setJob(job);
            try {
                ScanMetrics jobResult = scanBuilder.execute().get();
                long failures = jobResult.get(ScanMetrics.Metric.FAILURE);
                if (failures > 0) {
                    throw new JanusGraphException("Failed to process [" + failures + "] vertices in map phase. Computer is aborting.");
                }
                failures = jobResult.getCustom(VertexMapJob.MAP_JOB_FAILURE);
                if (failures > 0) {
                    throw new JanusGraphException("Failed to process [" + failures + "] individual map jobs. Computer is aborting.");
                }
            } catch (Exception e) {
                throw new JanusGraphException(e);
            }
            // Execute reduce phase and add to memory
            for (Map.Entry<MapReduce, FulgoraMapEmitter> mapJob : mapJobs.entrySet()) {
                FulgoraMapEmitter<?, ?> mapEmitter = mapJob.getValue();
                MapReduce mapReduce = mapJob.getKey();
                // sort results if a map output sort is defined
                mapEmitter.complete(mapReduce);
                if (mapReduce.doStage(MapReduce.Stage.REDUCE)) {
                    final FulgoraReduceEmitter<?, ?> reduceEmitter = new FulgoraReduceEmitter<>();
                    try (WorkerPool workers = new WorkerPool(numThreads)) {
                        workers.submit(() -> mapReduce.workerStart(MapReduce.Stage.REDUCE));
                        for (final Map.Entry queueEntry : mapEmitter.reduceMap.entrySet()) {
                            if (null == queueEntry)
                                break;
                            workers.submit(() -> mapReduce.reduce(queueEntry.getKey(), ((Iterable) queueEntry.getValue()).iterator(), reduceEmitter));
                        }
                        workers.submit(() -> mapReduce.workerEnd(MapReduce.Stage.REDUCE));
                    } catch (Exception e) {
                        throw new JanusGraphException("Exception while executing reduce phase", e);
                    }
                    // mapEmitter.reduceMap.entrySet().parallelStream().forEach(entry -> mapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter));
                    // sort results if a reduce output sort is defined
                    reduceEmitter.complete(mapReduce);
                    mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator());
                } else {
                    mapReduce.addResultToMemory(this.memory, mapEmitter.mapQueue.iterator());
                }
            }
        }
        memory.attachReferenceElements(graph);
        // #### Write mutated properties back into graph
        Graph resultgraph = graph;
        if (persistMode == Persist.NOTHING && resultGraphMode == ResultGraph.NEW) {
            resultgraph = EmptyGraph.instance();
        } else if (persistMode != Persist.NOTHING && vertexProgram != null && !vertexProgram.getVertexComputeKeys().isEmpty()) {
            // First, create property keys in graph if they don't already exist
            JanusGraphManagement management = graph.openManagement();
            try {
                for (VertexComputeKey key : vertexProgram.getVertexComputeKeys()) {
                    if (!management.containsPropertyKey(key.getKey()))
                        log.warn("Property key [{}] is not part of the schema and will be created. It is advised to initialize all keys.", key.getKey());
                    management.getOrCreatePropertyKey(key.getKey());
                }
                management.commit();
            } finally {
                if (management != null && management.isOpen())
                    management.rollback();
            }
            // TODO: Filter based on VertexProgram
            Map<Long, Map<String, Object>> mutatedProperties = Maps.transformValues(vertexMemory.getMutableVertexProperties(), new Function<Map<String, Object>, Map<String, Object>>() {

                @Nullable
                @Override
                public Map<String, Object> apply(final Map<String, Object> o) {
                    return Maps.filterKeys(o, s -> !VertexProgramHelper.isTransientVertexComputeKey(s, vertexProgram.getVertexComputeKeys()));
                }
            });
            if (resultGraphMode == ResultGraph.ORIGINAL) {
                AtomicInteger failures = new AtomicInteger(0);
                try (WorkerPool workers = new WorkerPool(numThreads)) {
                    List<Map.Entry<Long, Map<String, Object>>> subset = new ArrayList<>(writeBatchSize / vertexProgram.getVertexComputeKeys().size());
                    int currentSize = 0;
                    for (Map.Entry<Long, Map<String, Object>> entry : mutatedProperties.entrySet()) {
                        subset.add(entry);
                        currentSize += entry.getValue().size();
                        if (currentSize >= writeBatchSize) {
                            workers.submit(new VertexPropertyWriter(subset, failures));
                            subset = new ArrayList<>(subset.size());
                            currentSize = 0;
                        }
                    }
                    if (!subset.isEmpty())
                        workers.submit(new VertexPropertyWriter(subset, failures));
                } catch (Exception e) {
                    throw new JanusGraphException("Exception while attempting to persist result into graph", e);
                }
                if (failures.get() > 0)
                    throw new JanusGraphException("Could not persist program results to graph. Check log for details.");
            } else if (resultGraphMode == ResultGraph.NEW) {
                resultgraph = graph.newTransaction();
                for (Map.Entry<Long, Map<String, Object>> vertexProperty : mutatedProperties.entrySet()) {
                    Vertex v = resultgraph.vertices(vertexProperty.getKey()).next();
                    for (Map.Entry<String, Object> prop : vertexProperty.getValue().entrySet()) {
                        if (prop.getValue() instanceof List) {
                            ((List) prop.getValue()).forEach(value -> v.property(VertexProperty.Cardinality.list, prop.getKey(), value));
                        } else {
                            v.property(VertexProperty.Cardinality.single, prop.getKey(), prop.getValue());
                        }
                    }
                }
            }
        }
        // update runtime and return the newly computed graph
        this.memory.setRuntime(System.currentTimeMillis() - time);
        this.memory.complete();
        return new DefaultComputerResult(resultgraph, this.memory);
    });
}
Also used : JanusGraphManagement(org.janusgraph.core.schema.JanusGraphManagement) Vertex(org.apache.tinkerpop.gremlin.structure.Vertex) HashMap(java.util.HashMap) JanusGraphException(org.janusgraph.core.JanusGraphException) ArrayList(java.util.ArrayList) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) MapReduce(org.apache.tinkerpop.gremlin.process.computer.MapReduce) Function(com.google.common.base.Function) ArrayList(java.util.ArrayList) List(java.util.List) JanusGraphException(org.janusgraph.core.JanusGraphException) WorkerPool(org.janusgraph.graphdb.util.WorkerPool) Graph(org.apache.tinkerpop.gremlin.structure.Graph) StandardJanusGraph(org.janusgraph.graphdb.database.StandardJanusGraph) EmptyGraph(org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph) StandardScanner(org.janusgraph.diskstorage.keycolumnvalue.scan.StandardScanner) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DefaultComputerResult(org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult) VertexComputeKey(org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey) HashMap(java.util.HashMap) Map(java.util.Map)

Example 5 with ScanMetrics

use of org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics in project janusgraph by JanusGraph.

the class JanusGraphTest method testIndexUpdatesWithReindexAndRemove.

@Test
public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException {
    clopen(option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0), option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50), option(LOG_READ_INTERVAL, MANAGEMENT_LOG), Duration.ofMillis(250));
    // Types without index
    PropertyKey time = mgmt.makePropertyKey("time").dataType(Integer.class).make();
    PropertyKey name = mgmt.makePropertyKey("name").dataType(String.class).cardinality(Cardinality.SET).make();
    EdgeLabel friend = mgmt.makeEdgeLabel("friend").multiplicity(Multiplicity.MULTI).make();
    PropertyKey sensor = mgmt.makePropertyKey("sensor").dataType(Double.class).cardinality(Cardinality.LIST).make();
    finishSchema();
    RelationTypeIndex pindex, eindex;
    JanusGraphIndex graphIndex;
    // Add some sensor & friend data
    JanusGraphVertex v = tx.addVertex();
    for (int i = 0; i < 10; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        JanusGraphVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    newTx();
    // Indexes should not yet be in use
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 0, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 0, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 0, new boolean[] { false, true });
    newTx();
    // Create indexes after the fact
    finishSchema();
    sensor = mgmt.getPropertyKey("sensor");
    time = mgmt.getPropertyKey("time");
    name = mgmt.getPropertyKey("name");
    friend = mgmt.getEdgeLabel("friend");
    mgmt.buildPropertyIndex(sensor, "byTime", decr, time);
    mgmt.buildEdgeIndex(friend, "byTime", Direction.OUT, decr, time);
    mgmt.buildIndex("bySensorReading", Vertex.class).addKey(name).buildCompositeIndex();
    finishSchema();
    newTx();
    // Add some sensor & friend data that should already be indexed even though index is not yet enabled
    v = getV(tx, v);
    for (int i = 100; i < 110; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        JanusGraphVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    tx.commit();
    // Should not yet be able to enable since not yet registered
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    graphIndex = mgmt.getGraphIndex("bySensorReading");
    try {
        mgmt.updateIndex(pindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException ignored) {
    }
    try {
        mgmt.updateIndex(eindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException ignored) {
    }
    try {
        mgmt.updateIndex(graphIndex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException ignored) {
    }
    mgmt.commit();
    ManagementUtil.awaitVertexIndexUpdate(graph, "byTime", "sensor", 10, ChronoUnit.SECONDS);
    ManagementUtil.awaitGraphIndexUpdate(graph, "bySensorReading", 5, ChronoUnit.SECONDS);
    finishSchema();
    // Verify new status
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    graphIndex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.REGISTERED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.REGISTERED, eindex.getIndexStatus());
    assertEquals(SchemaStatus.REGISTERED, graphIndex.getIndexStatus(graphIndex.getFieldKeys()[0]));
    finishSchema();
    // Simply enable without reindex
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    mgmt.updateIndex(eindex, SchemaAction.ENABLE_INDEX);
    finishSchema();
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "byTime", "friend").status(SchemaStatus.ENABLED).timeout(10L, ChronoUnit.SECONDS).call().getSucceeded());
    // Reindex the other two
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    ScanMetrics reindexSensorByTime = mgmt.updateIndex(pindex, SchemaAction.REINDEX).get();
    finishSchema();
    graphIndex = mgmt.getGraphIndex("bySensorReading");
    ScanMetrics reindexBySensorReading = mgmt.updateIndex(graphIndex, SchemaAction.REINDEX).get();
    finishSchema();
    assertNotEquals(0, reindexSensorByTime.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    assertNotEquals(0, reindexBySensorReading.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    // Every index should now be enabled
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    graphIndex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.ENABLED, eindex.getIndexStatus());
    assertEquals(SchemaStatus.ENABLED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.ENABLED, graphIndex.getIndexStatus(graphIndex.getFieldKeys()[0]));
    // Add some more sensor & friend data
    newTx();
    v = getV(tx, v);
    for (int i = 200; i < 210; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        JanusGraphVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    newTx();
    // Use indexes now but only see new data for property and graph index
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 201, 205).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 0, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    evaluateQuery(tx.query().has("name", "v205"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    finishSchema();
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    ScanMetrics reindexFriendByTime = mgmt.updateIndex(eindex, SchemaAction.REINDEX).get();
    finishSchema();
    assertNotEquals(0, reindexFriendByTime.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    finishSchema();
    newTx();
    // It should now have all the answers
    v = getV(tx, v);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    graphIndex = mgmt.getGraphIndex("bySensorReading");
    mgmt.updateIndex(pindex, SchemaAction.DISABLE_INDEX);
    mgmt.updateIndex(graphIndex, SchemaAction.DISABLE_INDEX);
    mgmt.commit();
    tx.commit();
    ManagementUtil.awaitVertexIndexUpdate(graph, "byTime", "sensor", 10, ChronoUnit.SECONDS);
    ManagementUtil.awaitGraphIndexUpdate(graph, "bySensorReading", 5, ChronoUnit.SECONDS);
    finishSchema();
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    graphIndex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.DISABLED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.DISABLED, graphIndex.getIndexStatus(graphIndex.getFieldKeys()[0]));
    finishSchema();
    newTx();
    // The two disabled indexes should force full scans
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 201, 205).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v205"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    tx.commit();
    finishSchema();
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    graphIndex = mgmt.getGraphIndex("bySensorReading");
    ScanMetrics pmetrics = mgmt.updateIndex(pindex, SchemaAction.REMOVE_INDEX).get();
    ScanMetrics graphIndexMetrics = mgmt.updateIndex(graphIndex, SchemaAction.REMOVE_INDEX).get();
    finishSchema();
    assertEquals(30, pmetrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
    assertEquals(30, graphIndexMetrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) JanusGraphVertex(org.janusgraph.core.JanusGraphVertex) EdgeLabel(org.janusgraph.core.EdgeLabel) JanusGraphIndex(org.janusgraph.core.schema.JanusGraphIndex) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) RelationTypeIndex(org.janusgraph.core.schema.RelationTypeIndex) PropertyKey(org.janusgraph.core.PropertyKey) Test(org.junit.Test)

Aggregations

ScanMetrics (org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics)9 Test (org.junit.Test)7 JanusGraphBaseTest (org.janusgraph.graphdb.JanusGraphBaseTest)6 JanusGraphManagement (org.janusgraph.core.schema.JanusGraphManagement)5 PropertyKey (org.janusgraph.core.PropertyKey)3 JanusGraphIndex (org.janusgraph.core.schema.JanusGraphIndex)3 RelationTypeIndex (org.janusgraph.core.schema.RelationTypeIndex)3 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 EdgeLabel (org.janusgraph.core.EdgeLabel)2 JanusGraphVertex (org.janusgraph.core.JanusGraphVertex)2 Function (com.google.common.base.Function)1 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 ExecutionException (java.util.concurrent.ExecutionException)1 BaseConfiguration (org.apache.commons.configuration.BaseConfiguration)1 MapReduce (org.apache.tinkerpop.gremlin.process.computer.MapReduce)1 VertexComputeKey (org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey)1