Examples with ScanMetrics com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics used on opensource projects

Example 1 with ScanMetrics

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics in project titan by thinkaurelius.

the class OLAPTest method testVertexScan.

@Test
public void testVertexScan() throws Exception {
    int numV = 100;
    int numE = generateRandomGraph(numV);
    final String DEGREE_COUNT = "degree";
    final String VERTEX_COUNT = "numV";
    clopen();
    ScanMetrics result1 = executeScanJob(new VertexScanJob() {

        @Override
        public void process(TitanVertex vertex, ScanMetrics metrics) {
            long outDegree = vertex.query().labels("knows").direction(Direction.OUT).edgeCount();
            assertEquals(0, vertex.query().labels("knows").direction(Direction.IN).edgeCount());
            assertEquals(1, vertex.query().labels("uid").propertyCount());
            assertTrue(vertex.<Integer>property("uid").orElse(0) > 0);
            metrics.incrementCustom(DEGREE_COUNT, outDegree);
            metrics.incrementCustom(VERTEX_COUNT);
        }

        @Override
        public void getQueries(QueryContainer queries) {
            queries.addQuery().labels("knows").direction(Direction.OUT).edges();
            queries.addQuery().keys("uid").properties();
        }

        @Override
        public VertexScanJob clone() {
            return this;
        }
    });
    assertEquals(numV, result1.getCustom(VERTEX_COUNT));
    assertEquals(numE, result1.getCustom(DEGREE_COUNT));
    ScanMetrics result2 = executeScanJob(new VertexScanJob() {

        @Override
        public void process(TitanVertex vertex, ScanMetrics metrics) {
            metrics.incrementCustom(VERTEX_COUNT);
            assertEquals(1, vertex.query().labels("numvals").propertyCount());
            int numvals = vertex.value("numvals");
            assertEquals(numvals, vertex.query().labels("values").propertyCount());
        }

        @Override
        public void getQueries(QueryContainer queries) {
            queries.addQuery().keys("values").properties();
            queries.addQuery().keys("numvals").properties();
        }

        @Override
        public VertexScanJob clone() {
            return this;
        }
    });
    assertEquals(numV, result2.getCustom(VERTEX_COUNT));
}

Example 2 with ScanMetrics

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics in project titan by thinkaurelius.

the class AbstractIndexManagementIT method testRemoveGraphIndex.

@Test
public void testRemoveGraphIndex() throws InterruptedException, BackendException, ExecutionException {
    tx.commit();
    mgmt.commit();
    // Load the "Graph of the Gods" sample data
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
    // Disable the "name" composite index
    TitanManagement m = graph.openManagement();
    TitanGraphIndex nameIndex = m.getGraphIndex("name");
    m.updateIndex(nameIndex, SchemaAction.DISABLE_INDEX);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to DISABLED
    assertTrue(ManagementSystem.awaitGraphIndexStatus(graph, "name").status(SchemaStatus.DISABLED).call().getSucceeded());
    // Remove index
    MapReduceIndexManagement mri = new MapReduceIndexManagement(graph);
    m = graph.openManagement();
    TitanGraphIndex index = m.getGraphIndex("name");
    ScanMetrics metrics = mri.updateIndex(index, SchemaAction.REMOVE_INDEX).get();
    assertEquals(12, metrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}

Example 3 with ScanMetrics

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics in project titan by thinkaurelius.

the class FulgoraGraphComputer method submit.

@Override
public Future<ComputerResult> submit() {
    if (executed)
        throw Exceptions.computerHasAlreadyBeenSubmittedAVertexProgram();
    else
        executed = true;
    // it is not possible execute a computer if it has no vertex program nor mapreducers
    if (null == vertexProgram && mapReduces.isEmpty())
        throw GraphComputer.Exceptions.computerHasNoVertexProgramNorMapReducers();
    // it is possible to run mapreducers without a vertex program
    if (null != vertexProgram) {
        GraphComputerHelper.validateProgramOnComputer(this, vertexProgram);
        this.mapReduces.addAll(this.vertexProgram.getMapReducers());
    }
    // if the user didn't set desired persistence/resultgraph, then get from vertex program or else, no persistence
    this.persistMode = GraphComputerHelper.getPersistState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.persistMode));
    this.resultGraphMode = GraphComputerHelper.getResultGraphState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.resultGraphMode));
    // determine the legality persistence and result graph options
    if (!this.features().supportsResultGraphPersistCombination(this.resultGraphMode, this.persistMode))
        throw GraphComputer.Exceptions.resultGraphPersistCombinationNotSupported(this.resultGraphMode, this.persistMode);
    memory = new FulgoraMemory(vertexProgram, mapReduces);
    return CompletableFuture.<ComputerResult>supplyAsync(() -> {
        final long time = System.currentTimeMillis();
        if (null != vertexProgram) {
            // ##### Execute vertex program
            vertexMemory = new FulgoraVertexMemory(expectedNumVertices, graph.getIDManager(), vertexProgram);
            // execute the vertex program
            vertexProgram.setup(memory);
            memory.completeSubRound();
            for (int iteration = 1; ; iteration++) {
                vertexMemory.nextIteration(vertexProgram.getMessageScopes(memory));
                jobId = name + "#" + iteration;
                VertexProgramScanJob.Executor job = VertexProgramScanJob.getVertexProgramScanJob(graph, memory, vertexMemory, vertexProgram);
                StandardScanner.Builder scanBuilder = graph.getBackend().buildEdgeScanJob();
                scanBuilder.setJobId(jobId);
                scanBuilder.setNumProcessingThreads(numThreads);
                scanBuilder.setWorkBlockSize(readBatchSize);
                scanBuilder.setJob(job);
                PartitionedVertexProgramExecutor pvpe = new PartitionedVertexProgramExecutor(graph, memory, vertexMemory, vertexProgram);
                try {
                    //Iterates over all vertices and computes the vertex program on all non-partitioned vertices. For partitioned ones, the data is aggregated
                    ScanMetrics jobResult = scanBuilder.execute().get();
                    long failures = jobResult.get(ScanMetrics.Metric.FAILURE);
                    if (failures > 0) {
                        throw new TitanException("Failed to process [" + failures + "] vertices in vertex program iteration [" + iteration + "]. Computer is aborting.");
                    }
                    //Runs the vertex program on all aggregated, partitioned vertices.
                    pvpe.run(numThreads, jobResult);
                    failures = jobResult.getCustom(PartitionedVertexProgramExecutor.PARTITION_VERTEX_POSTFAIL);
                    if (failures > 0) {
                        throw new TitanException("Failed to process [" + failures + "] partitioned vertices in vertex program iteration [" + iteration + "]. Computer is aborting.");
                    }
                } catch (Exception e) {
                    throw new TitanException(e);
                }
                vertexMemory.completeIteration();
                memory.completeSubRound();
                try {
                    if (this.vertexProgram.terminate(this.memory)) {
                        break;
                    }
                } finally {
                    memory.incrIteration();
                    memory.completeSubRound();
                }
            }
        }
        // ##### Execute mapreduce jobs
        // Collect map jobs
        Map<MapReduce, FulgoraMapEmitter> mapJobs = new HashMap<>(mapReduces.size());
        for (MapReduce mapReduce : mapReduces) {
            if (mapReduce.doStage(MapReduce.Stage.MAP)) {
                FulgoraMapEmitter mapEmitter = new FulgoraMapEmitter<>(mapReduce.doStage(MapReduce.Stage.REDUCE));
                mapJobs.put(mapReduce, mapEmitter);
            }
        }
        // Execute map jobs
        jobId = name + "#map";
        VertexMapJob.Executor job = VertexMapJob.getVertexMapJob(graph, vertexMemory, mapJobs);
        StandardScanner.Builder scanBuilder = graph.getBackend().buildEdgeScanJob();
        scanBuilder.setJobId(jobId);
        scanBuilder.setNumProcessingThreads(numThreads);
        scanBuilder.setWorkBlockSize(readBatchSize);
        scanBuilder.setJob(job);
        try {
            ScanMetrics jobResult = scanBuilder.execute().get();
            long failures = jobResult.get(ScanMetrics.Metric.FAILURE);
            if (failures > 0) {
                throw new TitanException("Failed to process [" + failures + "] vertices in map phase. Computer is aborting.");
            }
            failures = jobResult.getCustom(VertexMapJob.MAP_JOB_FAILURE);
            if (failures > 0) {
                throw new TitanException("Failed to process [" + failures + "] individual map jobs. Computer is aborting.");
            }
        } catch (Exception e) {
            throw new TitanException(e);
        }
        // Execute reduce phase and add to memory
        for (Map.Entry<MapReduce, FulgoraMapEmitter> mapJob : mapJobs.entrySet()) {
            FulgoraMapEmitter<?, ?> mapEmitter = mapJob.getValue();
            MapReduce mapReduce = mapJob.getKey();
            // sort results if a map output sort is defined
            mapEmitter.complete(mapReduce);
            if (mapReduce.doStage(MapReduce.Stage.REDUCE)) {
                final FulgoraReduceEmitter<?, ?> reduceEmitter = new FulgoraReduceEmitter<>();
                try (WorkerPool workers = new WorkerPool(numThreads)) {
                    workers.submit(() -> mapReduce.workerStart(MapReduce.Stage.REDUCE));
                    for (final Map.Entry queueEntry : mapEmitter.reduceMap.entrySet()) {
                        workers.submit(() -> mapReduce.reduce(queueEntry.getKey(), ((Iterable) queueEntry.getValue()).iterator(), reduceEmitter));
                    }
                    workers.submit(() -> mapReduce.workerEnd(MapReduce.Stage.REDUCE));
                } catch (Exception e) {
                    throw new TitanException("Exception while executing reduce phase", e);
                }
                //                    mapEmitter.reduceMap.entrySet().parallelStream().forEach(entry -> mapReduce.reduce(entry.getKey(), entry.getValue().iterator(), reduceEmitter));
                // sort results if a reduce output sort is defined
                reduceEmitter.complete(mapReduce);
                mapReduce.addResultToMemory(this.memory, reduceEmitter.reduceQueue.iterator());
            } else {
                mapReduce.addResultToMemory(this.memory, mapEmitter.mapQueue.iterator());
            }
        }
        // #### Write mutated properties back into graph
        Graph resultgraph = graph;
        if (persistMode == Persist.NOTHING && resultGraphMode == ResultGraph.NEW) {
            resultgraph = EmptyGraph.instance();
        } else if (persistMode != Persist.NOTHING && vertexProgram != null && !vertexProgram.getElementComputeKeys().isEmpty()) {
            //First, create property keys in graph if they don't already exist
            TitanManagement mgmt = graph.openManagement();
            try {
                for (String key : vertexProgram.getElementComputeKeys()) {
                    if (!mgmt.containsPropertyKey(key))
                        log.warn("Property key [{}] is not part of the schema and will be created. It is advised to initialize all keys.", key);
                    mgmt.getOrCreatePropertyKey(key);
                }
                mgmt.commit();
            } finally {
                if (mgmt != null && mgmt.isOpen())
                    mgmt.rollback();
            }
            //TODO: Filter based on VertexProgram
            Map<Long, Map<String, Object>> mutatedProperties = Maps.transformValues(vertexMemory.getMutableVertexProperties(), new Function<Map<String, Object>, Map<String, Object>>() {

                @Nullable
                @Override
                public Map<String, Object> apply(@Nullable Map<String, Object> o) {
                    return Maps.filterKeys(o, s -> !NON_PERSISTING_KEYS.contains(s));
                }
            });
            if (resultGraphMode == ResultGraph.ORIGINAL) {
                AtomicInteger failures = new AtomicInteger(0);
                try (WorkerPool workers = new WorkerPool(numThreads)) {
                    List<Map.Entry<Long, Map<String, Object>>> subset = new ArrayList<>(writeBatchSize / vertexProgram.getElementComputeKeys().size());
                    int currentSize = 0;
                    for (Map.Entry<Long, Map<String, Object>> entry : mutatedProperties.entrySet()) {
                        subset.add(entry);
                        currentSize += entry.getValue().size();
                        if (currentSize >= writeBatchSize) {
                            workers.submit(new VertexPropertyWriter(subset, failures));
                            subset = new ArrayList<>(subset.size());
                            currentSize = 0;
                        }
                    }
                    if (!subset.isEmpty())
                        workers.submit(new VertexPropertyWriter(subset, failures));
                } catch (Exception e) {
                    throw new TitanException("Exception while attempting to persist result into graph", e);
                }
                if (failures.get() > 0)
                    throw new TitanException("Could not persist program results to graph. Check log for details.");
            } else if (resultGraphMode == ResultGraph.NEW) {
                resultgraph = graph.newTransaction();
                for (Map.Entry<Long, Map<String, Object>> vprop : mutatedProperties.entrySet()) {
                    Vertex v = resultgraph.vertices(vprop.getKey()).next();
                    for (Map.Entry<String, Object> prop : vprop.getValue().entrySet()) {
                        v.property(VertexProperty.Cardinality.single, prop.getKey(), prop.getValue());
                    }
                }
            }
        }
        // update runtime and return the newly computed graph
        this.memory.setRuntime(System.currentTimeMillis() - time);
        this.memory.complete();
        return new DefaultComputerResult(resultgraph, this.memory);
    });
}

Example 4 with ScanMetrics

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics in project titan by thinkaurelius.

the class TitanGraphTest method testIndexUpdatesWithReindexAndRemove.

@Test
public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException {
    clopen(option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0), option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50), option(LOG_READ_INTERVAL, MANAGEMENT_LOG), Duration.ofMillis(250));
    //Types without index
    PropertyKey time = mgmt.makePropertyKey("time").dataType(Integer.class).make();
    PropertyKey name = mgmt.makePropertyKey("name").dataType(String.class).cardinality(Cardinality.SET).make();
    EdgeLabel friend = mgmt.makeEdgeLabel("friend").multiplicity(Multiplicity.MULTI).make();
    PropertyKey sensor = mgmt.makePropertyKey("sensor").dataType(Double.class).cardinality(Cardinality.LIST).make();
    finishSchema();
    RelationTypeIndex pindex, eindex;
    TitanGraphIndex gindex;
    //Add some sensor & friend data
    TitanVertex v = tx.addVertex();
    for (int i = 0; i < 10; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        TitanVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    newTx();
    //Indexes should not yet be in use
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 0, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 0, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 0, new boolean[] { false, true });
    newTx();
    //Create indexes after the fact
    finishSchema();
    sensor = mgmt.getPropertyKey("sensor");
    time = mgmt.getPropertyKey("time");
    name = mgmt.getPropertyKey("name");
    friend = mgmt.getEdgeLabel("friend");
    mgmt.buildPropertyIndex(sensor, "byTime", decr, time);
    mgmt.buildEdgeIndex(friend, "byTime", Direction.OUT, decr, time);
    mgmt.buildIndex("bySensorReading", Vertex.class).addKey(name).buildCompositeIndex();
    finishSchema();
    newTx();
    //Add some sensor & friend data that should already be indexed even though index is not yet enabled
    v = getV(tx, v);
    for (int i = 100; i < 110; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        TitanVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    tx.commit();
    //Should not yet be able to enable since not yet registered
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    try {
        mgmt.updateIndex(pindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException e) {
    }
    try {
        mgmt.updateIndex(eindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException e) {
    }
    try {
        mgmt.updateIndex(gindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException e) {
    }
    mgmt.commit();
    ManagementUtil.awaitVertexIndexUpdate(graph, "byTime", "sensor", 10, ChronoUnit.SECONDS);
    ManagementUtil.awaitGraphIndexUpdate(graph, "bySensorReading", 5, ChronoUnit.SECONDS);
    finishSchema();
    //Verify new status
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.REGISTERED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.REGISTERED, eindex.getIndexStatus());
    assertEquals(SchemaStatus.REGISTERED, gindex.getIndexStatus(gindex.getFieldKeys()[0]));
    finishSchema();
    //Simply enable without reindex
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    mgmt.updateIndex(eindex, SchemaAction.ENABLE_INDEX);
    finishSchema();
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "byTime", "friend").status(SchemaStatus.ENABLED).timeout(10L, ChronoUnit.SECONDS).call().getSucceeded());
    //Reindex the other two
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    ScanMetrics reindexSensorByTime = mgmt.updateIndex(pindex, SchemaAction.REINDEX).get();
    finishSchema();
    gindex = mgmt.getGraphIndex("bySensorReading");
    ScanMetrics reindexBySensorReading = mgmt.updateIndex(gindex, SchemaAction.REINDEX).get();
    finishSchema();
    assertNotEquals(0, reindexSensorByTime.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    assertNotEquals(0, reindexBySensorReading.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    //Every index should now be enabled
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.ENABLED, eindex.getIndexStatus());
    assertEquals(SchemaStatus.ENABLED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.ENABLED, gindex.getIndexStatus(gindex.getFieldKeys()[0]));
    //Add some more sensor & friend data
    newTx();
    v = getV(tx, v);
    for (int i = 200; i < 210; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        TitanVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    newTx();
    //Use indexes now but only see new data for property and graph index
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 201, 205).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 0, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    evaluateQuery(tx.query().has("name", "v205"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    finishSchema();
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    ScanMetrics reindexFriendByTime = mgmt.updateIndex(eindex, SchemaAction.REINDEX).get();
    finishSchema();
    assertNotEquals(0, reindexFriendByTime.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    finishSchema();
    newTx();
    //It should now have all the answers
    v = getV(tx, v);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    mgmt.updateIndex(pindex, SchemaAction.DISABLE_INDEX);
    mgmt.updateIndex(gindex, SchemaAction.DISABLE_INDEX);
    mgmt.commit();
    tx.commit();
    ManagementUtil.awaitVertexIndexUpdate(graph, "byTime", "sensor", 10, ChronoUnit.SECONDS);
    ManagementUtil.awaitGraphIndexUpdate(graph, "bySensorReading", 5, ChronoUnit.SECONDS);
    finishSchema();
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.DISABLED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.DISABLED, gindex.getIndexStatus(gindex.getFieldKeys()[0]));
    finishSchema();
    newTx();
    //The two disabled indexes should force full scans
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 201, 205).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v205"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    tx.commit();
    finishSchema();
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    ScanMetrics pmetrics = mgmt.updateIndex(pindex, SchemaAction.REMOVE_INDEX).get();
    ScanMetrics gmetrics = mgmt.updateIndex(gindex, SchemaAction.REMOVE_INDEX).get();
    finishSchema();
    assertEquals(30, pmetrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
    assertEquals(30, gmetrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}

Example 5 with ScanMetrics

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.scan.ScanMetrics in project titan by thinkaurelius.

the class OLAPTest method removeGhostVertices.

@Test
public void removeGhostVertices() throws Exception {
    TitanVertex v1 = tx.addVertex("person");
    v1.property("name", "stephen");
    TitanVertex v2 = tx.addVertex("person");
    v1.property("name", "marko");
    TitanVertex v3 = tx.addVertex("person");
    v1.property("name", "dan");
    v2.addEdge("knows", v3);
    v1.addEdge("knows", v2);
    newTx();
    long v3id = getId(v3);
    long v1id = getId(v1);
    assertTrue(v3id > 0);
    v3 = getV(tx, v3id);
    assertNotNull(v3);
    v3.remove();
    tx.commit();
    TitanTransaction xx = graph.buildTransaction().checkExternalVertexExistence(false).start();
    v3 = getV(xx, v3id);
    assertNotNull(v3);
    v1 = getV(xx, v1id);
    assertNotNull(v1);
    v3.property("name", "deleted");
    v3.addEdge("knows", v1);
    xx.commit();
    newTx();
    assertNull(getV(tx, v3id));
    v1 = getV(tx, v1id);
    assertNotNull(v1);
    assertEquals(v3id, v1.query().direction(Direction.IN).labels("knows").vertices().iterator().next().longId());
    tx.commit();
    mgmt.commit();
    ScanMetrics result = executeScanJob(new GhostVertexRemover(graph));
    assertEquals(1, result.getCustom(GhostVertexRemover.REMOVED_VERTEX_COUNT));
    assertEquals(2, result.getCustom(GhostVertexRemover.REMOVED_RELATION_COUNT));
    assertEquals(0, result.getCustom(GhostVertexRemover.SKIPPED_GHOST_LIMIT_COUNT));
}