Examples with RelationTypeIndex com.thinkaurelius.titan.core.schema.RelationTypeIndex used on opensource projects

Example 6 with RelationTypeIndex

use of com.thinkaurelius.titan.core.schema.RelationTypeIndex in project titan by thinkaurelius.

the class ManagementSystem method updateIndex.

/* --------------
    Schema Update
     --------------- */
@Override
public IndexJobFuture updateIndex(TitanIndex index, SchemaAction updateAction) {
    Preconditions.checkArgument(index != null, "Need to provide an index");
    Preconditions.checkArgument(updateAction != null, "Need to provide update action");
    TitanSchemaVertex schemaVertex = getSchemaVertex(index);
    Set<TitanSchemaVertex> dependentTypes;
    Set<PropertyKeyVertex> keySubset = ImmutableSet.of();
    if (index instanceof RelationTypeIndex) {
        dependentTypes = ImmutableSet.of((TitanSchemaVertex) ((InternalRelationType) schemaVertex).getBaseType());
        if (!updateAction.isApplicableStatus(schemaVertex.getStatus()))
            return null;
    } else if (index instanceof TitanGraphIndex) {
        IndexType indexType = schemaVertex.asIndexType();
        dependentTypes = Sets.newHashSet();
        if (indexType.isCompositeIndex()) {
            if (!updateAction.isApplicableStatus(schemaVertex.getStatus()))
                return null;
            for (PropertyKey key : ((TitanGraphIndex) index).getFieldKeys()) {
                dependentTypes.add((PropertyKeyVertex) key);
            }
        } else {
            keySubset = Sets.newHashSet();
            MixedIndexType cindexType = (MixedIndexType) indexType;
            Set<SchemaStatus> applicableStatus = updateAction.getApplicableStatus();
            for (ParameterIndexField field : cindexType.getFieldKeys()) {
                if (applicableStatus.contains(field.getStatus()))
                    keySubset.add((PropertyKeyVertex) field.getFieldKey());
            }
            if (keySubset.isEmpty())
                return null;
            dependentTypes.addAll(keySubset);
        }
    } else
        throw new UnsupportedOperationException("Updates not supported for index: " + index);
    IndexIdentifier indexId = new IndexIdentifier(index);
    StandardScanner.Builder builder;
    IndexJobFuture future;
    switch(updateAction) {
        case REGISTER_INDEX:
            setStatus(schemaVertex, SchemaStatus.INSTALLED, keySubset);
            updatedTypes.add(schemaVertex);
            updatedTypes.addAll(dependentTypes);
            setUpdateTrigger(new UpdateStatusTrigger(graph, schemaVertex, SchemaStatus.REGISTERED, keySubset));
            future = new EmptyIndexJobFuture();
            break;
        case REINDEX:
            builder = graph.getBackend().buildEdgeScanJob();
            builder.setFinishJob(indexId.getIndexJobFinisher(graph, SchemaAction.ENABLE_INDEX));
            builder.setJobId(indexId);
            builder.setJob(VertexJobConverter.convert(graph, new IndexRepairJob(indexId.indexName, indexId.relationTypeName)));
            try {
                future = builder.execute();
            } catch (BackendException e) {
                throw new TitanException(e);
            }
            break;
        case ENABLE_INDEX:
            setStatus(schemaVertex, SchemaStatus.ENABLED, keySubset);
            updatedTypes.add(schemaVertex);
            if (!keySubset.isEmpty())
                updatedTypes.addAll(dependentTypes);
            future = new EmptyIndexJobFuture();
            break;
        case DISABLE_INDEX:
            setStatus(schemaVertex, SchemaStatus.INSTALLED, keySubset);
            updatedTypes.add(schemaVertex);
            if (!keySubset.isEmpty())
                updatedTypes.addAll(dependentTypes);
            setUpdateTrigger(new UpdateStatusTrigger(graph, schemaVertex, SchemaStatus.DISABLED, keySubset));
            future = new EmptyIndexJobFuture();
            break;
        case REMOVE_INDEX:
            if (index instanceof RelationTypeIndex) {
                builder = graph.getBackend().buildEdgeScanJob();
            } else {
                TitanGraphIndex gindex = (TitanGraphIndex) index;
                if (gindex.isMixedIndex())
                    throw new UnsupportedOperationException("External mixed indexes must be removed in the indexing system directly.");
                builder = graph.getBackend().buildGraphIndexScanJob();
            }
            builder.setFinishJob(indexId.getIndexJobFinisher());
            builder.setJobId(indexId);
            builder.setJob(new IndexRemoveJob(graph, indexId.indexName, indexId.relationTypeName));
            try {
                future = builder.execute();
            } catch (BackendException e) {
                throw new TitanException(e);
            }
            break;
        default:
            throw new UnsupportedOperationException("Update action not supported: " + updateAction);
    }
    return future;
}

Example 7 with RelationTypeIndex

use of com.thinkaurelius.titan.core.schema.RelationTypeIndex in project titan by thinkaurelius.

the class RelationIndexStatusWatcher method call.

/**
     * Poll a relation index until it has a certain {@link SchemaStatus},
     * or until a configurable timeout is exceeded.
     *
     * @return a report with information about schema state, execution duration, and the index
     */
@Override
public RelationIndexStatusReport call() throws InterruptedException {
    Preconditions.checkNotNull(g, "Graph instance must not be null");
    Preconditions.checkNotNull(relationIndexName, "Index name must not be null");
    Preconditions.checkNotNull(status, "Target status must not be null");
    RelationTypeIndex idx;
    Timer t = new Timer(TimestampProviders.MILLI).start();
    boolean timedOut;
    while (true) {
        SchemaStatus actualStatus = null;
        TitanManagement mgmt = null;
        try {
            mgmt = g.openManagement();
            idx = mgmt.getRelationIndex(mgmt.getRelationType(relationTypeName), relationIndexName);
            actualStatus = idx.getIndexStatus();
            LOGGER.info("Index {} (relation type {}) has status {}", relationIndexName, relationTypeName, actualStatus);
            if (status.equals(actualStatus)) {
                return new RelationIndexStatusReport(true, relationIndexName, relationTypeName, actualStatus, status, t.elapsed());
            }
        } finally {
            if (null != mgmt)
                // Let an exception here propagate up the stack
                mgmt.rollback();
        }
        timedOut = null != timeout && 0 < t.elapsed().compareTo(timeout);
        if (timedOut) {
            LOGGER.info("Timed out ({}) while waiting for index {} (relation type {}) to reach status {}", timeout, relationIndexName, relationTypeName, status);
            return new RelationIndexStatusReport(false, relationIndexName, relationTypeName, actualStatus, status, t.elapsed());
        }
        Thread.sleep(poll.toMillis());
    }
}

Example 8 with RelationTypeIndex

use of com.thinkaurelius.titan.core.schema.RelationTypeIndex in project titan by thinkaurelius.

the class AbstractIndexManagementIT method testRepairRelationIndex.

@Test
public void testRepairRelationIndex() throws InterruptedException, BackendException, ExecutionException {
    tx.commit();
    mgmt.commit();
    // Load the "Graph of the Gods" sample data (WITHOUT mixed index coverage)
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
    // Create and enable a relation index on lives edges by reason
    TitanManagement m = graph.openManagement();
    PropertyKey reason = m.getPropertyKey("reason");
    EdgeLabel lives = m.getEdgeLabel("lives");
    m.buildEdgeIndex(lives, "livesByReason", Direction.BOTH, Order.decr, reason);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to REGISTERED
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "livesByReason", "lives").status(SchemaStatus.REGISTERED).call().getSucceeded());
    m = graph.openManagement();
    RelationTypeIndex index = m.getRelationIndex(m.getRelationType("lives"), "livesByReason");
    m.updateIndex(index, SchemaAction.ENABLE_INDEX);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to ENABLED
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "livesByReason", "lives").status(SchemaStatus.ENABLED).call().getSucceeded());
    // Run a query that hits the index but erroneously returns nothing because we haven't repaired yet
    //assertFalse(graph.query().has("reason", "no fear of death").edges().iterator().hasNext());
    // Repair
    MapReduceIndexManagement mri = new MapReduceIndexManagement(graph);
    m = graph.openManagement();
    index = m.getRelationIndex(m.getRelationType("lives"), "livesByReason");
    ScanMetrics metrics = mri.updateIndex(index, SchemaAction.REINDEX).get();
    assertEquals(8, metrics.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
}

Example 9 with RelationTypeIndex

use of com.thinkaurelius.titan.core.schema.RelationTypeIndex in project titan by thinkaurelius.

the class AbstractIndexManagementIT method testRemoveRelationIndex.

@Test
public void testRemoveRelationIndex() throws InterruptedException, BackendException, ExecutionException {
    tx.commit();
    mgmt.commit();
    // Load the "Graph of the Gods" sample data
    GraphOfTheGodsFactory.loadWithoutMixedIndex(graph, true);
    // Disable the "battlesByTime" index
    TitanManagement m = graph.openManagement();
    RelationType battled = m.getRelationType("battled");
    RelationTypeIndex battlesByTime = m.getRelationIndex(battled, "battlesByTime");
    m.updateIndex(battlesByTime, SchemaAction.DISABLE_INDEX);
    m.commit();
    graph.tx().commit();
    // Block until the SchemaStatus transitions to DISABLED
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "battlesByTime", "battled").status(SchemaStatus.DISABLED).call().getSucceeded());
    // Remove index
    MapReduceIndexManagement mri = new MapReduceIndexManagement(graph);
    m = graph.openManagement();
    battled = m.getRelationType("battled");
    battlesByTime = m.getRelationIndex(battled, "battlesByTime");
    ScanMetrics metrics = mri.updateIndex(battlesByTime, SchemaAction.REMOVE_INDEX).get();
    assertEquals(6, metrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}

Example 10 with RelationTypeIndex

use of com.thinkaurelius.titan.core.schema.RelationTypeIndex in project titan by thinkaurelius.

the class TitanGraphTest method testIndexUpdatesWithReindexAndRemove.

@Test
public void testIndexUpdatesWithReindexAndRemove() throws InterruptedException, ExecutionException {
    clopen(option(LOG_SEND_DELAY, MANAGEMENT_LOG), Duration.ofMillis(0), option(KCVSLog.LOG_READ_LAG_TIME, MANAGEMENT_LOG), Duration.ofMillis(50), option(LOG_READ_INTERVAL, MANAGEMENT_LOG), Duration.ofMillis(250));
    //Types without index
    PropertyKey time = mgmt.makePropertyKey("time").dataType(Integer.class).make();
    PropertyKey name = mgmt.makePropertyKey("name").dataType(String.class).cardinality(Cardinality.SET).make();
    EdgeLabel friend = mgmt.makeEdgeLabel("friend").multiplicity(Multiplicity.MULTI).make();
    PropertyKey sensor = mgmt.makePropertyKey("sensor").dataType(Double.class).cardinality(Cardinality.LIST).make();
    finishSchema();
    RelationTypeIndex pindex, eindex;
    TitanGraphIndex gindex;
    //Add some sensor & friend data
    TitanVertex v = tx.addVertex();
    for (int i = 0; i < 10; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        TitanVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    newTx();
    //Indexes should not yet be in use
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 0, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 0, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 0, new boolean[] { false, true });
    newTx();
    //Create indexes after the fact
    finishSchema();
    sensor = mgmt.getPropertyKey("sensor");
    time = mgmt.getPropertyKey("time");
    name = mgmt.getPropertyKey("name");
    friend = mgmt.getEdgeLabel("friend");
    mgmt.buildPropertyIndex(sensor, "byTime", decr, time);
    mgmt.buildEdgeIndex(friend, "byTime", Direction.OUT, decr, time);
    mgmt.buildIndex("bySensorReading", Vertex.class).addKey(name).buildCompositeIndex();
    finishSchema();
    newTx();
    //Add some sensor & friend data that should already be indexed even though index is not yet enabled
    v = getV(tx, v);
    for (int i = 100; i < 110; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        TitanVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    tx.commit();
    //Should not yet be able to enable since not yet registered
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    try {
        mgmt.updateIndex(pindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException e) {
    }
    try {
        mgmt.updateIndex(eindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException e) {
    }
    try {
        mgmt.updateIndex(gindex, SchemaAction.ENABLE_INDEX);
        fail();
    } catch (IllegalArgumentException e) {
    }
    mgmt.commit();
    ManagementUtil.awaitVertexIndexUpdate(graph, "byTime", "sensor", 10, ChronoUnit.SECONDS);
    ManagementUtil.awaitGraphIndexUpdate(graph, "bySensorReading", 5, ChronoUnit.SECONDS);
    finishSchema();
    //Verify new status
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.REGISTERED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.REGISTERED, eindex.getIndexStatus());
    assertEquals(SchemaStatus.REGISTERED, gindex.getIndexStatus(gindex.getFieldKeys()[0]));
    finishSchema();
    //Simply enable without reindex
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    mgmt.updateIndex(eindex, SchemaAction.ENABLE_INDEX);
    finishSchema();
    assertTrue(ManagementSystem.awaitRelationIndexStatus(graph, "byTime", "friend").status(SchemaStatus.ENABLED).timeout(10L, ChronoUnit.SECONDS).call().getSucceeded());
    //Reindex the other two
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    ScanMetrics reindexSensorByTime = mgmt.updateIndex(pindex, SchemaAction.REINDEX).get();
    finishSchema();
    gindex = mgmt.getGraphIndex("bySensorReading");
    ScanMetrics reindexBySensorReading = mgmt.updateIndex(gindex, SchemaAction.REINDEX).get();
    finishSchema();
    assertNotEquals(0, reindexSensorByTime.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    assertNotEquals(0, reindexBySensorReading.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    //Every index should now be enabled
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.ENABLED, eindex.getIndexStatus());
    assertEquals(SchemaStatus.ENABLED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.ENABLED, gindex.getIndexStatus(gindex.getFieldKeys()[0]));
    //Add some more sensor & friend data
    newTx();
    v = getV(tx, v);
    for (int i = 200; i < 210; i++) {
        v.property("sensor", i, "time", i);
        v.property("name", "v" + i);
        TitanVertex o = tx.addVertex();
        v.addEdge("friend", o, "time", i);
    }
    newTx();
    //Use indexes now but only see new data for property and graph index
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 201, 205).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 0, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    evaluateQuery(tx.query().has("name", "v205"), ElementCategory.VERTEX, 1, new boolean[] { true, true }, "bySensorReading");
    finishSchema();
    eindex = mgmt.getRelationIndex(mgmt.getRelationType("friend"), "byTime");
    ScanMetrics reindexFriendByTime = mgmt.updateIndex(eindex, SchemaAction.REINDEX).get();
    finishSchema();
    assertNotEquals(0, reindexFriendByTime.getCustom(IndexRepairJob.ADDED_RECORDS_COUNT));
    finishSchema();
    newTx();
    //It should now have all the answers
    v = getV(tx, v);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    mgmt.updateIndex(pindex, SchemaAction.DISABLE_INDEX);
    mgmt.updateIndex(gindex, SchemaAction.DISABLE_INDEX);
    mgmt.commit();
    tx.commit();
    ManagementUtil.awaitVertexIndexUpdate(graph, "byTime", "sensor", 10, ChronoUnit.SECONDS);
    ManagementUtil.awaitGraphIndexUpdate(graph, "bySensorReading", 5, ChronoUnit.SECONDS);
    finishSchema();
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    assertEquals(SchemaStatus.DISABLED, pindex.getIndexStatus());
    assertEquals(SchemaStatus.DISABLED, gindex.getIndexStatus(gindex.getFieldKeys()[0]));
    finishSchema();
    newTx();
    //The two disabled indexes should force full scans
    v = getV(tx, v);
    evaluateQuery(v.query().keys("sensor").interval("time", 1, 5).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 101, 105).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().keys("sensor").interval("time", 201, 205).orderBy("time", decr), PROPERTY, 4, 1, new boolean[] { false, false }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 1, 5).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 101, 105).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(v.query().labels("friend").direction(OUT).interval("time", 201, 205).orderBy("time", decr), EDGE, 4, 1, new boolean[] { true, true }, tx.getPropertyKey("time"), Order.DESC);
    evaluateQuery(tx.query().has("name", "v5"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v105"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    evaluateQuery(tx.query().has("name", "v205"), ElementCategory.VERTEX, 1, new boolean[] { false, true });
    tx.commit();
    finishSchema();
    pindex = mgmt.getRelationIndex(mgmt.getRelationType("sensor"), "byTime");
    gindex = mgmt.getGraphIndex("bySensorReading");
    ScanMetrics pmetrics = mgmt.updateIndex(pindex, SchemaAction.REMOVE_INDEX).get();
    ScanMetrics gmetrics = mgmt.updateIndex(gindex, SchemaAction.REMOVE_INDEX).get();
    finishSchema();
    assertEquals(30, pmetrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
    assertEquals(30, gmetrics.getCustom(IndexRemoveJob.DELETED_RECORDS_COUNT));
}