Examples with CompositeIndexType com.thinkaurelius.titan.graphdb.types.CompositeIndexType used on opensource projects

Example 1 with CompositeIndexType

use of com.thinkaurelius.titan.graphdb.types.CompositeIndexType in project titan by thinkaurelius.

the class StandardTitanGraph method prepareCommit.

public ModificationSummary prepareCommit(final Collection<InternalRelation> addedRelations, final Collection<InternalRelation> deletedRelations, final Predicate<InternalRelation> filter, final BackendTransaction mutator, final StandardTitanTx tx, final boolean acquireLocks) throws BackendException {
    ListMultimap<Long, InternalRelation> mutations = ArrayListMultimap.create();
    ListMultimap<InternalVertex, InternalRelation> mutatedProperties = ArrayListMultimap.create();
    List<IndexSerializer.IndexUpdate> indexUpdates = Lists.newArrayList();
    //1) Collect deleted edges and their index updates and acquire edge locks
    for (InternalRelation del : Iterables.filter(deletedRelations, filter)) {
        Preconditions.checkArgument(del.isRemoved());
        for (int pos = 0; pos < del.getLen(); pos++) {
            InternalVertex vertex = del.getVertex(pos);
            if (pos == 0 || !del.isLoop()) {
                if (del.isProperty())
                    mutatedProperties.put(vertex, del);
                mutations.put(vertex.longId(), del);
            }
            if (acquireLock(del, pos, acquireLocks)) {
                Entry entry = edgeSerializer.writeRelation(del, pos, tx);
                mutator.acquireEdgeLock(idManager.getKey(vertex.longId()), entry);
            }
        }
        indexUpdates.addAll(indexSerializer.getIndexUpdates(del));
    }
    //2) Collect added edges and their index updates and acquire edge locks
    for (InternalRelation add : Iterables.filter(addedRelations, filter)) {
        Preconditions.checkArgument(add.isNew());
        for (int pos = 0; pos < add.getLen(); pos++) {
            InternalVertex vertex = add.getVertex(pos);
            if (pos == 0 || !add.isLoop()) {
                if (add.isProperty())
                    mutatedProperties.put(vertex, add);
                mutations.put(vertex.longId(), add);
            }
            if (!vertex.isNew() && acquireLock(add, pos, acquireLocks)) {
                Entry entry = edgeSerializer.writeRelation(add, pos, tx);
                mutator.acquireEdgeLock(idManager.getKey(vertex.longId()), entry.getColumn());
            }
        }
        indexUpdates.addAll(indexSerializer.getIndexUpdates(add));
    }
    //3) Collect all index update for vertices
    for (InternalVertex v : mutatedProperties.keySet()) {
        indexUpdates.addAll(indexSerializer.getIndexUpdates(v, mutatedProperties.get(v)));
    }
    //4) Acquire index locks (deletions first)
    for (IndexSerializer.IndexUpdate update : indexUpdates) {
        if (!update.isCompositeIndex() || !update.isDeletion())
            continue;
        CompositeIndexType iIndex = (CompositeIndexType) update.getIndex();
        if (acquireLock(iIndex, acquireLocks)) {
            mutator.acquireIndexLock((StaticBuffer) update.getKey(), (Entry) update.getEntry());
        }
    }
    for (IndexSerializer.IndexUpdate update : indexUpdates) {
        if (!update.isCompositeIndex() || !update.isAddition())
            continue;
        CompositeIndexType iIndex = (CompositeIndexType) update.getIndex();
        if (acquireLock(iIndex, acquireLocks)) {
            mutator.acquireIndexLock((StaticBuffer) update.getKey(), ((Entry) update.getEntry()).getColumn());
        }
    }
    //5) Add relation mutations
    for (Long vertexid : mutations.keySet()) {
        Preconditions.checkArgument(vertexid > 0, "Vertex has no id: %s", vertexid);
        List<InternalRelation> edges = mutations.get(vertexid);
        List<Entry> additions = new ArrayList<Entry>(edges.size());
        List<Entry> deletions = new ArrayList<Entry>(Math.max(10, edges.size() / 10));
        for (InternalRelation edge : edges) {
            InternalRelationType baseType = (InternalRelationType) edge.getType();
            assert baseType.getBaseType() == null;
            for (InternalRelationType type : baseType.getRelationIndexes()) {
                if (type.getStatus() == SchemaStatus.DISABLED)
                    continue;
                for (int pos = 0; pos < edge.getArity(); pos++) {
                    if (!type.isUnidirected(Direction.BOTH) && !type.isUnidirected(EdgeDirection.fromPosition(pos)))
                        //Directionality is not covered
                        continue;
                    if (edge.getVertex(pos).longId() == vertexid) {
                        StaticArrayEntry entry = edgeSerializer.writeRelation(edge, type, pos, tx);
                        if (edge.isRemoved()) {
                            deletions.add(entry);
                        } else {
                            Preconditions.checkArgument(edge.isNew());
                            int ttl = getTTL(edge);
                            if (ttl > 0) {
                                entry.setMetaData(EntryMetaData.TTL, ttl);
                            }
                            additions.add(entry);
                        }
                    }
                }
            }
        }
        StaticBuffer vertexKey = idManager.getKey(vertexid);
        mutator.mutateEdges(vertexKey, additions, deletions);
    }
    //6) Add index updates
    boolean has2iMods = false;
    for (IndexSerializer.IndexUpdate indexUpdate : indexUpdates) {
        assert indexUpdate.isAddition() || indexUpdate.isDeletion();
        if (indexUpdate.isCompositeIndex()) {
            IndexSerializer.IndexUpdate<StaticBuffer, Entry> update = indexUpdate;
            if (update.isAddition())
                mutator.mutateIndex(update.getKey(), Lists.newArrayList(update.getEntry()), KCVSCache.NO_DELETIONS);
            else
                mutator.mutateIndex(update.getKey(), KeyColumnValueStore.NO_ADDITIONS, Lists.newArrayList(update.getEntry()));
        } else {
            IndexSerializer.IndexUpdate<String, IndexEntry> update = indexUpdate;
            has2iMods = true;
            IndexTransaction itx = mutator.getIndexTransaction(update.getIndex().getBackingIndexName());
            String indexStore = ((MixedIndexType) update.getIndex()).getStoreName();
            if (update.isAddition())
                itx.add(indexStore, update.getKey(), update.getEntry(), update.getElement().isNew());
            else
                itx.delete(indexStore, update.getKey(), update.getEntry().field, update.getEntry().value, update.getElement().isRemoved());
        }
    }
    return new ModificationSummary(!mutations.isEmpty(), has2iMods);
}

Example 2 with CompositeIndexType

use of com.thinkaurelius.titan.graphdb.types.CompositeIndexType in project titan by thinkaurelius.

the class IndexRemoveJob method validateIndexStatus.

@Override
protected void validateIndexStatus() {
    if (index instanceof RelationTypeIndex) {
    //Nothing specific to be done
    } else if (index instanceof TitanGraphIndex) {
        TitanGraphIndex gindex = (TitanGraphIndex) index;
        if (gindex.isMixedIndex())
            throw new UnsupportedOperationException("Cannot remove mixed indexes through Titan. This can " + "only be accomplished in the indexing system directly.");
        CompositeIndexType indexType = (CompositeIndexType) mgmt.getSchemaVertex(index).asIndexType();
        graphIndexId = indexType.getID();
    } else
        throw new UnsupportedOperationException("Unsupported index found: " + index);
    //Must be a relation type index or a composite graph index
    TitanSchemaVertex schemaVertex = mgmt.getSchemaVertex(index);
    SchemaStatus actualStatus = schemaVertex.getStatus();
    Preconditions.checkArgument(actualStatus == SchemaStatus.DISABLED, "The index [%s] must be disabled before it can be removed", indexName);
}

Example 3 with CompositeIndexType

use of com.thinkaurelius.titan.graphdb.types.CompositeIndexType in project titan by thinkaurelius.

the class TitanOperationCountingTest method testReadOperations.

public void testReadOperations(boolean cache) {
    metricsPrefix = "testReadOperations" + cache;
    resetEdgeCacheCounts();
    makeVertexIndexedUniqueKey("uid", Integer.class);
    mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK);
    finishSchema();
    if (cache)
        clopen(option(DB_CACHE), true, option(DB_CACHE_CLEAN_WAIT), 0, option(DB_CACHE_TIME), 0);
    else
        clopen();
    TitanTransaction tx = graph.buildTransaction().groupName(metricsPrefix).start();
    tx.makePropertyKey("name").dataType(String.class).make();
    tx.makeEdgeLabel("knows").make();
    tx.makeVertexLabel("person").make();
    tx.commit();
    verifyStoreMetrics(EDGESTORE_NAME);
    verifyLockingOverwrite(INDEXSTORE_NAME, 3);
    verifyStoreMetrics(METRICS_STOREMANAGER_NAME, ImmutableMap.of(M_MUTATE, 1l));
    resetMetrics();
    metricsPrefix = GraphDatabaseConfiguration.METRICS_SCHEMA_PREFIX_DEFAULT;
    resetMetrics();
    //Test schema caching
    for (int t = 0; t < 10; t++) {
        tx = graph.buildTransaction().groupName(metricsPrefix).start();
        //Retrieve name by index (one backend call each)
        assertTrue(tx.containsRelationType("name"));
        assertTrue(tx.containsRelationType("knows"));
        assertTrue(tx.containsVertexLabel("person"));
        PropertyKey name = tx.getPropertyKey("name");
        EdgeLabel knows = tx.getEdgeLabel("knows");
        VertexLabel person = tx.getVertexLabel("person");
        PropertyKey uid = tx.getPropertyKey("uid");
        //Retrieve name as property (one backend call each)
        assertEquals("name", name.name());
        assertEquals("knows", knows.name());
        assertEquals("person", person.name());
        assertEquals("uid", uid.name());
        //Looking up the definition (one backend call each)
        assertEquals(Cardinality.SINGLE, name.cardinality());
        assertEquals(Multiplicity.MULTI, knows.multiplicity());
        assertFalse(person.isPartitioned());
        assertEquals(Integer.class, uid.dataType());
        //Retrieving in and out relations for the relation types...
        InternalRelationType namei = (InternalRelationType) name;
        InternalRelationType knowsi = (InternalRelationType) knows;
        InternalRelationType uidi = (InternalRelationType) uid;
        assertNull(namei.getBaseType());
        assertNull(knowsi.getBaseType());
        IndexType index = Iterables.getOnlyElement(uidi.getKeyIndexes());
        assertEquals(1, index.getFieldKeys().length);
        assertEquals(ElementCategory.VERTEX, index.getElement());
        assertEquals(ConsistencyModifier.LOCK, ((CompositeIndexType) index).getConsistencyModifier());
        assertEquals(1, Iterables.size(uidi.getRelationIndexes()));
        assertEquals(1, Iterables.size(namei.getRelationIndexes()));
        assertEquals(namei, Iterables.getOnlyElement(namei.getRelationIndexes()));
        assertEquals(knowsi, Iterables.getOnlyElement(knowsi.getRelationIndexes()));
        //.. and vertex labels
        assertEquals(0, ((InternalVertexLabel) person).getTTL());
        tx.commit();
        //Needs to read on first iteration, after that it doesn't change anymore
        verifyStoreMetrics(EDGESTORE_NAME, ImmutableMap.of(M_GET_SLICE, 19l));
        verifyStoreMetrics(INDEXSTORE_NAME, ImmutableMap.of(M_GET_SLICE, 4l, /* name, knows, person, uid */
        M_ACQUIRE_LOCK, 0l));
    }
    //Create some graph data
    metricsPrefix = "add" + cache;
    tx = graph.buildTransaction().groupName(metricsPrefix).start();
    TitanVertex v = tx.addVertex(), u = tx.addVertex("person");
    v.property(VertexProperty.Cardinality.single, "uid", 1);
    u.property(VertexProperty.Cardinality.single, "name", "juju");
    Edge e = v.addEdge("knows", u);
    e.property("name", "edge");
    tx.commit();
    verifyStoreMetrics(EDGESTORE_NAME);
    verifyLockingOverwrite(INDEXSTORE_NAME, 1);
    for (int i = 1; i <= 30; i++) {
        metricsPrefix = "op" + i + cache;
        tx = graph.buildTransaction().groupName(metricsPrefix).start();
        v = getOnlyElement(tx.query().has("uid", 1).vertices());
        assertEquals(1, v.<Integer>value("uid").intValue());
        u = getOnlyElement(v.query().direction(Direction.BOTH).labels("knows").vertices());
        e = getOnlyElement(u.query().direction(Direction.IN).labels("knows").edges());
        assertEquals("juju", u.value("name"));
        assertEquals("edge", e.value("name"));
        tx.commit();
        if (!cache || i == 0) {
            verifyStoreMetrics(EDGESTORE_NAME, ImmutableMap.of(M_GET_SLICE, 4l));
            verifyStoreMetrics(INDEXSTORE_NAME, ImmutableMap.of(M_GET_SLICE, 1l));
        } else if (cache && i > 20) {
            //Needs a couple of iterations for cache to be cleaned
            verifyStoreMetrics(EDGESTORE_NAME);
            verifyStoreMetrics(INDEXSTORE_NAME);
        }
    }
}

Example 4 with CompositeIndexType

use of com.thinkaurelius.titan.graphdb.types.CompositeIndexType in project titan by thinkaurelius.

the class IndexRepairJob method process.

@Override
public void process(TitanVertex vertex, ScanMetrics metrics) {
    try {
        BackendTransaction mutator = writeTx.getTxHandle();
        if (index instanceof RelationTypeIndex) {
            RelationTypeIndexWrapper wrapper = (RelationTypeIndexWrapper) index;
            InternalRelationType wrappedType = wrapper.getWrappedType();
            EdgeSerializer edgeSerializer = writeTx.getEdgeSerializer();
            List<Entry> additions = new ArrayList<>();
            for (TitanRelation relation : vertex.query().types(indexRelationTypeName).direction(Direction.OUT).relations()) {
                InternalRelation titanRelation = (InternalRelation) relation;
                for (int pos = 0; pos < titanRelation.getArity(); pos++) {
                    if (!wrappedType.isUnidirected(Direction.BOTH) && !wrappedType.isUnidirected(EdgeDirection.fromPosition(pos)))
                        //Directionality is not covered
                        continue;
                    Entry entry = edgeSerializer.writeRelation(titanRelation, wrappedType, pos, writeTx);
                    additions.add(entry);
                }
            }
            StaticBuffer vertexKey = writeTx.getIdInspector().getKey(vertex.longId());
            mutator.mutateEdges(vertexKey, additions, KCVSCache.NO_DELETIONS);
            metrics.incrementCustom(ADDED_RECORDS_COUNT, additions.size());
        } else if (index instanceof TitanGraphIndex) {
            IndexType indexType = mgmt.getSchemaVertex(index).asIndexType();
            assert indexType != null;
            IndexSerializer indexSerializer = graph.getIndexSerializer();
            //Gather elements to index
            List<TitanElement> elements;
            switch(indexType.getElement()) {
                case VERTEX:
                    elements = ImmutableList.of(vertex);
                    break;
                case PROPERTY:
                    elements = Lists.newArrayList();
                    for (TitanVertexProperty p : addIndexSchemaConstraint(vertex.query(), indexType).properties()) {
                        elements.add(p);
                    }
                    break;
                case EDGE:
                    elements = Lists.newArrayList();
                    for (TitanEdge e : addIndexSchemaConstraint(vertex.query().direction(Direction.OUT), indexType).edges()) {
                        elements.add(e);
                    }
                    break;
                default:
                    throw new AssertionError("Unexpected category: " + indexType.getElement());
            }
            if (indexType.isCompositeIndex()) {
                for (TitanElement element : elements) {
                    Set<IndexSerializer.IndexUpdate<StaticBuffer, Entry>> updates = indexSerializer.reindexElement(element, (CompositeIndexType) indexType);
                    for (IndexSerializer.IndexUpdate<StaticBuffer, Entry> update : updates) {
                        log.debug("Mutating index {}: {}", indexType, update.getEntry());
                        mutator.mutateIndex(update.getKey(), Lists.newArrayList(update.getEntry()), KCVSCache.NO_DELETIONS);
                        metrics.incrementCustom(ADDED_RECORDS_COUNT);
                    }
                }
            } else {
                assert indexType.isMixedIndex();
                Map<String, Map<String, List<IndexEntry>>> documentsPerStore = new HashMap<>();
                for (TitanElement element : elements) {
                    indexSerializer.reindexElement(element, (MixedIndexType) indexType, documentsPerStore);
                    metrics.incrementCustom(DOCUMENT_UPDATES_COUNT);
                }
                mutator.getIndexTransaction(indexType.getBackingIndexName()).restore(documentsPerStore);
            }
        } else
            throw new UnsupportedOperationException("Unsupported index found: " + index);
    } catch (final Exception e) {
        mgmt.rollback();
        writeTx.rollback();
        metrics.incrementCustom(FAILED_TX);
        throw new TitanException(e.getMessage(), e);
    }
}