Examples with CompositeIndexType org.janusgraph.graphdb.types.CompositeIndexType used on opensource projects

Example 1 with CompositeIndexType

use of org.janusgraph.graphdb.types.CompositeIndexType in project janusgraph by JanusGraph.

the class JanusGraphOperationCountingTest method testReadOperations.

@SuppressWarnings("unchecked")
public void testReadOperations(boolean cache) {
    metricsPrefix = "testReadOperations" + cache;
    resetEdgeCacheCounts();
    makeVertexIndexedUniqueKey("uid", Integer.class);
    mgmt.setConsistency(mgmt.getGraphIndex("uid"), ConsistencyModifier.LOCK);
    finishSchema();
    if (cache)
        clopen(option(DB_CACHE), true, option(DB_CACHE_CLEAN_WAIT), 0, option(DB_CACHE_TIME), 0);
    else
        clopen();
    JanusGraphTransaction tx = graph.buildTransaction().groupName(metricsPrefix).start();
    tx.makePropertyKey("name").dataType(String.class).make();
    tx.makeEdgeLabel("knows").make();
    tx.makeVertexLabel("person").make();
    tx.commit();
    verifyStoreMetrics(EDGESTORE_NAME);
    verifyLockingOverwrite(3);
    verifyStoreMetrics(METRICS_STOREMANAGER_NAME, ImmutableMap.of(M_MUTATE, 1L));
    resetMetrics();
    metricsPrefix = GraphDatabaseConfiguration.METRICS_SCHEMA_PREFIX_DEFAULT;
    resetMetrics();
    // Test schema caching
    for (int t = 0; t < 10; t++) {
        tx = graph.buildTransaction().groupName(metricsPrefix).start();
        // Retrieve name by index (one backend call each)
        assertTrue(tx.containsRelationType("name"));
        assertTrue(tx.containsRelationType("knows"));
        assertTrue(tx.containsVertexLabel("person"));
        PropertyKey name = tx.getPropertyKey("name");
        EdgeLabel knows = tx.getEdgeLabel("knows");
        VertexLabel person = tx.getVertexLabel("person");
        PropertyKey uid = tx.getPropertyKey("uid");
        // Retrieve name as property (one backend call each)
        assertEquals("name", name.name());
        assertEquals("knows", knows.name());
        assertEquals("person", person.name());
        assertEquals("uid", uid.name());
        // Looking up the definition (one backend call each)
        assertEquals(Cardinality.SINGLE, name.cardinality());
        assertEquals(Multiplicity.MULTI, knows.multiplicity());
        assertFalse(person.isPartitioned());
        assertEquals(Integer.class, uid.dataType());
        // Retrieving in and out relations for the relation types...
        InternalRelationType nameInternal = (InternalRelationType) name;
        InternalRelationType knowsInternal = (InternalRelationType) knows;
        InternalRelationType uidInternal = (InternalRelationType) uid;
        assertNull(nameInternal.getBaseType());
        assertNull(knowsInternal.getBaseType());
        IndexType index = Iterables.getOnlyElement(uidInternal.getKeyIndexes());
        assertEquals(1, index.getFieldKeys().length);
        assertEquals(ElementCategory.VERTEX, index.getElement());
        assertEquals(ConsistencyModifier.LOCK, ((CompositeIndexType) index).getConsistencyModifier());
        assertEquals(1, Iterables.size(uidInternal.getRelationIndexes()));
        assertEquals(1, Iterables.size(nameInternal.getRelationIndexes()));
        assertEquals(nameInternal, Iterables.getOnlyElement(nameInternal.getRelationIndexes()));
        assertEquals(knowsInternal, Iterables.getOnlyElement(knowsInternal.getRelationIndexes()));
        // .. and vertex labels
        assertEquals(0, ((InternalVertexLabel) person).getTTL());
        tx.commit();
        // Needs to read on first iteration, after that it doesn't change anymore
        verifyStoreMetrics(EDGESTORE_NAME, ImmutableMap.of(M_GET_SLICE, 19L));
        verifyStoreMetrics(INDEXSTORE_NAME, ImmutableMap.of(M_GET_SLICE, 4L, /* name, knows, person, uid */
        M_ACQUIRE_LOCK, 0L));
    }
    // Create some graph data
    metricsPrefix = "add" + cache;
    tx = graph.buildTransaction().groupName(metricsPrefix).start();
    JanusGraphVertex v = tx.addVertex(), u = tx.addVertex("person");
    v.property(VertexProperty.Cardinality.single, "uid", 1);
    u.property(VertexProperty.Cardinality.single, "name", "juju");
    Edge e = v.addEdge("knows", u);
    e.property("name", "edge");
    tx.commit();
    verifyStoreMetrics(EDGESTORE_NAME);
    verifyLockingOverwrite(1);
    for (int i = 1; i <= 30; i++) {
        metricsPrefix = "op" + i + cache;
        tx = graph.buildTransaction().groupName(metricsPrefix).start();
        v = getOnlyElement(tx.query().has("uid", 1).vertices());
        assertEquals(1, v.<Integer>value("uid").intValue());
        u = getOnlyElement(v.query().direction(Direction.BOTH).labels("knows").vertices());
        e = getOnlyElement(u.query().direction(Direction.IN).labels("knows").edges());
        assertEquals("juju", u.value("name"));
        assertEquals("edge", e.value("name"));
        tx.commit();
        if (!cache) {
            verifyStoreMetrics(EDGESTORE_NAME, ImmutableMap.of(M_GET_SLICE, 4L));
            verifyStoreMetrics(INDEXSTORE_NAME, ImmutableMap.of(M_GET_SLICE, 1L));
        } else if (i > 20) {
            // Needs a couple of iterations for cache to be cleaned
            verifyStoreMetrics(EDGESTORE_NAME);
            verifyStoreMetrics(INDEXSTORE_NAME);
        }
    }
}

Example 2 with CompositeIndexType

use of org.janusgraph.graphdb.types.CompositeIndexType in project janusgraph by JanusGraph.

the class IndexRemoveJob method validateIndexStatus.

@Override
protected void validateIndexStatus() {
    if (!(index instanceof RelationTypeIndex || index instanceof JanusGraphIndex)) {
        throw new UnsupportedOperationException("Unsupported index found: " + index);
    }
    if (index instanceof JanusGraphIndex) {
        JanusGraphIndex graphIndex = (JanusGraphIndex) index;
        if (graphIndex.isMixedIndex())
            throw new UnsupportedOperationException("Cannot remove mixed indexes through JanusGraph. This can " + "only be accomplished in the indexing system directly.");
        CompositeIndexType indexType = (CompositeIndexType) managementSystem.getSchemaVertex(index).asIndexType();
        graphIndexId = indexType.getID();
    }
    // Must be a relation type index or a composite graph index
    JanusGraphSchemaVertex schemaVertex = managementSystem.getSchemaVertex(index);
    SchemaStatus actualStatus = schemaVertex.getStatus();
    Preconditions.checkArgument(actualStatus == SchemaStatus.DISABLED, "The index [%s] must be disabled before it can be removed", indexName);
}

Example 3 with CompositeIndexType

use of org.janusgraph.graphdb.types.CompositeIndexType in project janusgraph by JanusGraph.

the class IndexRepairJob method process.

@Override
public void process(JanusGraphVertex vertex, ScanMetrics metrics) {
    try {
        BackendTransaction mutator = writeTx.getTxHandle();
        if (index instanceof RelationTypeIndex) {
            RelationTypeIndexWrapper wrapper = (RelationTypeIndexWrapper) index;
            InternalRelationType wrappedType = wrapper.getWrappedType();
            EdgeSerializer edgeSerializer = writeTx.getEdgeSerializer();
            List<Entry> additions = new ArrayList<>();
            for (Object relation : vertex.query().types(indexRelationTypeName).direction(Direction.OUT).relations()) {
                InternalRelation janusgraphRelation = (InternalRelation) relation;
                for (int pos = 0; pos < janusgraphRelation.getArity(); pos++) {
                    if (!wrappedType.isUnidirected(Direction.BOTH) && !wrappedType.isUnidirected(EdgeDirection.fromPosition(pos)))
                        // Directionality is not covered
                        continue;
                    Entry entry = edgeSerializer.writeRelation(janusgraphRelation, wrappedType, pos, writeTx);
                    additions.add(entry);
                }
            }
            StaticBuffer vertexKey = writeTx.getIdInspector().getKey(vertex.longId());
            mutator.mutateEdges(vertexKey, additions, KCVSCache.NO_DELETIONS);
            metrics.incrementCustom(ADDED_RECORDS_COUNT, additions.size());
        } else if (index instanceof JanusGraphIndex) {
            IndexType indexType = managementSystem.getSchemaVertex(index).asIndexType();
            assert indexType != null;
            IndexSerializer indexSerializer = graph.getIndexSerializer();
            // Gather elements to index
            List<JanusGraphElement> elements;
            switch(indexType.getElement()) {
                case VERTEX:
                    elements = ImmutableList.of(vertex);
                    break;
                case PROPERTY:
                    elements = Lists.newArrayList();
                    for (JanusGraphVertexProperty p : addIndexSchemaConstraint(vertex.query(), indexType).properties()) {
                        elements.add(p);
                    }
                    break;
                case EDGE:
                    elements = Lists.newArrayList();
                    for (Object e : addIndexSchemaConstraint(vertex.query().direction(Direction.OUT), indexType).edges()) {
                        elements.add((JanusGraphEdge) e);
                    }
                    break;
                default:
                    throw new AssertionError("Unexpected category: " + indexType.getElement());
            }
            if (indexType.isCompositeIndex()) {
                for (JanusGraphElement element : elements) {
                    Set<IndexSerializer.IndexUpdate<StaticBuffer, Entry>> updates = indexSerializer.reindexElement(element, (CompositeIndexType) indexType);
                    for (IndexSerializer.IndexUpdate<StaticBuffer, Entry> update : updates) {
                        log.debug("Mutating index {}: {}", indexType, update.getEntry());
                        mutator.mutateIndex(update.getKey(), Lists.newArrayList(update.getEntry()), KCVSCache.NO_DELETIONS);
                        metrics.incrementCustom(ADDED_RECORDS_COUNT);
                    }
                }
            } else {
                assert indexType.isMixedIndex();
                Map<String, Map<String, List<IndexEntry>>> documentsPerStore = new HashMap<>();
                for (JanusGraphElement element : elements) {
                    indexSerializer.reindexElement(element, (MixedIndexType) indexType, documentsPerStore);
                    metrics.incrementCustom(DOCUMENT_UPDATES_COUNT);
                }
                mutator.getIndexTransaction(indexType.getBackingIndexName()).restore(documentsPerStore);
            }
        } else
            throw new UnsupportedOperationException("Unsupported index found: " + index);
    } catch (final Exception e) {
        managementSystem.rollback();
        writeTx.rollback();
        metrics.incrementCustom(FAILED_TX);
        throw new JanusGraphException(e.getMessage(), e);
    }
}

Example 4 with CompositeIndexType

use of org.janusgraph.graphdb.types.CompositeIndexType in project janusgraph by JanusGraph.

the class StandardJanusGraph method prepareCommit.

public ModificationSummary prepareCommit(final Collection<InternalRelation> addedRelations, final Collection<InternalRelation> deletedRelations, final Predicate<InternalRelation> filter, final BackendTransaction mutator, final StandardJanusGraphTx tx, final boolean acquireLocks) throws BackendException {
    ListMultimap<Long, InternalRelation> mutations = ArrayListMultimap.create();
    ListMultimap<InternalVertex, InternalRelation> mutatedProperties = ArrayListMultimap.create();
    List<IndexSerializer.IndexUpdate> indexUpdates = Lists.newArrayList();
    // 1) Collect deleted edges and their index updates and acquire edge locks
    for (InternalRelation del : Iterables.filter(deletedRelations, filter)) {
        Preconditions.checkArgument(del.isRemoved());
        for (int pos = 0; pos < del.getLen(); pos++) {
            InternalVertex vertex = del.getVertex(pos);
            if (pos == 0 || !del.isLoop()) {
                if (del.isProperty())
                    mutatedProperties.put(vertex, del);
                mutations.put(vertex.longId(), del);
            }
            if (acquireLock(del, pos, acquireLocks)) {
                Entry entry = edgeSerializer.writeRelation(del, pos, tx);
                mutator.acquireEdgeLock(idManager.getKey(vertex.longId()), entry);
            }
        }
        indexUpdates.addAll(indexSerializer.getIndexUpdates(del));
    }
    // 2) Collect added edges and their index updates and acquire edge locks
    for (InternalRelation add : Iterables.filter(addedRelations, filter)) {
        Preconditions.checkArgument(add.isNew());
        for (int pos = 0; pos < add.getLen(); pos++) {
            InternalVertex vertex = add.getVertex(pos);
            if (pos == 0 || !add.isLoop()) {
                if (add.isProperty())
                    mutatedProperties.put(vertex, add);
                mutations.put(vertex.longId(), add);
            }
            if (!vertex.isNew() && acquireLock(add, pos, acquireLocks)) {
                Entry entry = edgeSerializer.writeRelation(add, pos, tx);
                mutator.acquireEdgeLock(idManager.getKey(vertex.longId()), entry.getColumn());
            }
        }
        indexUpdates.addAll(indexSerializer.getIndexUpdates(add));
    }
    // 3) Collect all index update for vertices
    for (InternalVertex v : mutatedProperties.keySet()) {
        indexUpdates.addAll(indexSerializer.getIndexUpdates(v, mutatedProperties.get(v)));
    }
    // 4) Acquire index locks (deletions first)
    for (IndexSerializer.IndexUpdate update : indexUpdates) {
        if (!update.isCompositeIndex() || !update.isDeletion())
            continue;
        CompositeIndexType iIndex = (CompositeIndexType) update.getIndex();
        if (acquireLock(iIndex, acquireLocks)) {
            mutator.acquireIndexLock((StaticBuffer) update.getKey(), (Entry) update.getEntry());
        }
    }
    for (IndexSerializer.IndexUpdate update : indexUpdates) {
        if (!update.isCompositeIndex() || !update.isAddition())
            continue;
        CompositeIndexType iIndex = (CompositeIndexType) update.getIndex();
        if (acquireLock(iIndex, acquireLocks)) {
            mutator.acquireIndexLock((StaticBuffer) update.getKey(), ((Entry) update.getEntry()).getColumn());
        }
    }
    // 5) Add relation mutations
    for (Long vertexId : mutations.keySet()) {
        Preconditions.checkArgument(vertexId > 0, "Vertex has no id: %s", vertexId);
        final List<InternalRelation> edges = mutations.get(vertexId);
        final List<Entry> additions = new ArrayList<>(edges.size());
        final List<Entry> deletions = new ArrayList<>(Math.max(10, edges.size() / 10));
        for (final InternalRelation edge : edges) {
            final InternalRelationType baseType = (InternalRelationType) edge.getType();
            assert baseType.getBaseType() == null;
            for (InternalRelationType type : baseType.getRelationIndexes()) {
                if (type.getStatus() == SchemaStatus.DISABLED)
                    continue;
                for (int pos = 0; pos < edge.getArity(); pos++) {
                    if (!type.isUnidirected(Direction.BOTH) && !type.isUnidirected(EdgeDirection.fromPosition(pos)))
                        // Directionality is not covered
                        continue;
                    if (edge.getVertex(pos).longId() == vertexId) {
                        StaticArrayEntry entry = edgeSerializer.writeRelation(edge, type, pos, tx);
                        if (edge.isRemoved()) {
                            deletions.add(entry);
                        } else {
                            Preconditions.checkArgument(edge.isNew());
                            int ttl = getTTL(edge);
                            if (ttl > 0) {
                                entry.setMetaData(EntryMetaData.TTL, ttl);
                            }
                            additions.add(entry);
                        }
                    }
                }
            }
        }
        StaticBuffer vertexKey = idManager.getKey(vertexId);
        mutator.mutateEdges(vertexKey, additions, deletions);
    }
    // 6) Add index updates
    boolean has2iMods = false;
    for (IndexSerializer.IndexUpdate indexUpdate : indexUpdates) {
        assert indexUpdate.isAddition() || indexUpdate.isDeletion();
        if (indexUpdate.isCompositeIndex()) {
            final IndexSerializer.IndexUpdate<StaticBuffer, Entry> update = indexUpdate;
            if (update.isAddition())
                mutator.mutateIndex(update.getKey(), Lists.newArrayList(update.getEntry()), KCVSCache.NO_DELETIONS);
            else
                mutator.mutateIndex(update.getKey(), KeyColumnValueStore.NO_ADDITIONS, Lists.newArrayList(update.getEntry()));
        } else {
            final IndexSerializer.IndexUpdate<String, IndexEntry> update = indexUpdate;
            has2iMods = true;
            IndexTransaction itx = mutator.getIndexTransaction(update.getIndex().getBackingIndexName());
            String indexStore = ((MixedIndexType) update.getIndex()).getStoreName();
            if (update.isAddition())
                itx.add(indexStore, update.getKey(), update.getEntry(), update.getElement().isNew());
            else
                itx.delete(indexStore, update.getKey(), update.getEntry().field, update.getEntry().value, update.getElement().isRemoved());
        }
    }
    return new ModificationSummary(!mutations.isEmpty(), has2iMods);
}