Search in sources :

Example 1 with MixedIndexType

use of org.janusgraph.graphdb.types.MixedIndexType in project janusgraph by JanusGraph.

the class IndexRepairJob method process.

@Override
public void process(JanusGraphVertex vertex, ScanMetrics metrics) {
    try {
        BackendTransaction mutator = writeTx.getTxHandle();
        if (index instanceof RelationTypeIndex) {
            RelationTypeIndexWrapper wrapper = (RelationTypeIndexWrapper) index;
            InternalRelationType wrappedType = wrapper.getWrappedType();
            EdgeSerializer edgeSerializer = writeTx.getEdgeSerializer();
            List<Entry> outAdditions = new ArrayList<>();
            Map<StaticBuffer, List<Entry>> inAdditionsMap = new HashMap<>();
            for (Object relation : vertex.query().types(indexRelationTypeName).direction(Direction.OUT).relations()) {
                InternalRelation janusgraphRelation = (InternalRelation) relation;
                for (int pos = 0; pos < janusgraphRelation.getArity(); pos++) {
                    if (!wrappedType.isUnidirected(Direction.BOTH) && !wrappedType.isUnidirected(EdgeDirection.fromPosition(pos)))
                        // Directionality is not covered
                        continue;
                    Entry entry = edgeSerializer.writeRelation(janusgraphRelation, wrappedType, pos, writeTx);
                    if (pos == 0) {
                        outAdditions.add(entry);
                    } else {
                        assert pos == 1;
                        InternalVertex otherVertex = janusgraphRelation.getVertex(1);
                        StaticBuffer otherVertexKey = writeTx.getIdInspector().getKey(otherVertex.longId());
                        inAdditionsMap.computeIfAbsent(otherVertexKey, k -> new ArrayList<>()).add(entry);
                    }
                }
            }
            // Mutating all OUT relationships for the current vertex
            StaticBuffer vertexKey = writeTx.getIdInspector().getKey(vertex.longId());
            mutator.mutateEdges(vertexKey, outAdditions, KCVSCache.NO_DELETIONS);
            // Mutating all IN relationships for the current vertex
            int totalInAdditions = 0;
            for (Map.Entry<StaticBuffer, List<Entry>> entry : inAdditionsMap.entrySet()) {
                StaticBuffer otherVertexKey = entry.getKey();
                List<Entry> inAdditions = entry.getValue();
                totalInAdditions += inAdditions.size();
                mutator.mutateEdges(otherVertexKey, inAdditions, KCVSCache.NO_DELETIONS);
            }
            metrics.incrementCustom(ADDED_RECORDS_COUNT, outAdditions.size() + totalInAdditions);
        } else if (index instanceof JanusGraphIndex) {
            IndexType indexType = managementSystem.getSchemaVertex(index).asIndexType();
            assert indexType != null;
            IndexSerializer indexSerializer = graph.getIndexSerializer();
            // Gather elements to index
            List<JanusGraphElement> elements;
            switch(indexType.getElement()) {
                case VERTEX:
                    elements = Collections.singletonList(vertex);
                    break;
                case PROPERTY:
                    elements = new ArrayList<>();
                    addIndexSchemaConstraint(vertex.query(), indexType).properties().forEach(elements::add);
                    break;
                case EDGE:
                    elements = new ArrayList<>();
                    addIndexSchemaConstraint(vertex.query().direction(Direction.OUT), indexType).edges().forEach(elements::add);
                    break;
                default:
                    throw new AssertionError("Unexpected category: " + indexType.getElement());
            }
            if (indexType.isCompositeIndex()) {
                for (JanusGraphElement element : elements) {
                    Set<IndexSerializer.IndexUpdate<StaticBuffer, Entry>> updates = indexSerializer.reindexElement(element, (CompositeIndexType) indexType);
                    for (IndexSerializer.IndexUpdate<StaticBuffer, Entry> update : updates) {
                        log.debug("Mutating index {}: {}", indexType, update.getEntry());
                        mutator.mutateIndex(update.getKey(), new ArrayList<Entry>(1) {

                            {
                                add(update.getEntry());
                            }
                        }, KCVSCache.NO_DELETIONS);
                        metrics.incrementCustom(ADDED_RECORDS_COUNT);
                    }
                }
            } else {
                assert indexType.isMixedIndex();
                for (JanusGraphElement element : elements) {
                    if (indexSerializer.reindexElement(element, (MixedIndexType) indexType, documentsPerStore)) {
                        metrics.incrementCustom(DOCUMENT_UPDATES_COUNT);
                    }
                }
            }
        } else
            throw new UnsupportedOperationException("Unsupported index found: " + index);
    } catch (final Exception e) {
        managementSystem.rollback();
        writeTx.rollback();
        metrics.incrementCustom(FAILED_TX);
        throw new JanusGraphException(e.getMessage(), e);
    }
}
Also used : InternalVertex(org.janusgraph.graphdb.internal.InternalVertex) JanusGraphVertex(org.janusgraph.core.JanusGraphVertex) StringUtils(org.janusgraph.util.StringUtils) BaseVertexQuery(org.janusgraph.core.BaseVertexQuery) BackendTransaction(org.janusgraph.diskstorage.BackendTransaction) HashMap(java.util.HashMap) SchemaAction(org.janusgraph.core.schema.SchemaAction) IndexSerializer(org.janusgraph.graphdb.database.IndexSerializer) ArrayList(java.util.ArrayList) ScanMetrics(org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics) IndexEntry(org.janusgraph.diskstorage.indexing.IndexEntry) SchemaStatus(org.janusgraph.core.schema.SchemaStatus) QueryContainer(org.janusgraph.graphdb.olap.QueryContainer) JanusGraphIndex(org.janusgraph.core.schema.JanusGraphIndex) InternalRelation(org.janusgraph.graphdb.internal.InternalRelation) Map(java.util.Map) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) JanusGraphException(org.janusgraph.core.JanusGraphException) IndexType(org.janusgraph.graphdb.types.IndexType) JanusGraphElement(org.janusgraph.core.JanusGraphElement) JanusGraphSchemaType(org.janusgraph.core.schema.JanusGraphSchemaType) BackendException(org.janusgraph.diskstorage.BackendException) RelationType(org.janusgraph.core.RelationType) VertexScanJob(org.janusgraph.graphdb.olap.VertexScanJob) CompositeIndexType(org.janusgraph.graphdb.types.CompositeIndexType) PropertyKey(org.janusgraph.core.PropertyKey) KCVSCache(org.janusgraph.diskstorage.keycolumnvalue.cache.KCVSCache) BaseLabel(org.janusgraph.graphdb.types.system.BaseLabel) Set(java.util.Set) JanusGraphSchemaVertex(org.janusgraph.graphdb.types.vertices.JanusGraphSchemaVertex) EdgeSerializer(org.janusgraph.graphdb.database.EdgeSerializer) EdgeDirection(org.janusgraph.graphdb.relations.EdgeDirection) Direction(org.apache.tinkerpop.gremlin.structure.Direction) List(java.util.List) Entry(org.janusgraph.diskstorage.Entry) Preconditions(com.google.common.base.Preconditions) RelationTypeIndex(org.janusgraph.core.schema.RelationTypeIndex) RelationTypeIndexWrapper(org.janusgraph.graphdb.database.management.RelationTypeIndexWrapper) InternalRelationType(org.janusgraph.graphdb.internal.InternalRelationType) Collections(java.util.Collections) Set(java.util.Set) HashMap(java.util.HashMap) JanusGraphException(org.janusgraph.core.JanusGraphException) ArrayList(java.util.ArrayList) InternalRelation(org.janusgraph.graphdb.internal.InternalRelation) RelationTypeIndex(org.janusgraph.core.schema.RelationTypeIndex) IndexEntry(org.janusgraph.diskstorage.indexing.IndexEntry) Entry(org.janusgraph.diskstorage.Entry) JanusGraphElement(org.janusgraph.core.JanusGraphElement) EdgeSerializer(org.janusgraph.graphdb.database.EdgeSerializer) RelationTypeIndexWrapper(org.janusgraph.graphdb.database.management.RelationTypeIndexWrapper) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) ArrayList(java.util.ArrayList) List(java.util.List) JanusGraphIndex(org.janusgraph.core.schema.JanusGraphIndex) MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) IndexType(org.janusgraph.graphdb.types.IndexType) CompositeIndexType(org.janusgraph.graphdb.types.CompositeIndexType) BackendTransaction(org.janusgraph.diskstorage.BackendTransaction) MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) IndexSerializer(org.janusgraph.graphdb.database.IndexSerializer) JanusGraphException(org.janusgraph.core.JanusGraphException) BackendException(org.janusgraph.diskstorage.BackendException) InternalVertex(org.janusgraph.graphdb.internal.InternalVertex) CompositeIndexType(org.janusgraph.graphdb.types.CompositeIndexType) InternalRelationType(org.janusgraph.graphdb.internal.InternalRelationType) HashMap(java.util.HashMap) Map(java.util.Map)

Example 2 with MixedIndexType

use of org.janusgraph.graphdb.types.MixedIndexType in project janusgraph by JanusGraph.

the class StandardTransactionLogProcessor method restoreExternalIndexes.

private void restoreExternalIndexes(Predicate<String> isFailedIndex, TransactionLogHeader.Entry entry) {
    // 1) Collect all elements (vertices and relations) and the indexes for which they need to be restored
    SetMultimap<String, IndexRestore> indexRestores = HashMultimap.create();
    BackendOperation.execute(() -> {
        final StandardJanusGraphTx tx = (StandardJanusGraphTx) graph.newTransaction();
        try {
            entry.getContentAsModifications(serializer).stream().map(m -> ModificationDeserializer.parseRelation(m, tx)).forEach(rel -> {
                // Collect affected vertex indexes
                for (final MixedIndexType index : getMixedIndexes(rel.getType())) {
                    if (index.getElement() == ElementCategory.VERTEX && isFailedIndex.apply(index.getBackingIndexName())) {
                        assert rel.isProperty();
                        indexRestores.put(index.getBackingIndexName(), new IndexRestore(rel.getVertex(0).longId(), ElementCategory.VERTEX, getIndexId(index)));
                    }
                }
                // See if relation itself is affected
                for (final RelationType relType : rel.getPropertyKeysDirect()) {
                    for (final MixedIndexType index : getMixedIndexes(relType)) {
                        if (index.getElement().isInstance(rel) && isFailedIndex.apply(index.getBackingIndexName())) {
                            assert rel.id() instanceof RelationIdentifier;
                            indexRestores.put(index.getBackingIndexName(), new IndexRestore(rel.id(), ElementCategory.getByClazz(rel.getClass()), getIndexId(index)));
                        }
                    }
                }
            });
        } finally {
            if (tx.isOpen())
                tx.rollback();
        }
        return true;
    }, readTime);
    // 2) Restore elements per backing index
    for (final String indexName : indexRestores.keySet()) {
        final StandardJanusGraphTx tx = (StandardJanusGraphTx) graph.newTransaction();
        try {
            BackendTransaction btx = tx.getTxHandle();
            final IndexTransaction indexTx = btx.getIndexTransaction(indexName);
            BackendOperation.execute(new Callable<Boolean>() {

                @Override
                public Boolean call() throws Exception {
                    Map<String, Map<String, List<IndexEntry>>> restoredDocs = Maps.newHashMap();
                    indexRestores.get(indexName).forEach(restore -> {
                        JanusGraphSchemaVertex indexV = (JanusGraphSchemaVertex) tx.getVertex(restore.indexId);
                        MixedIndexType index = (MixedIndexType) indexV.asIndexType();
                        JanusGraphElement element = restore.retrieve(tx);
                        if (element != null) {
                            graph.getIndexSerializer().reindexElement(element, index, restoredDocs);
                        } else {
                            // Element is deleted
                            graph.getIndexSerializer().removeElement(restore.elementId, index, restoredDocs);
                        }
                    });
                    indexTx.restore(restoredDocs);
                    indexTx.commit();
                    return true;
                }

                @Override
                public String toString() {
                    return "IndexMutation";
                }
            }, persistenceTime);
        } finally {
            if (tx.isOpen())
                tx.rollback();
        }
    }
}
Also used : ReadBuffer(org.janusgraph.diskstorage.ReadBuffer) Log(org.janusgraph.diskstorage.log.Log) StandardJanusGraphTx(org.janusgraph.graphdb.transaction.StandardJanusGraphTx) MessageReader(org.janusgraph.diskstorage.log.MessageReader) BackendOperation(org.janusgraph.diskstorage.util.BackendOperation) LoggerFactory(org.slf4j.LoggerFactory) JanusGraphTransaction(org.janusgraph.core.JanusGraphTransaction) Future(java.util.concurrent.Future) HashMultimap(com.google.common.collect.HashMultimap) Serializer(org.janusgraph.graphdb.database.serialize.Serializer) IndexEntry(org.janusgraph.diskstorage.indexing.IndexEntry) Duration(java.time.Duration) Map(java.util.Map) StaticBuffer(org.janusgraph.diskstorage.StaticBuffer) MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) IndexType(org.janusgraph.graphdb.types.IndexType) LogTxStatus(org.janusgraph.graphdb.database.log.LogTxStatus) TransactionRecovery(org.janusgraph.core.log.TransactionRecovery) RelationType(org.janusgraph.core.RelationType) TimestampProvider(org.janusgraph.diskstorage.util.time.TimestampProvider) StandardJanusGraph(org.janusgraph.graphdb.database.StandardJanusGraph) JanusGraphSchemaVertex(org.janusgraph.graphdb.types.vertices.JanusGraphSchemaVertex) LogTxMeta(org.janusgraph.graphdb.database.log.LogTxMeta) Instant(java.time.Instant) Objects(java.util.Objects) List(java.util.List) RelationIdentifier(org.janusgraph.graphdb.relations.RelationIdentifier) Predicate(com.google.common.base.Predicate) CacheBuilder(com.google.common.cache.CacheBuilder) InternalRelationType(org.janusgraph.graphdb.internal.InternalRelationType) Iterables(com.google.common.collect.Iterables) SchemaSource(org.janusgraph.graphdb.types.SchemaSource) BackgroundThread(org.janusgraph.util.system.BackgroundThread) IndexTransaction(org.janusgraph.diskstorage.indexing.IndexTransaction) BackendTransaction(org.janusgraph.diskstorage.BackendTransaction) Callable(java.util.concurrent.Callable) IndexTypeWrapper(org.janusgraph.graphdb.types.indextype.IndexTypeWrapper) Predicates(com.google.common.base.Predicates) JanusGraphException(org.janusgraph.core.JanusGraphException) Message(org.janusgraph.diskstorage.log.Message) JanusGraphElement(org.janusgraph.core.JanusGraphElement) TransactionLogHeader(org.janusgraph.graphdb.database.log.TransactionLogHeader) Logger(org.slf4j.Logger) Maps(com.google.common.collect.Maps) SetMultimap(com.google.common.collect.SetMultimap) ElementCategory(org.janusgraph.graphdb.internal.ElementCategory) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) RemovalCause(com.google.common.cache.RemovalCause) GraphDatabaseConfiguration(org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration) Preconditions(com.google.common.base.Preconditions) RemovalListener(com.google.common.cache.RemovalListener) Cache(com.google.common.cache.Cache) ReadMarker(org.janusgraph.diskstorage.log.ReadMarker) Collections(java.util.Collections) MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) IndexTransaction(org.janusgraph.diskstorage.indexing.IndexTransaction) RelationIdentifier(org.janusgraph.graphdb.relations.RelationIdentifier) JanusGraphException(org.janusgraph.core.JanusGraphException) ExecutionException(java.util.concurrent.ExecutionException) JanusGraphElement(org.janusgraph.core.JanusGraphElement) StandardJanusGraphTx(org.janusgraph.graphdb.transaction.StandardJanusGraphTx) RelationType(org.janusgraph.core.RelationType) InternalRelationType(org.janusgraph.graphdb.internal.InternalRelationType) JanusGraphSchemaVertex(org.janusgraph.graphdb.types.vertices.JanusGraphSchemaVertex) List(java.util.List) Map(java.util.Map) BackendTransaction(org.janusgraph.diskstorage.BackendTransaction)

Example 3 with MixedIndexType

use of org.janusgraph.graphdb.types.MixedIndexType in project janusgraph by JanusGraph.

the class IndexSerializer method executeQuery.

public Stream<RawQuery.Result> executeQuery(IndexQueryBuilder query, final ElementCategory resultType, final BackendTransaction backendTx, final StandardJanusGraphTx transaction) {
    final MixedIndexType index = getMixedIndex(query.getIndex(), transaction);
    final String queryStr = createQueryString(query, resultType, transaction, index);
    ImmutableList<IndexQuery.OrderEntry> orders = getOrders(query, resultType, transaction, index);
    final RawQuery rawQuery = new RawQuery(index.getStoreName(), queryStr, orders, query.getParameters());
    if (query.hasLimit())
        rawQuery.setLimit(query.getLimit());
    rawQuery.setOffset(query.getOffset());
    return backendTx.rawQuery(index.getBackingIndexName(), rawQuery).map(result -> new RawQuery.Result(string2ElementId(result.getResult()), result.getScore()));
}
Also used : MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) RawQuery(org.janusgraph.diskstorage.indexing.RawQuery)

Example 4 with MixedIndexType

use of org.janusgraph.graphdb.types.MixedIndexType in project janusgraph by JanusGraph.

the class IndexSerializer method getIndexUpdates.

public Collection<IndexUpdate> getIndexUpdates(InternalRelation relation) {
    assert relation.isNew() || relation.isRemoved();
    final Set<IndexUpdate> updates = Sets.newHashSet();
    final IndexUpdate.Type updateType = getUpdateType(relation);
    final int ttl = updateType == IndexUpdate.Type.ADD ? StandardJanusGraph.getTTL(relation) : 0;
    for (final PropertyKey type : relation.getPropertyKeysDirect()) {
        if (type == null)
            continue;
        for (final IndexType index : ((InternalRelationType) type).getKeyIndexes()) {
            if (!indexAppliesTo(index, relation))
                continue;
            IndexUpdate update;
            if (index instanceof CompositeIndexType) {
                final CompositeIndexType iIndex = (CompositeIndexType) index;
                final RecordEntry[] record = indexMatch(relation, iIndex);
                if (record == null)
                    continue;
                update = new IndexUpdate<>(iIndex, updateType, getIndexKey(iIndex, record), getIndexEntry(iIndex, record, relation), relation);
            } else {
                assert relation.valueOrNull(type) != null;
                if (((MixedIndexType) index).getField(type).getStatus() == SchemaStatus.DISABLED)
                    continue;
                update = getMixedIndexUpdate(relation, type, relation.valueOrNull(type), (MixedIndexType) index, updateType);
            }
            if (ttl > 0)
                update.setTTL(ttl);
            updates.add(update);
        }
    }
    return updates;
}
Also used : MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) CompositeIndexType(org.janusgraph.graphdb.types.CompositeIndexType) InternalRelationType(org.janusgraph.graphdb.internal.InternalRelationType) MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) IndexType(org.janusgraph.graphdb.types.IndexType) CompositeIndexType(org.janusgraph.graphdb.types.CompositeIndexType) PropertyKey(org.janusgraph.core.PropertyKey)

Example 5 with MixedIndexType

use of org.janusgraph.graphdb.types.MixedIndexType in project janusgraph by JanusGraph.

the class IndexSerializer method executeTotals.

public Long executeTotals(IndexQueryBuilder query, final ElementCategory resultType, final BackendTransaction backendTx, final StandardJanusGraphTx transaction) {
    final MixedIndexType index = getMixedIndex(query.getIndex(), transaction);
    final String queryStr = createQueryString(query, resultType, transaction, index);
    final RawQuery rawQuery = new RawQuery(index.getStoreName(), queryStr, query.getParameters());
    if (query.hasLimit())
        rawQuery.setLimit(query.getLimit());
    rawQuery.setOffset(query.getOffset());
    return backendTx.totals(index.getBackingIndexName(), rawQuery);
}
Also used : MixedIndexType(org.janusgraph.graphdb.types.MixedIndexType) RawQuery(org.janusgraph.diskstorage.indexing.RawQuery)

Aggregations

MixedIndexType (org.janusgraph.graphdb.types.MixedIndexType)18 IndexType (org.janusgraph.graphdb.types.IndexType)11 CompositeIndexType (org.janusgraph.graphdb.types.CompositeIndexType)10 Map (java.util.Map)7 JanusGraphElement (org.janusgraph.core.JanusGraphElement)6 IndexEntry (org.janusgraph.diskstorage.indexing.IndexEntry)6 HashSet (java.util.HashSet)5 JanusGraphException (org.janusgraph.core.JanusGraphException)5 PropertyKey (org.janusgraph.core.PropertyKey)5 ParameterIndexField (org.janusgraph.graphdb.types.ParameterIndexField)5 JanusGraphSchemaVertex (org.janusgraph.graphdb.types.vertices.JanusGraphSchemaVertex)5 ArrayList (java.util.ArrayList)4 HashMap (java.util.HashMap)4 Set (java.util.Set)4 BackendException (org.janusgraph.diskstorage.BackendException)4 InternalRelationType (org.janusgraph.graphdb.internal.InternalRelationType)4 Preconditions (com.google.common.base.Preconditions)3 Collections (java.util.Collections)3 List (java.util.List)3 Test (org.junit.jupiter.api.Test)3