Examples with IndexEntry org.janusgraph.diskstorage.indexing.IndexEntry used on opensource projects

Example 1 with IndexEntry

use of org.janusgraph.diskstorage.indexing.IndexEntry in project janusgraph by JanusGraph.

the class StandardTransactionLogProcessor method restoreExternalIndexes.

private void restoreExternalIndexes(Predicate<String> isFailedIndex, TransactionLogHeader.Entry entry) {
    // 1) Collect all elements (vertices and relations) and the indexes for which they need to be restored
    SetMultimap<String, IndexRestore> indexRestores = HashMultimap.create();
    BackendOperation.execute(() -> {
        final StandardJanusGraphTx tx = (StandardJanusGraphTx) graph.newTransaction();
        try {
            entry.getContentAsModifications(serializer).stream().map(m -> ModificationDeserializer.parseRelation(m, tx)).forEach(rel -> {
                // Collect affected vertex indexes
                for (final MixedIndexType index : getMixedIndexes(rel.getType())) {
                    if (index.getElement() == ElementCategory.VERTEX && isFailedIndex.apply(index.getBackingIndexName())) {
                        assert rel.isProperty();
                        indexRestores.put(index.getBackingIndexName(), new IndexRestore(rel.getVertex(0).longId(), ElementCategory.VERTEX, getIndexId(index)));
                    }
                }
                // See if relation itself is affected
                for (final RelationType relType : rel.getPropertyKeysDirect()) {
                    for (final MixedIndexType index : getMixedIndexes(relType)) {
                        if (index.getElement().isInstance(rel) && isFailedIndex.apply(index.getBackingIndexName())) {
                            assert rel.id() instanceof RelationIdentifier;
                            indexRestores.put(index.getBackingIndexName(), new IndexRestore(rel.id(), ElementCategory.getByClazz(rel.getClass()), getIndexId(index)));
                        }
                    }
                }
            });
        } finally {
            if (tx.isOpen())
                tx.rollback();
        }
        return true;
    }, readTime);
    // 2) Restore elements per backing index
    for (final String indexName : indexRestores.keySet()) {
        final StandardJanusGraphTx tx = (StandardJanusGraphTx) graph.newTransaction();
        try {
            BackendTransaction btx = tx.getTxHandle();
            final IndexTransaction indexTx = btx.getIndexTransaction(indexName);
            BackendOperation.execute(new Callable<Boolean>() {

                @Override
                public Boolean call() throws Exception {
                    Map<String, Map<String, List<IndexEntry>>> restoredDocs = Maps.newHashMap();
                    indexRestores.get(indexName).forEach(restore -> {
                        JanusGraphSchemaVertex indexV = (JanusGraphSchemaVertex) tx.getVertex(restore.indexId);
                        MixedIndexType index = (MixedIndexType) indexV.asIndexType();
                        JanusGraphElement element = restore.retrieve(tx);
                        if (element != null) {
                            graph.getIndexSerializer().reindexElement(element, index, restoredDocs);
                        } else {
                            // Element is deleted
                            graph.getIndexSerializer().removeElement(restore.elementId, index, restoredDocs);
                        }
                    });
                    indexTx.restore(restoredDocs);
                    indexTx.commit();
                    return true;
                }

                @Override
                public String toString() {
                    return "IndexMutation";
                }
            }, persistenceTime);
        } finally {
            if (tx.isOpen())
                tx.rollback();
        }
    }
}

Example 2 with IndexEntry

use of org.janusgraph.diskstorage.indexing.IndexEntry in project janusgraph by JanusGraph.

the class IndexRepairJob method process.

@Override
public void process(JanusGraphVertex vertex, ScanMetrics metrics) {
    try {
        BackendTransaction mutator = writeTx.getTxHandle();
        if (index instanceof RelationTypeIndex) {
            RelationTypeIndexWrapper wrapper = (RelationTypeIndexWrapper) index;
            InternalRelationType wrappedType = wrapper.getWrappedType();
            EdgeSerializer edgeSerializer = writeTx.getEdgeSerializer();
            List<Entry> additions = new ArrayList<>();
            for (Object relation : vertex.query().types(indexRelationTypeName).direction(Direction.OUT).relations()) {
                InternalRelation janusgraphRelation = (InternalRelation) relation;
                for (int pos = 0; pos < janusgraphRelation.getArity(); pos++) {
                    if (!wrappedType.isUnidirected(Direction.BOTH) && !wrappedType.isUnidirected(EdgeDirection.fromPosition(pos)))
                        // Directionality is not covered
                        continue;
                    Entry entry = edgeSerializer.writeRelation(janusgraphRelation, wrappedType, pos, writeTx);
                    additions.add(entry);
                }
            }
            StaticBuffer vertexKey = writeTx.getIdInspector().getKey(vertex.longId());
            mutator.mutateEdges(vertexKey, additions, KCVSCache.NO_DELETIONS);
            metrics.incrementCustom(ADDED_RECORDS_COUNT, additions.size());
        } else if (index instanceof JanusGraphIndex) {
            IndexType indexType = managementSystem.getSchemaVertex(index).asIndexType();
            assert indexType != null;
            IndexSerializer indexSerializer = graph.getIndexSerializer();
            // Gather elements to index
            List<JanusGraphElement> elements;
            switch(indexType.getElement()) {
                case VERTEX:
                    elements = ImmutableList.of(vertex);
                    break;
                case PROPERTY:
                    elements = Lists.newArrayList();
                    for (JanusGraphVertexProperty p : addIndexSchemaConstraint(vertex.query(), indexType).properties()) {
                        elements.add(p);
                    }
                    break;
                case EDGE:
                    elements = Lists.newArrayList();
                    for (Object e : addIndexSchemaConstraint(vertex.query().direction(Direction.OUT), indexType).edges()) {
                        elements.add((JanusGraphEdge) e);
                    }
                    break;
                default:
                    throw new AssertionError("Unexpected category: " + indexType.getElement());
            }
            if (indexType.isCompositeIndex()) {
                for (JanusGraphElement element : elements) {
                    Set<IndexSerializer.IndexUpdate<StaticBuffer, Entry>> updates = indexSerializer.reindexElement(element, (CompositeIndexType) indexType);
                    for (IndexSerializer.IndexUpdate<StaticBuffer, Entry> update : updates) {
                        log.debug("Mutating index {}: {}", indexType, update.getEntry());
                        mutator.mutateIndex(update.getKey(), Lists.newArrayList(update.getEntry()), KCVSCache.NO_DELETIONS);
                        metrics.incrementCustom(ADDED_RECORDS_COUNT);
                    }
                }
            } else {
                assert indexType.isMixedIndex();
                Map<String, Map<String, List<IndexEntry>>> documentsPerStore = new HashMap<>();
                for (JanusGraphElement element : elements) {
                    indexSerializer.reindexElement(element, (MixedIndexType) indexType, documentsPerStore);
                    metrics.incrementCustom(DOCUMENT_UPDATES_COUNT);
                }
                mutator.getIndexTransaction(indexType.getBackingIndexName()).restore(documentsPerStore);
            }
        } else
            throw new UnsupportedOperationException("Unsupported index found: " + index);
    } catch (final Exception e) {
        managementSystem.rollback();
        writeTx.rollback();
        metrics.incrementCustom(FAILED_TX);
        throw new JanusGraphException(e.getMessage(), e);
    }
}

Example 3 with IndexEntry

use of org.janusgraph.diskstorage.indexing.IndexEntry in project janusgraph by JanusGraph.

the class ElasticSearchIndex method getDeletionScript.

private String getDeletionScript(KeyInformation.IndexRetriever information, String storeName, IndexMutation mutation) throws PermanentBackendException {
    final StringBuilder script = new StringBuilder();
    final String INDEX_NAME = "index";
    int i = 0;
    for (final IndexEntry deletion : mutation.getDeletions()) {
        final KeyInformation keyInformation = information.get(storeName).get(deletion.field);
        switch(keyInformation.getCardinality()) {
            case SINGLE:
                script.append("ctx._source.remove(\"").append(deletion.field).append("\");");
                if (hasDualStringMapping(information.get(storeName, deletion.field))) {
                    script.append("ctx._source.remove(\"").append(getDualMappingName(deletion.field)).append("\");");
                }
                break;
            case SET:
            case LIST:
                final String jsValue = convertToJsType(deletion.value, compat.scriptLang(), Mapping.getMapping(keyInformation));
                String index = INDEX_NAME + i++;
                script.append("def ").append(index).append(" = ctx._source[\"").append(deletion.field).append("\"].indexOf(").append(jsValue).append("); ctx._source[\"").append(deletion.field).append("\"].remove(").append(index).append(");");
                if (hasDualStringMapping(information.get(storeName, deletion.field))) {
                    index = INDEX_NAME + i++;
                    script.append("def ").append(index).append(" = ctx._source[\"").append(getDualMappingName(deletion.field)).append("\"].indexOf(").append(jsValue).append("); ctx._source[\"").append(getDualMappingName(deletion.field)).append("\"].remove(").append(index).append(");");
                }
                break;
        }
    }
    return script.toString();
}

Example 4 with IndexEntry

use of org.janusgraph.diskstorage.indexing.IndexEntry in project janusgraph by JanusGraph.

the class ElasticSearchIndex method getAdditionScript.

private String getAdditionScript(KeyInformation.IndexRetriever information, String storeName, IndexMutation mutation) throws PermanentBackendException {
    final StringBuilder script = new StringBuilder();
    for (final IndexEntry e : mutation.getAdditions()) {
        final KeyInformation keyInformation = information.get(storeName).get(e.field);
        switch(keyInformation.getCardinality()) {
            case SET:
            case LIST:
                script.append("if(ctx._source[\"").append(e.field).append("\"] == null) ctx._source[\"").append(e.field).append("\"] = [];");
                script.append("ctx._source[\"").append(e.field).append("\"].add(").append(convertToJsType(e.value, compat.scriptLang(), Mapping.getMapping(keyInformation))).append(");");
                if (hasDualStringMapping(keyInformation)) {
                    script.append("if(ctx._source[\"").append(getDualMappingName(e.field)).append("\"] == null) ctx._source[\"").append(getDualMappingName(e.field)).append("\"] = [];");
                    script.append("ctx._source[\"").append(getDualMappingName(e.field)).append("\"].add(").append(convertToJsType(e.value, compat.scriptLang(), Mapping.getMapping(keyInformation))).append(");");
                }
                break;
            default:
                break;
        }
    }
    return script.toString();
}

Example 5 with IndexEntry

use of org.janusgraph.diskstorage.indexing.IndexEntry in project janusgraph by JanusGraph.

the class SolrIndex method restore.

@Override
public void restore(Map<String, Map<String, List<IndexEntry>>> documents, KeyInformation.IndexRetriever information, BaseTransaction tx) throws BackendException {
    try {
        for (final Map.Entry<String, Map<String, List<IndexEntry>>> stores : documents.entrySet()) {
            final String collectionName = stores.getKey();
            final List<String> deleteIds = new ArrayList<>();
            final List<SolrInputDocument> newDocuments = new ArrayList<>();
            for (final Map.Entry<String, List<IndexEntry>> entry : stores.getValue().entrySet()) {
                final String docID = entry.getKey();
                final List<IndexEntry> content = entry.getValue();
                if (content == null || content.isEmpty()) {
                    if (logger.isTraceEnabled())
                        logger.trace("Deleting document [{}]", docID);
                    deleteIds.add(docID);
                    continue;
                }
                final SolrInputDocument doc = new SolrInputDocument();
                doc.setField(getKeyFieldId(collectionName), docID);
                final Map<String, Object> adds = collectFieldValues(content, collectionName, information);
                adds.forEach(doc::setField);
                newDocuments.add(doc);
            }
            commitDeletes(collectionName, deleteIds);
            commitChanges(collectionName, newDocuments);
        }
    } catch (final Exception e) {
        throw new TemporaryBackendException("Could not restore Solr index", e);
    }
}