Examples with IndexRule org.neo4j.kernel.impl.store.record.IndexRule used on opensource projects

Example 26 with IndexRule

use of org.neo4j.kernel.impl.store.record.IndexRule in project neo4j by neo4j.

the class SchemaStoreTest method storeAndLoadSchemaRule.

@Test
public void storeAndLoadSchemaRule() throws Exception {
    // GIVEN
    IndexRule indexRule = IndexRule.indexRule(store.nextId(), NewIndexDescriptorFactory.forLabel(1, 4), PROVIDER_DESCRIPTOR);
    // WHEN
    IndexRule readIndexRule = (IndexRule) SchemaRuleSerialization.deserialize(indexRule.getId(), wrap(indexRule.serialize()));
    // THEN
    assertEquals(indexRule.getId(), readIndexRule.getId());
    assertEquals(indexRule.schema(), readIndexRule.schema());
    assertEquals(indexRule.getIndexDescriptor(), readIndexRule.getIndexDescriptor());
    assertEquals(indexRule.getProviderDescriptor(), readIndexRule.getProviderDescriptor());
}

Example 27 with IndexRule

use of org.neo4j.kernel.impl.store.record.IndexRule in project neo4j by neo4j.

the class ConsistencyCheckTasks method createTasksForFullCheck.

public List<ConsistencyCheckerTask> createTasksForFullCheck(boolean checkLabelScanStore, boolean checkIndexes, boolean checkGraph) {
    List<ConsistencyCheckerTask> tasks = new ArrayList<>();
    if (checkGraph) {
        MandatoryProperties mandatoryProperties = new MandatoryProperties(nativeStores);
        StoreProcessor processor = multiPass.processor(CheckStage.Stage1_NS_PropsLabels, PROPERTIES);
        tasks.add(create(CheckStage.Stage1_NS_PropsLabels.name(), nativeStores.getNodeStore(), processor, ROUND_ROBIN));
        //ReltionshipStore pass - check label counts using cached labels, check properties, skip nodes and relationships
        processor = multiPass.processor(CheckStage.Stage2_RS_Labels, LABELS);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckForwardPass());
        tasks.add(create(CheckStage.Stage2_RS_Labels.name(), nativeStores.getRelationshipStore(), processor, ROUND_ROBIN));
        //NodeStore pass - just cache nextRel and inUse
        tasks.add(new CacheTask.CacheNextRel(CheckStage.Stage3_NS_NextRel, cacheAccess, Scanner.scan(nativeStores.getNodeStore())));
        //RelationshipStore pass - check nodes inUse, FirstInFirst, FirstInSecond using cached info
        processor = multiPass.processor(CheckStage.Stage4_RS_NextRel, NODES);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckBackwardPass(new PropertyChain<>(mandatoryProperties.forRelationships(reporter))));
        tasks.add(create(CheckStage.Stage4_RS_NextRel.name(), nativeStores.getRelationshipStore(), processor, ROUND_ROBIN));
        //NodeStore pass - just cache nextRel and inUse
        multiPass.reDecorateNode(processor, NodeRecordCheck.toCheckNextRel(), true);
        multiPass.reDecorateNode(processor, NodeRecordCheck.toCheckNextRelationshipGroup(), false);
        tasks.add(new CacheTask.CheckNextRel(CheckStage.Stage5_Check_NextRel, cacheAccess, nativeStores, processor));
        // source chain
        //RelationshipStore pass - forward scan of source chain using the cache.
        processor = multiPass.processor(CheckStage.Stage6_RS_Forward, RELATIONSHIPS);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckSourceChain());
        tasks.add(create(CheckStage.Stage6_RS_Forward.name(), nativeStores.getRelationshipStore(), processor, QueueDistribution.RELATIONSHIPS));
        //RelationshipStore pass - reverse scan of source chain using the cache.
        processor = multiPass.processor(CheckStage.Stage7_RS_Backward, RELATIONSHIPS);
        multiPass.reDecorateRelationship(processor, RelationshipRecordCheck.relationshipRecordCheckSourceChain());
        tasks.add(create(CheckStage.Stage7_RS_Backward.name(), nativeStores.getRelationshipStore(), processor, QueueDistribution.RELATIONSHIPS));
        //relationshipGroup
        StoreProcessor relGrpProcessor = multiPass.processor(Stage.PARALLEL_FORWARD, RELATIONSHIP_GROUPS);
        tasks.add(create("RelationshipGroupStore-RelGrp", nativeStores.getRelationshipGroupStore(), relGrpProcessor, ROUND_ROBIN));
        PropertyReader propertyReader = new PropertyReader(nativeStores);
        tasks.add(recordScanner(CheckStage.Stage8_PS_Props.name(), new IterableStore<>(nativeStores.getNodeStore(), true), new PropertyAndNode2LabelIndexProcessor(reporter, checkIndexes ? indexes : null, propertyReader, cacheAccess, mandatoryProperties.forNodes(reporter)), CheckStage.Stage8_PS_Props, ROUND_ROBIN, new IterableStore<>(nativeStores.getPropertyStore(), true)));
        tasks.add(create("StringStore-Str", nativeStores.getStringStore(), multiPass.processor(Stage.SEQUENTIAL_FORWARD, STRINGS), ROUND_ROBIN));
        tasks.add(create("ArrayStore-Arrays", nativeStores.getArrayStore(), multiPass.processor(Stage.SEQUENTIAL_FORWARD, ARRAYS), ROUND_ROBIN));
    }
    // The schema store is verified in multiple passes that share state since it fits into memory
    // and we care about the consistency of back references (cf. SemanticCheck)
    // PASS 1: Dynamic record chains
    tasks.add(create("SchemaStore", nativeStores.getSchemaStore(), ROUND_ROBIN));
    // PASS 2: Rule integrity and obligation build up
    final SchemaRecordCheck schemaCheck = new SchemaRecordCheck(new SchemaStorage(nativeStores.getSchemaStore()));
    tasks.add(new SchemaStoreProcessorTask<>("SchemaStoreProcessor-check_rules", statistics, numberOfThreads, nativeStores.getSchemaStore(), nativeStores, "check_rules", schemaCheck, progress, cacheAccess, defaultProcessor, ROUND_ROBIN));
    // PASS 3: Obligation verification and semantic rule uniqueness
    tasks.add(new SchemaStoreProcessorTask<>("SchemaStoreProcessor-check_obligations", statistics, numberOfThreads, nativeStores.getSchemaStore(), nativeStores, "check_obligations", schemaCheck.forObligationChecking(), progress, cacheAccess, defaultProcessor, ROUND_ROBIN));
    if (checkGraph) {
        tasks.add(create("RelationshipTypeTokenStore", nativeStores.getRelationshipTypeTokenStore(), ROUND_ROBIN));
        tasks.add(create("PropertyKeyTokenStore", nativeStores.getPropertyKeyTokenStore(), ROUND_ROBIN));
        tasks.add(create("LabelTokenStore", nativeStores.getLabelTokenStore(), ROUND_ROBIN));
        tasks.add(create("RelationshipTypeNameStore", nativeStores.getRelationshipTypeNameStore(), ROUND_ROBIN));
        tasks.add(create("PropertyKeyNameStore", nativeStores.getPropertyKeyNameStore(), ROUND_ROBIN));
        tasks.add(create("LabelNameStore", nativeStores.getLabelNameStore(), ROUND_ROBIN));
        tasks.add(create("NodeDynamicLabelStore", nativeStores.getNodeDynamicLabelStore(), ROUND_ROBIN));
    }
    ConsistencyReporter filteredReporter = multiPass.reporter(NODES);
    if (checkLabelScanStore) {
        tasks.add(recordScanner("LabelScanStore", labelScanStore.allNodeLabelRanges(), new LabelScanDocumentProcessor(filteredReporter, new LabelScanCheck()), Stage.SEQUENTIAL_FORWARD, ROUND_ROBIN));
    }
    if (checkIndexes) {
        for (IndexRule indexRule : indexes.rules()) {
            tasks.add(recordScanner(format("Index_%d", indexRule.getId()), new IndexIterator(indexes.accessorFor(indexRule)), new IndexEntryProcessor(filteredReporter, new IndexCheck(indexRule)), Stage.SEQUENTIAL_FORWARD, ROUND_ROBIN));
        }
    }
    return tasks;
}

Example 28 with IndexRule

use of org.neo4j.kernel.impl.store.record.IndexRule in project neo4j by neo4j.

the class PropertyAndNodeIndexedCheck method checkIndexToLabels.

private void checkIndexToLabels(NodeRecord record, CheckerEngine<NodeRecord, ConsistencyReport.NodeConsistencyReport> engine, RecordAccess records, Collection<PropertyRecord> propertyRecs) {
    Set<Long> labels = NodeLabelReader.getListOfLabels(record, records, engine);
    List<PropertyBlock> properties = null;
    for (IndexRule indexRule : indexes.rules()) {
        long labelId = indexRule.schema().getLabelId();
        if (!labels.contains(labelId)) {
            continue;
        }
        if (properties == null) {
            properties = propertyReader.propertyBlocks(propertyRecs);
        }
        // assuming 1 property always
        int propertyId = indexRule.schema().getPropertyId();
        PropertyBlock property = propertyWithKey(properties, propertyId);
        if (property == null) {
            continue;
        }
        try (IndexReader reader = indexes.accessorFor(indexRule).newReader()) {
            Object propertyValue = propertyReader.propertyValue(property).value();
            long nodeId = record.getId();
            if (indexRule.canSupportUniqueConstraint()) {
                verifyNodeCorrectlyIndexedUniquely(nodeId, property.getKeyIndexId(), propertyValue, engine, indexRule, reader);
            } else {
                verifyNodeCorrectlyIndexed(nodeId, propertyValue, engine, indexRule, reader);
            }
        }
    }
}

Example 29 with IndexRule

use of org.neo4j.kernel.impl.store.record.IndexRule in project neo4j by neo4j.

the class IndexingService method init.

/**
     * Called while the database starts up, before recovery.
     */
@Override
public void init() {
    IndexMap indexMap = indexMapRef.indexMapSnapshot();
    for (IndexRule indexRule : indexRules) {
        IndexProxy indexProxy;
        long indexId = indexRule.getId();
        NewIndexDescriptor descriptor = indexRule.getIndexDescriptor();
        SchemaIndexProvider.Descriptor providerDescriptor = indexRule.getProviderDescriptor();
        SchemaIndexProvider provider = providerMap.apply(providerDescriptor);
        InternalIndexState initialState = provider.getInitialState(indexId, descriptor);
        log.info(indexStateInfo("init", indexId, initialState, descriptor));
        boolean constraintIndex = indexRule.canSupportUniqueConstraint();
        switch(initialState) {
            case ONLINE:
                indexProxy = indexProxyCreator.createOnlineIndexProxy(indexId, descriptor, providerDescriptor);
                break;
            case POPULATING:
                // The database was shut down during population, or a crash has occurred, or some other sad thing.
                if (constraintIndex && indexRule.getOwningConstraint() == null) {
                    // don't bother rebuilding if we are going to throw the index away anyhow
                    indexProxy = indexProxyCreator.createFailedIndexProxy(indexId, descriptor, providerDescriptor, failure("Constraint for index was not committed."));
                } else {
                    indexProxy = indexProxyCreator.createRecoveringIndexProxy(descriptor, providerDescriptor);
                }
                break;
            case FAILED:
                IndexPopulationFailure failure = failure(provider.getPopulationFailure(indexId));
                indexProxy = indexProxyCreator.createFailedIndexProxy(indexId, descriptor, providerDescriptor, failure);
                break;
            default:
                throw new IllegalArgumentException("" + initialState);
        }
        indexMap.putIndexProxy(indexId, indexProxy);
    }
    indexMapRef.setIndexMap(indexMap);
}

Example 30 with IndexRule

use of org.neo4j.kernel.impl.store.record.IndexRule in project neo4j by neo4j.

the class IndexingService method createIndexes.

/**
     * Creates one or more indexes. They will all be populated by one and the same store scan.
     *
     * This code is called from the transaction infrastructure during transaction commits, which means that
     * it is *vital* that it is stable, and handles errors very well. Failing here means that the entire db
     * will shut down.
     */
public void createIndexes(IndexRule... rules) throws IOException {
    IndexMap indexMap = indexMapRef.indexMapSnapshot();
    IndexPopulationJob populationJob = null;
    for (IndexRule rule : rules) {
        long ruleId = rule.getId();
        IndexProxy index = indexMap.getIndexProxy(ruleId);
        if (index != null && state == State.NOT_STARTED) {
            // During recovery we might run into this scenario:
            // - We're starting recovery on a database, where init() is called and all indexes that
            //   are found in the store, instantiated and put into the IndexMap. Among them is index X.
            // - While we recover the database we bump into a transaction creating index Y, with the
            //   same IndexDescriptor, i.e. same label/property, as X. This is possible since this took
            //   place before the creation of X.
            // - When Y is dropped in between this creation and the creation of X (it will have to be
            //   otherwise X wouldn't have had an opportunity to be created) the index is removed from
            //   the IndexMap, both by id AND descriptor.
            //
            // Because of the scenario above we need to put this created index into the IndexMap
            // again, otherwise it will disappear from the IndexMap (at least for lookup by descriptor)
            // and not be able to accept changes applied from recovery later on.
            indexMap.putIndexProxy(ruleId, index);
            indexMapRef.setIndexMap(indexMap);
            continue;
        }
        final NewIndexDescriptor descriptor = rule.getIndexDescriptor();
        SchemaIndexProvider.Descriptor providerDescriptor = rule.getProviderDescriptor();
        boolean flipToTentative = rule.canSupportUniqueConstraint();
        if (state == State.RUNNING) {
            populationJob = populationJob == null ? newIndexPopulationJob() : populationJob;
            index = indexProxyCreator.createPopulatingIndexProxy(ruleId, descriptor, providerDescriptor, flipToTentative, monitor, populationJob);
            index.start();
        } else {
            index = indexProxyCreator.createRecoveringIndexProxy(descriptor, providerDescriptor);
        }
        indexMap.putIndexProxy(rule.getId(), index);
    }
    if (populationJob != null) {
        startIndexPopulation(populationJob);
    }
    indexMapRef.setIndexMap(indexMap);
}