Examples with IndexDescriptor org.neo4j.internal.schema.IndexDescriptor used on opensource projects

Example 56 with IndexDescriptor

use of org.neo4j.internal.schema.IndexDescriptor in project neo4j by neo4j.

the class IndexingService method init.

/**
 * Called while the database starts up, before recovery.
 */
@Override
public void init() throws IOException {
    validateDefaultProviderExisting();
    try (var cursorContext = new CursorContext(pageCacheTracer.createPageCursorTracer(INIT_TAG))) {
        indexMapRef.modify(indexMap -> {
            Map<InternalIndexState, List<IndexLogRecord>> indexStates = new EnumMap<>(InternalIndexState.class);
            for (IndexDescriptor descriptor : indexDescriptors) {
                // No index (except NLI) is allowed to have the name generated for NLI.
                if (descriptor.getName().equals(IndexDescriptor.NLI_GENERATED_NAME) && !(descriptor.schema().isAnyTokenSchemaDescriptor() && descriptor.schema().entityType() == NODE)) {
                    throw new IllegalStateException("Index '" + descriptor.userDescription(tokenNameLookup) + "' is using a reserved name: '" + IndexDescriptor.NLI_GENERATED_NAME + "'. This index must be removed on an earlier version " + "to be able to use binaries for version 4.3 or newer.");
                }
                IndexProxy indexProxy;
                IndexProviderDescriptor providerDescriptor = descriptor.getIndexProvider();
                IndexProvider provider = providerMap.lookup(providerDescriptor);
                InternalIndexState initialState = provider.getInitialState(descriptor, cursorContext);
                indexStates.computeIfAbsent(initialState, internalIndexState -> new ArrayList<>()).add(new IndexLogRecord(descriptor));
                internalLog.debug(indexStateInfo("init", initialState, descriptor));
                switch(initialState) {
                    case ONLINE:
                        monitor.initialState(databaseName, descriptor, ONLINE);
                        indexProxy = indexProxyCreator.createOnlineIndexProxy(descriptor);
                        break;
                    case POPULATING:
                        // The database was shut down during population, or a crash has occurred, or some other sad thing.
                        monitor.initialState(databaseName, descriptor, POPULATING);
                        indexProxy = indexProxyCreator.createRecoveringIndexProxy(descriptor);
                        break;
                    case FAILED:
                        monitor.initialState(databaseName, descriptor, FAILED);
                        IndexPopulationFailure failure = failure(provider.getPopulationFailure(descriptor, cursorContext));
                        indexProxy = indexProxyCreator.createFailedIndexProxy(descriptor, failure);
                        break;
                    default:
                        throw new IllegalArgumentException("" + initialState);
                }
                indexMap.putIndexProxy(indexProxy);
            }
            logIndexStateSummary("init", indexStates);
            return indexMap;
        });
    }
    indexStatisticsStore.init();
}

Example 57 with IndexDescriptor

use of org.neo4j.internal.schema.IndexDescriptor in project neo4j by neo4j.

the class ConstraintIndexCreator method createUniquenessConstraintIndex.

/**
 * You MUST hold a label write lock before you call this method.
 * However the label write lock is temporarily released while populating the index backing the constraint.
 * It goes a little like this:
 * <ol>
 * <li>Prerequisite: Getting here means that there's an open schema transaction which has acquired the
 * LABEL WRITE lock.</li>
 * <li>Index schema rule which is backing the constraint is created in a nested mini-transaction
 * which doesn't acquire any locking, merely adds tx state and commits so that the index rule is applied
 * to the store, which triggers the index population</li>
 * <li>Release the LABEL WRITE lock</li>
 * <li>Await index population to complete</li>
 * <li>Acquire the LABEL WRITE lock (effectively blocking concurrent transactions changing
 * data related to this constraint, and it so happens, most other transactions as well) and verify
 * the uniqueness of the built index</li>
 * <li>Leave this method, knowing that the uniqueness constraint rule will be added to tx state
 * and this tx committed, which will create the uniqueness constraint</li>
 * </ol>
 */
public IndexDescriptor createUniquenessConstraintIndex(KernelTransactionImplementation transaction, IndexBackedConstraintDescriptor constraint, IndexPrototype prototype) throws TransactionFailureException, CreateConstraintFailureException, UniquePropertyValueValidationException, AlreadyConstrainedException {
    String constraintString = constraint.userDescription(transaction.tokenRead());
    log.info("Starting constraint creation: %s.", constraintString);
    IndexDescriptor index;
    SchemaRead schemaRead = transaction.schemaRead();
    try {
        index = checkAndCreateConstraintIndex(schemaRead, transaction.tokenRead(), constraint, prototype);
    } catch (AlreadyConstrainedException e) {
        throw e;
    } catch (KernelException e) {
        throw new CreateConstraintFailureException(constraint, e);
    }
    boolean success = false;
    boolean reacquiredLabelLock = false;
    Client locks = transaction.lockClient();
    ResourceType keyType = constraint.schema().keyType();
    long[] lockingKeys = constraint.schema().lockingKeys();
    try {
        locks.acquireShared(transaction.lockTracer(), keyType, lockingKeys);
        IndexProxy proxy = indexingService.getIndexProxy(index);
        // Release the LABEL WRITE lock during index population.
        // At this point the integrity of the constraint to be created was checked
        // while holding the lock and the index rule backing the soon-to-be-created constraint
        // has been created. Now it's just the population left, which can take a long time
        locks.releaseExclusive(keyType, lockingKeys);
        awaitConstraintIndexPopulation(constraint, proxy, transaction);
        log.info("Constraint %s populated, starting verification.", constraintString);
        // Index population was successful, but at this point we don't know if the uniqueness constraint holds.
        // Acquire LABEL WRITE lock and verify the constraints here in this user transaction
        // and if everything checks out then it will be held until after the constraint has been
        // created and activated.
        locks.acquireExclusive(transaction.lockTracer(), keyType, lockingKeys);
        reacquiredLabelLock = true;
        try (NodePropertyAccessor propertyAccessor = new DefaultNodePropertyAccessor(transaction.newStorageReader(), transaction.cursorContext(), transaction.memoryTracker())) {
            indexingService.getIndexProxy(index).verifyDeferredConstraints(propertyAccessor);
        }
        log.info("Constraint %s verified.", constraintString);
        success = true;
        return index;
    } catch (IndexNotFoundKernelException e) {
        String indexString = index.userDescription(transaction.tokenRead());
        throw new TransactionFailureException(format("Index (%s) that we just created does not exist.", indexString), e);
    } catch (IndexEntryConflictException e) {
        throw new UniquePropertyValueValidationException(constraint, VERIFICATION, e, transaction.tokenRead());
    } catch (InterruptedException | IOException e) {
        throw new CreateConstraintFailureException(constraint, e);
    } finally {
        if (!success) {
            if (!reacquiredLabelLock) {
                locks.acquireExclusive(transaction.lockTracer(), keyType, lockingKeys);
            }
            if (indexStillExists(schemaRead, index)) {
                dropUniquenessConstraintIndex(index);
            }
        }
    }
}

Example 58 with IndexDescriptor

use of org.neo4j.internal.schema.IndexDescriptor in project neo4j by neo4j.

the class TxState method accept.

@Override
public void accept(final TxStateVisitor visitor) throws KernelException {
    if (nodes != null) {
        nodes.getAdded().each(visitor::visitCreatedNode);
    }
    if (relationships != null) {
        try (HeapTrackingArrayList<NodeRelationshipIds> sortedNodeRelState = HeapTrackingArrayList.newArrayList(nodeStatesMap.size(), memoryTracker)) {
            nodeStatesMap.forEachValue(nodeState -> {
                if (nodeState.isDeleted() && nodeState.isAddedInThisTx()) {
                    return;
                }
                if (nodeState.hasAddedRelationships() || nodeState.hasRemovedRelationships()) {
                    sortedNodeRelState.add(StateNodeRelationshipIds.createStateNodeRelationshipIds(nodeState, this::relationshipVisit, memoryTracker));
                }
            });
            sortedNodeRelState.sort(Comparator.comparingLong(NodeRelationshipIds::nodeId));
            // Visit relationships, this will grab all the locks needed to do the updates
            visitor.visitRelationshipModifications(new RelationshipModifications() {

                @Override
                public void forEachSplit(IdsVisitor visitor) {
                    sortedNodeRelState.forEach(visitor);
                }

                @Override
                public RelationshipBatch creations() {
                    return idsAsBatch(relationships.getAdded(), TxState.this::relationshipVisit);
                }

                @Override
                public RelationshipBatch deletions() {
                    return idsAsBatch(relationships.getRemoved());
                }
            });
        }
    }
    if (nodes != null) {
        nodes.getRemoved().each(visitor::visitDeletedNode);
    }
    for (NodeState node : modifiedNodes()) {
        if (node.hasPropertyChanges()) {
            visitor.visitNodePropertyChanges(node.getId(), node.addedProperties(), node.changedProperties(), node.removedProperties());
        }
        final LongDiffSets labelDiffSets = node.labelDiffSets();
        if (!labelDiffSets.isEmpty()) {
            visitor.visitNodeLabelChanges(node.getId(), labelDiffSets.getAdded(), labelDiffSets.getRemoved());
        }
    }
    for (RelationshipState rel : modifiedRelationships()) {
        visitor.visitRelPropertyChanges(rel.getId(), rel.addedProperties(), rel.changedProperties(), rel.removedProperties());
    }
    if (indexChanges != null) {
        for (IndexDescriptor indexDescriptor : indexChanges.getAdded()) {
            visitor.visitAddedIndex(indexDescriptor);
        }
        indexChanges.getRemoved().forEach(visitor::visitRemovedIndex);
    }
    if (constraintsChanges != null) {
        for (ConstraintDescriptor added : constraintsChanges.getAdded()) {
            visitor.visitAddedConstraint(added);
        }
        constraintsChanges.getRemoved().forEach(visitor::visitRemovedConstraint);
    }
    if (createdLabelTokens != null) {
        createdLabelTokens.forEachKeyValue((id, token) -> visitor.visitCreatedLabelToken(id, token.name, token.internal));
    }
    if (createdPropertyKeyTokens != null) {
        createdPropertyKeyTokens.forEachKeyValue((id, token) -> visitor.visitCreatedPropertyKeyToken(id, token.name, token.internal));
    }
    if (createdRelationshipTypeTokens != null) {
        createdRelationshipTypeTokens.forEachKeyValue((id, token) -> visitor.visitCreatedRelationshipTypeToken(id, token.name, token.internal));
    }
}

Example 59 with IndexDescriptor

use of org.neo4j.internal.schema.IndexDescriptor in project neo4j by neo4j.

the class IndexSamplingController method recoverIndexSamples.

public void recoverIndexSamples() {
    samplingLock.lock();
    try {
        IndexMap indexMap = indexMapSnapshotProvider.indexMapSnapshot();
        final LongIterator indexIds = indexMap.indexIds();
        List<IndexSamplingJobHandle> asyncSamplingJobs = Lists.mutable.of();
        while (indexIds.hasNext()) {
            long indexId = indexIds.next();
            IndexDescriptor descriptor = indexMap.getIndexProxy(indexId).getDescriptor();
            if (indexRecoveryCondition.test(descriptor) && descriptor.getIndexType() != IndexType.LOOKUP) {
                if (logRecoverIndexSamples) {
                    log.info("Index requires sampling, id=%d, name=%s.", indexId, descriptor.getName());
                }
                if (asyncRecoverIndexSamples) {
                    asyncSamplingJobs.add(sampleIndexOnTracker(indexMap, indexId));
                } else {
                    sampleIndexOnCurrentThread(indexMap, indexId);
                }
            } else {
                if (logRecoverIndexSamples) {
                    log.info("Index does not require sampling, id=%d, name=%s.", indexId, descriptor.getName());
                }
            }
        }
        if (asyncRecoverIndexSamplesWait) {
            waitForAsyncIndexSamples(asyncSamplingJobs);
        }
    } finally {
        samplingLock.unlock();
    }
}

Example 60 with IndexDescriptor

use of org.neo4j.internal.schema.IndexDescriptor in project neo4j by neo4j.

the class IndexSamplingControllerFactory method createIndexRecoveryCondition.

private RecoveryCondition createIndexRecoveryCondition(final LogProvider logProvider, final TokenNameLookup tokenNameLookup) {
    return new RecoveryCondition() {

        private final Log log = logProvider.getLog(IndexSamplingController.class);

        @Override
        public boolean test(IndexDescriptor descriptor) {
            IndexSample indexSample = indexStatisticsStore.indexSample(descriptor.getId());
            long samples = indexSample.sampleSize();
            long size = indexSample.indexSize();
            boolean empty = (samples == 0) || (size == 0);
            if (empty) {
                log.debug("Recovering index sampling for index %s", descriptor.schema().userDescription(tokenNameLookup));
            }
            return empty;
        }
    };
}