Examples with ONLINE org.neo4j.internal.kernel.api.InternalIndexState.ONLINE used on opensource projects

Example 1 with ONLINE

use of org.neo4j.internal.kernel.api.InternalIndexState.ONLINE in project neo4j by neo4j.

the class IndexingService method init.

/**
 * Called while the database starts up, before recovery.
 */
@Override
public void init() throws IOException {
    validateDefaultProviderExisting();
    try (var cursorContext = new CursorContext(pageCacheTracer.createPageCursorTracer(INIT_TAG))) {
        indexMapRef.modify(indexMap -> {
            Map<InternalIndexState, List<IndexLogRecord>> indexStates = new EnumMap<>(InternalIndexState.class);
            for (IndexDescriptor descriptor : indexDescriptors) {
                // No index (except NLI) is allowed to have the name generated for NLI.
                if (descriptor.getName().equals(IndexDescriptor.NLI_GENERATED_NAME) && !(descriptor.schema().isAnyTokenSchemaDescriptor() && descriptor.schema().entityType() == NODE)) {
                    throw new IllegalStateException("Index '" + descriptor.userDescription(tokenNameLookup) + "' is using a reserved name: '" + IndexDescriptor.NLI_GENERATED_NAME + "'. This index must be removed on an earlier version " + "to be able to use binaries for version 4.3 or newer.");
                }
                IndexProxy indexProxy;
                IndexProviderDescriptor providerDescriptor = descriptor.getIndexProvider();
                IndexProvider provider = providerMap.lookup(providerDescriptor);
                InternalIndexState initialState = provider.getInitialState(descriptor, cursorContext);
                indexStates.computeIfAbsent(initialState, internalIndexState -> new ArrayList<>()).add(new IndexLogRecord(descriptor));
                internalLog.debug(indexStateInfo("init", initialState, descriptor));
                switch(initialState) {
                    case ONLINE:
                        monitor.initialState(databaseName, descriptor, ONLINE);
                        indexProxy = indexProxyCreator.createOnlineIndexProxy(descriptor);
                        break;
                    case POPULATING:
                        // The database was shut down during population, or a crash has occurred, or some other sad thing.
                        monitor.initialState(databaseName, descriptor, POPULATING);
                        indexProxy = indexProxyCreator.createRecoveringIndexProxy(descriptor);
                        break;
                    case FAILED:
                        monitor.initialState(databaseName, descriptor, FAILED);
                        IndexPopulationFailure failure = failure(provider.getPopulationFailure(descriptor, cursorContext));
                        indexProxy = indexProxyCreator.createFailedIndexProxy(descriptor, failure);
                        break;
                    default:
                        throw new IllegalArgumentException("" + initialState);
                }
                indexMap.putIndexProxy(indexProxy);
            }
            logIndexStateSummary("init", indexStates);
            return indexMap;
        });
    }
    indexStatisticsStore.init();
}

Example 2 with ONLINE

use of org.neo4j.internal.kernel.api.InternalIndexState.ONLINE in project neo4j by neo4j.

the class IndexRecoveryIT method shouldBeAbleToRecoverAndUpdateOnlineIndex.

@Test
void shouldBeAbleToRecoverAndUpdateOnlineIndex() throws Exception {
    // Given
    startDb();
    IndexPopulator populator = mock(IndexPopulator.class);
    when(mockedIndexProvider.getPopulator(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(), any(), any(TokenNameLookup.class))).thenReturn(populator);
    when(populator.sample(any(CursorContext.class))).thenReturn(new IndexSample());
    IndexAccessor mockedAccessor = mock(IndexAccessor.class);
    when(mockedAccessor.newUpdater(any(IndexUpdateMode.class), any(CursorContext.class))).thenReturn(SwallowingIndexUpdater.INSTANCE);
    when(mockedIndexProvider.getOnlineAccessor(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(TokenNameLookup.class))).thenReturn(mockedAccessor);
    createIndexAndAwaitPopulation(myLabel);
    // rotate logs
    rotateLogsAndCheckPoint();
    // make updates
    Set<IndexEntryUpdate<?>> expectedUpdates = createSomeBananas(myLabel);
    // And Given
    killDb();
    when(mockedIndexProvider.getInitialState(any(IndexDescriptor.class), any(CursorContext.class))).thenReturn(ONLINE);
    GatheringIndexWriter writer = new GatheringIndexWriter();
    when(mockedIndexProvider.getOnlineAccessor(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(TokenNameLookup.class))).thenReturn(writer);
    // When
    startDb();
    // Then
    try (Transaction transaction = db.beginTx()) {
        assertThat(transaction.schema().getIndexes(myLabel)).hasSize(1);
        assertThat(transaction.schema().getIndexes(myLabel)).extracting(i -> transaction.schema().getIndexState(i)).containsOnly(Schema.IndexState.ONLINE);
    }
    verify(mockedIndexProvider).getPopulator(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(), any(), any(TokenNameLookup.class));
    // once when we create the index, and once when we restart the db
    int onlineAccessorInvocationCount = 3;
    verify(mockedIndexProvider, times(onlineAccessorInvocationCount)).getOnlineAccessor(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(TokenNameLookup.class));
    assertEquals(expectedUpdates, writer.batchedUpdates);
}

Example 3 with ONLINE

use of org.neo4j.internal.kernel.api.InternalIndexState.ONLINE in project neo4j by neo4j.

the class IndexingService method start.

// Recovery semantics: This is to be called after init, and after the database has run recovery.
@Override
public void start() throws Exception {
    state = State.STARTING;
    // Recovery will not do refresh (update read views) while applying recovered transactions and instead
    // do it at one point after recovery... i.e. here
    indexMapRef.indexMapSnapshot().forEachIndexProxy(indexProxyOperation("refresh", IndexProxy::refresh));
    final MutableLongObjectMap<IndexDescriptor> rebuildingDescriptors = new LongObjectHashMap<>();
    indexMapRef.modify(indexMap -> {
        Map<InternalIndexState, List<IndexLogRecord>> indexStates = new EnumMap<>(InternalIndexState.class);
        Map<IndexProviderDescriptor, List<IndexLogRecord>> indexProviders = new HashMap<>();
        // Find all indexes that are not already online, do not require rebuilding, and create them
        indexMap.forEachIndexProxy((indexId, proxy) -> {
            InternalIndexState state = proxy.getState();
            IndexDescriptor descriptor = proxy.getDescriptor();
            IndexProviderDescriptor providerDescriptor = descriptor.getIndexProvider();
            IndexLogRecord indexLogRecord = new IndexLogRecord(descriptor);
            indexStates.computeIfAbsent(state, internalIndexState -> new ArrayList<>()).add(indexLogRecord);
            indexProviders.computeIfAbsent(providerDescriptor, indexProviderDescriptor -> new ArrayList<>()).add(indexLogRecord);
            internalLog.debug(indexStateInfo("start", state, descriptor));
            switch(state) {
                case ONLINE:
                case FAILED:
                    proxy.start();
                    break;
                case POPULATING:
                    // Remember for rebuilding right below in this method
                    rebuildingDescriptors.put(indexId, descriptor);
                    break;
                default:
                    throw new IllegalStateException("Unknown state: " + state);
            }
        });
        logIndexStateSummary("start", indexStates);
        logIndexProviderSummary(indexProviders);
        dontRebuildIndexesInReadOnlyMode(rebuildingDescriptors);
        // Drop placeholder proxies for indexes that need to be rebuilt
        dropRecoveringIndexes(indexMap, rebuildingDescriptors.keySet());
        // Rebuild indexes by recreating and repopulating them
        populateIndexesOfAllTypes(rebuildingDescriptors, indexMap);
        return indexMap;
    });
    indexStatisticsStore.start();
    samplingController.recoverIndexSamples();
    samplingController.start();
    // So at this point we've started population of indexes that needs to be rebuilt in the background.
    // Indexes backing uniqueness constraints are normally built within the transaction creating the constraint
    // and so we shouldn't leave such indexes in a populating state after recovery.
    // This is why we now go and wait for those indexes to be fully populated.
    rebuildingDescriptors.forEachKeyValue((indexId, index) -> {
        if (!index.isUnique()) {
            // It's not a uniqueness constraint, so don't wait for it to be rebuilt
            return;
        }
        IndexProxy proxy;
        try {
            proxy = getIndexProxy(index);
        } catch (IndexNotFoundKernelException e) {
            throw new IllegalStateException("What? This index was seen during recovery just now, why isn't it available now?", e);
        }
        if (proxy.getDescriptor().getOwningConstraintId().isEmpty()) {
            // so there's no gain in waiting for this index.
            return;
        }
        monitor.awaitingPopulationOfRecoveredIndex(index);
        awaitOnlineAfterRecovery(proxy);
    });
    state = State.RUNNING;
}