Examples with IndexEntryUpdate org.neo4j.storageengine.api.IndexEntryUpdate used on opensource projects

Example 16 with IndexEntryUpdate

use of org.neo4j.storageengine.api.IndexEntryUpdate in project neo4j by neo4j.

the class IndexRecoveryIT method shouldBeAbleToRecoverAndUpdateOnlineIndex.

@Test
void shouldBeAbleToRecoverAndUpdateOnlineIndex() throws Exception {
    // Given
    startDb();
    IndexPopulator populator = mock(IndexPopulator.class);
    when(mockedIndexProvider.getPopulator(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(), any(), any(TokenNameLookup.class))).thenReturn(populator);
    when(populator.sample(any(CursorContext.class))).thenReturn(new IndexSample());
    IndexAccessor mockedAccessor = mock(IndexAccessor.class);
    when(mockedAccessor.newUpdater(any(IndexUpdateMode.class), any(CursorContext.class))).thenReturn(SwallowingIndexUpdater.INSTANCE);
    when(mockedIndexProvider.getOnlineAccessor(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(TokenNameLookup.class))).thenReturn(mockedAccessor);
    createIndexAndAwaitPopulation(myLabel);
    // rotate logs
    rotateLogsAndCheckPoint();
    // make updates
    Set<IndexEntryUpdate<?>> expectedUpdates = createSomeBananas(myLabel);
    // And Given
    killDb();
    when(mockedIndexProvider.getInitialState(any(IndexDescriptor.class), any(CursorContext.class))).thenReturn(ONLINE);
    GatheringIndexWriter writer = new GatheringIndexWriter();
    when(mockedIndexProvider.getOnlineAccessor(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(TokenNameLookup.class))).thenReturn(writer);
    // When
    startDb();
    // Then
    try (Transaction transaction = db.beginTx()) {
        assertThat(transaction.schema().getIndexes(myLabel)).hasSize(1);
        assertThat(transaction.schema().getIndexes(myLabel)).extracting(i -> transaction.schema().getIndexState(i)).containsOnly(Schema.IndexState.ONLINE);
    }
    verify(mockedIndexProvider).getPopulator(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(), any(), any(TokenNameLookup.class));
    // once when we create the index, and once when we restart the db
    int onlineAccessorInvocationCount = 3;
    verify(mockedIndexProvider, times(onlineAccessorInvocationCount)).getOnlineAccessor(any(IndexDescriptor.class), any(IndexSamplingConfig.class), any(TokenNameLookup.class));
    assertEquals(expectedUpdates, writer.batchedUpdates);
}

Example 17 with IndexEntryUpdate

use of org.neo4j.storageengine.api.IndexEntryUpdate in project neo4j by neo4j.

the class BlockBasedIndexPopulatorTest method shouldBuildNonUniqueSampleAsPartOfScanCompleted.

@Test
void shouldBuildNonUniqueSampleAsPartOfScanCompleted() throws IndexEntryConflictException, IOException {
    // given
    ThreadSafePeakMemoryTracker memoryTracker = new ThreadSafePeakMemoryTracker();
    ByteBufferFactory bufferFactory = new ByteBufferFactory(UnsafeDirectByteBufferAllocator::new, 100);
    BlockBasedIndexPopulator<GenericKey, NativeIndexValue> populator = instantiatePopulator(NO_MONITOR, bufferFactory, memoryTracker);
    Collection<IndexEntryUpdate<?>> populationUpdates = batchOfUpdates();
    populator.add(populationUpdates, NULL);
    // when
    populator.scanCompleted(nullInstance, populationWorkScheduler, NULL);
    // Also a couple of updates afterwards
    int numberOfUpdatesAfterCompleted = 4;
    try (IndexUpdater updater = populator.newPopulatingUpdater(NULL)) {
        for (int i = 0; i < numberOfUpdatesAfterCompleted; i++) {
            updater.process(IndexEntryUpdate.add(10_000 + i, SCHEMA_DESCRIPTOR, intValue(i)));
        }
    }
    populator.close(true, NULL);
    // then
    IndexSample sample = populator.sample(NULL);
    assertEquals(populationUpdates.size(), sample.indexSize());
    assertEquals(populationUpdates.size(), sample.sampleSize());
    assertEquals(populationUpdates.size(), sample.uniqueValues());
    assertEquals(numberOfUpdatesAfterCompleted, sample.updates());
}

Example 18 with IndexEntryUpdate

use of org.neo4j.storageengine.api.IndexEntryUpdate in project neo4j by neo4j.

the class BlockBasedIndexPopulatorTest method shouldDeallocateAllAllocatedMemoryOnDrop.

@Test
void shouldDeallocateAllAllocatedMemoryOnDrop() throws IndexEntryConflictException, IOException {
    // given
    ThreadSafePeakMemoryTracker memoryTracker = new ThreadSafePeakMemoryTracker();
    ByteBufferFactory bufferFactory = new ByteBufferFactory(UnsafeDirectByteBufferAllocator::new, 100);
    BlockBasedIndexPopulator<GenericKey, NativeIndexValue> populator = instantiatePopulator(NO_MONITOR, bufferFactory, memoryTracker);
    boolean closed = false;
    try {
        // when
        Collection<IndexEntryUpdate<?>> updates = batchOfUpdates();
        populator.add(updates, NULL);
        int nextId = updates.size();
        externalUpdates(populator, nextId, nextId + 10);
        nextId = nextId + 10;
        long memoryBeforeScanCompleted = memoryTracker.usedNativeMemory();
        populator.scanCompleted(nullInstance, populationWorkScheduler, NULL);
        externalUpdates(populator, nextId, nextId + 10);
        // then
        assertTrue(memoryTracker.peakMemoryUsage() > memoryBeforeScanCompleted, "expected some memory to have been temporarily allocated in scanCompleted");
        populator.drop();
        closed = true;
        bufferFactory.close();
        assertEquals(0, memoryTracker.usedNativeMemory());
    } finally {
        if (!closed) {
            populator.close(true, NULL);
        }
    }
}

Example 19 with IndexEntryUpdate

use of org.neo4j.storageengine.api.IndexEntryUpdate in project neo4j by neo4j.

the class GenericAccessorPointsTest method mustHandlePointArraysWithinSameTile.

/**
 * This test verify that we correctly handle unique point arrays where every point in every array belong to the same tile on the space filling curve.
 * We verify this by asserting that we always get exactly one hit on an exact match and that the value is what we expect.
 */
@Test
void mustHandlePointArraysWithinSameTile() throws IndexEntryConflictException, IndexNotApplicableKernelException {
    // given
    // Many random points that all are close enough to each other to belong to the same tile on the space filling curve.
    int nbrOfValues = 10000;
    PointValue origin = Values.pointValue(WGS84, 0.0, 0.0);
    Long derivedValueForCenterPoint = curve.derivedValueFor(origin.coordinate());
    double[] centerPoint = curve.centerPointFor(derivedValueForCenterPoint);
    double xWidthMultiplier = curve.getTileWidth(0, curve.getMaxLevel()) / 2;
    double yWidthMultiplier = curve.getTileWidth(1, curve.getMaxLevel()) / 2;
    List<Value> pointArrays = new ArrayList<>();
    List<IndexEntryUpdate<?>> updates = new ArrayList<>();
    for (int i = 0; i < nbrOfValues; i++) {
        int arrayLength = random.nextInt(5) + 1;
        PointValue[] pointValues = new PointValue[arrayLength];
        for (int j = 0; j < arrayLength; j++) {
            double x = (random.nextDouble() * 2 - 1) * xWidthMultiplier;
            double y = (random.nextDouble() * 2 - 1) * yWidthMultiplier;
            PointValue value = Values.pointValue(WGS84, centerPoint[0] + x, centerPoint[1] + y);
            assertDerivedValue(derivedValueForCenterPoint, value);
            pointValues[j] = value;
        }
        PointArray array = Values.pointArray(pointValues);
        pointArrays.add(array);
        updates.add(IndexEntryUpdate.add(i, descriptor, array));
    }
    processAll(updates);
    // then
    exactMatchOnAllValues(pointArrays);
}

Example 20 with IndexEntryUpdate

use of org.neo4j.storageengine.api.IndexEntryUpdate in project neo4j by neo4j.

the class GenericAccessorPointsTest method mustHandlePointsWithinSameTile.

/**
 * This test verify that we correctly handle unique points that all belong to the same tile on the space filling curve.
 * All points share at least one dimension coordinate with another point to exercise minimal splitter.
 * We verify this by asserting that we always get exactly one hit on an exact match and that the value is what we expect.
 */
@Test
void mustHandlePointsWithinSameTile() throws IndexEntryConflictException, IndexNotApplicableKernelException {
    // given
    // Many random points that all are close enough to each other to belong to the same tile on the space filling curve.
    int nbrOfValues = 10000;
    PointValue origin = Values.pointValue(WGS84, 0.0, 0.0);
    Long derivedValueForCenterPoint = curve.derivedValueFor(origin.coordinate());
    double[] centerPoint = curve.centerPointFor(derivedValueForCenterPoint);
    double xWidthMultiplier = curve.getTileWidth(0, curve.getMaxLevel()) / 2;
    double yWidthMultiplier = curve.getTileWidth(1, curve.getMaxLevel()) / 2;
    List<Value> pointValues = new ArrayList<>();
    List<IndexEntryUpdate<?>> updates = new ArrayList<>();
    long nodeId = 1;
    for (int i = 0; i < nbrOfValues / 4; i++) {
        double x1 = (random.nextDouble() * 2 - 1) * xWidthMultiplier;
        double x2 = (random.nextDouble() * 2 - 1) * xWidthMultiplier;
        double y1 = (random.nextDouble() * 2 - 1) * yWidthMultiplier;
        double y2 = (random.nextDouble() * 2 - 1) * yWidthMultiplier;
        PointValue value11 = Values.pointValue(WGS84, centerPoint[0] + x1, centerPoint[1] + y1);
        PointValue value12 = Values.pointValue(WGS84, centerPoint[0] + x1, centerPoint[1] + y2);
        PointValue value21 = Values.pointValue(WGS84, centerPoint[0] + x2, centerPoint[1] + y1);
        PointValue value22 = Values.pointValue(WGS84, centerPoint[0] + x2, centerPoint[1] + y2);
        assertDerivedValue(derivedValueForCenterPoint, value11, value12, value21, value22);
        nodeId = addPointsToLists(pointValues, updates, nodeId, value11, value12, value21, value22);
    }
    processAll(updates);
    // then
    exactMatchOnAllValues(pointValues);
}