Examples with Configuration org.neo4j.internal.batchimport.Configuration used on opensource projects

Example 11 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class ReadEntityIdsStepUsingTokenIndexTest method shouldSeeRecentUpdatesRightInFrontOfExternalUpdatesPoint.

@Test
void shouldSeeRecentUpdatesRightInFrontOfExternalUpdatesPoint() throws Exception {
    // given
    long entityCount = 1_000 + random.nextInt(100);
    BitSet expectedEntityIds = new BitSet();
    BitSet seenEntityIds = new BitSet();
    try (var indexAccessor = indexAccessor()) {
        populateTokenIndex(indexAccessor, expectedEntityIds, entityCount);
        Configuration configuration = withBatchSize(DEFAULT, 100);
        Stage stage = new Stage("Test", null, configuration, 0) {

            {
                add(new ReadEntityIdsStep(control(), configuration, cursorContext -> new TokenIndexScanIdIterator(indexAccessor.newTokenReader(), new int[] { TOKEN_ID }, CursorContext.NULL), NULL, new ControlledUpdatesCheck(indexAccessor, expectedEntityIds), new AtomicBoolean(true)));
                add(new CollectEntityIdsStep(control(), configuration, seenEntityIds));
            }
        };
        // when
        stage.execute().awaitCompletion();
        // then
        assertThat(seenEntityIds).isEqualTo(expectedEntityIds);
    }
}

Example 12 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class CoarseBoundedProgressExecutionMonitorTest method progressOnMultipleExecutions.

@ParameterizedTest
@ValueSource(ints = { 1, 10, 123 })
void progressOnMultipleExecutions(int batchSize) {
    Configuration config = config(batchSize);
    ProgressExecutionMonitor progressExecutionMonitor = new ProgressExecutionMonitor(batchSize, config);
    long total = progressExecutionMonitor.total();
    for (int i = 0; i < 4; i++) {
        progressExecutionMonitor.start(execution(0, config));
        progressExecutionMonitor.check(execution(total / 4, config));
    }
    progressExecutionMonitor.done(true, 0, "Completed");
    assertEquals(total, progressExecutionMonitor.getProgress(), "Each item should be completed");
}

Example 13 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class CoarseBoundedProgressExecutionMonitorTest method shouldReportProgressOnSingleExecution.

@ParameterizedTest
@ValueSource(ints = { 1, 10, 123 })
void shouldReportProgressOnSingleExecution(int batchSize) {
    // GIVEN
    Configuration config = config(batchSize);
    ProgressExecutionMonitor progressExecutionMonitor = new ProgressExecutionMonitor(batchSize, config(batchSize));
    // WHEN
    long total = monitorSingleStageExecution(progressExecutionMonitor, config);
    // THEN
    assertEquals(total, progressExecutionMonitor.getProgress());
}

Example 14 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldAssignProcessorsToSlowestStep.

@Test
void shouldAssignProcessorsToSlowestStep() {
    // GIVEN
    Configuration config = config(10, 5);
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
    ControlledStep<?> slowStep = stepWithStats("slow", 0, Keys.avg_processing_time, 10L, Keys.done_batches, 10L);
    ControlledStep<?> fastStep = stepWithStats("fast", 0, Keys.avg_processing_time, 2L, Keys.done_batches, 10L);
    StageExecution execution = executionOf(config, slowStep, fastStep);
    assigner.start(execution);
    // WHEN
    assigner.check(execution);
    // THEN
    assertEquals(4, slowStep.processors(0));
    assertEquals(1, fastStep.processors(0));
}

Example 15 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldNotMoveProcessorFromFastStepSoThatItBecomesBottleneck.

@Test
void shouldNotMoveProcessorFromFastStepSoThatItBecomesBottleneck() {
    // GIVEN
    Configuration config = config(10, 4);
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
    ControlledStep<?> slowStep = stepWithStats("slow", 1, Keys.avg_processing_time, 10L, Keys.done_batches, 10L);
    ControlledStep<?> fastStep = stepWithStats("fast", 0, Keys.avg_processing_time, 7L, Keys.done_batches, 10L).setProcessors(3);
    StageExecution execution = executionOf(config, slowStep, fastStep);
    assigner.start(execution);
    // WHEN checking
    assigner.check(execution);
    // THEN
    assertEquals(3, fastStep.processors(0));
    assertEquals(1, slowStep.processors(0));
}