Search in sources :

Example 1 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class HumanUnderstandableExecutionMonitorIT method shouldReportProgressOfNodeImport.

@Test
void shouldReportProgressOfNodeImport() throws Exception {
    // given
    CapturingMonitor progress = new CapturingMonitor();
    HumanUnderstandableExecutionMonitor monitor = new HumanUnderstandableExecutionMonitor(progress);
    IdType idType = IdType.INTEGER;
    Input input = new DataGeneratorInput(NODE_COUNT, RELATIONSHIP_COUNT, idType, random.seed(), 0, bareboneNodeHeader(idType, new Extractors(';')), bareboneRelationshipHeader(idType, new Extractors(';')), 1, 1, 0, 0);
    Configuration configuration = new Configuration.Overridden(Configuration.DEFAULT) {

        @Override
        public long pageCacheMemory() {
            return mebiBytes(8);
        }
    };
    // when
    try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        new ParallelBatchImporter(databaseLayout, fileSystem, NULL, configuration, NullLogService.getInstance(), monitor, EMPTY, defaults(), LATEST_RECORD_FORMATS, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE).doImport(input);
        // then
        progress.assertAllProgressReachedEnd();
    }
}
Also used : JobScheduler(org.neo4j.scheduler.JobScheduler) ThreadPoolJobScheduler(org.neo4j.test.scheduler.ThreadPoolJobScheduler) ParallelBatchImporter(org.neo4j.internal.batchimport.ParallelBatchImporter) DataGeneratorInput(org.neo4j.internal.batchimport.input.DataGeneratorInput) Input(org.neo4j.internal.batchimport.input.Input) Extractors(org.neo4j.csv.reader.Extractors) Configuration(org.neo4j.internal.batchimport.Configuration) ThreadPoolJobScheduler(org.neo4j.test.scheduler.ThreadPoolJobScheduler) DataGeneratorInput(org.neo4j.internal.batchimport.input.DataGeneratorInput) IdType(org.neo4j.internal.batchimport.input.IdType) Test(org.junit.jupiter.api.Test)

Example 2 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class ForkedProcessorStepTest method shouldBeAbleToProgressUnderStressfulProcessorChanges.

private static void shouldBeAbleToProgressUnderStressfulProcessorChanges(int orderingGuarantees) throws Exception {
    // given
    int batches = 100;
    int processors = Runtime.getRuntime().availableProcessors() * 10;
    Configuration config = new Configuration.Overridden(Configuration.DEFAULT) {

        @Override
        public int maxNumberOfProcessors() {
            return processors;
        }
    };
    Stage stage = new StressStage(config, orderingGuarantees, batches);
    StageExecution execution = stage.execute();
    List<Step<?>> steps = asList(execution.steps());
    steps.get(1).processors(processors / 3);
    // when
    ThreadLocalRandom random = ThreadLocalRandom.current();
    while (execution.stillExecuting()) {
        steps.get(2).processors(random.nextInt(-2, 5));
        Thread.sleep(1);
    }
    execution.assertHealthy();
    // then
    assertEquals(batches, steps.get(steps.size() - 1).stats().stat(Keys.done_batches).asLong());
    closeAllSilently(steps);
}
Also used : Configuration(org.neo4j.internal.batchimport.Configuration) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom)

Example 3 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldMoveProcessorFromOverlyAssignedStep.

@Test
void shouldMoveProcessorFromOverlyAssignedStep() {
    // GIVEN
    Configuration config = config(10, 5);
    // available processors = 2 is enough because it will see the fast step as only using 20% of a processor
    // and it rounds down. So there's room for assigning one more.
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
    ControlledStep<?> slowStep = stepWithStats("slow", 0, Keys.avg_processing_time, 6L, Keys.done_batches, 10L).setProcessors(2);
    ControlledStep<?> fastStep = stepWithStats("fast", 0, Keys.avg_processing_time, 2L, Keys.done_batches, 10L).setProcessors(3);
    StageExecution execution = executionOf(config, slowStep, fastStep);
    assigner.start(execution);
    // WHEN checking
    assigner.check(execution);
    // THEN one processor should have been moved from the fast step to the slower step
    assertEquals(2, fastStep.processors(0));
    assertEquals(3, slowStep.processors(0));
}
Also used : Configuration(org.neo4j.internal.batchimport.Configuration) Test(org.junit.jupiter.api.Test)

Example 4 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldMoveProcessorFromNotOnlyFastestStep.

@Test
void shouldMoveProcessorFromNotOnlyFastestStep() {
    // The point is that not only the fastest step is subject to have processors removed,
    // it's the relationship between all pairs of steps.
    // GIVEN
    Configuration config = config(10, 5);
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
    Step<?> wayFastest = stepWithStats("wayFastest", 0, Keys.avg_processing_time, 50L, Keys.done_batches, 20L);
    Step<?> fast = stepWithStats("fast", 0, Keys.avg_processing_time, 100L, Keys.done_batches, 20L).setProcessors(3);
    Step<?> slow = stepWithStats("slow", 2, Keys.avg_processing_time, 220L, Keys.done_batches, 20L);
    StageExecution execution = executionOf(config, slow, wayFastest, fast);
    assigner.start(execution);
    // WHEN
    assigner.check(execution);
    // THEN
    assertEquals(2, fast.processors(0));
    assertEquals(2, slow.processors(0));
}
Also used : Configuration(org.neo4j.internal.batchimport.Configuration) Test(org.junit.jupiter.api.Test)

Example 5 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldHandleZeroAverage.

@Test
void shouldHandleZeroAverage() {
    // GIVEN
    Configuration config = config(10, 5);
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
    ControlledStep<?> aStep = stepWithStats("slow", 0, Keys.avg_processing_time, 0L, Keys.done_batches, 0L);
    ControlledStep<?> anotherStep = stepWithStats("fast", 0, Keys.avg_processing_time, 0L, Keys.done_batches, 0L);
    StageExecution execution = executionOf(config, aStep, anotherStep);
    assigner.start(execution);
    // WHEN
    assigner.check(execution);
    // THEN
    assertEquals(1, aStep.processors(0));
    assertEquals(1, anotherStep.processors(0));
}
Also used : Configuration(org.neo4j.internal.batchimport.Configuration) Test(org.junit.jupiter.api.Test)

Aggregations

Configuration (org.neo4j.internal.batchimport.Configuration)17 Test (org.junit.jupiter.api.Test)11 CursorContext (org.neo4j.io.pagecache.context.CursorContext)3 CountDownLatch (java.util.concurrent.CountDownLatch)2 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)2 ValueSource (org.junit.jupiter.params.provider.ValueSource)2 BitSet (java.util.BitSet)1 ThreadLocalRandom (java.util.concurrent.ThreadLocalRandom)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1 Assertions.assertThat (org.assertj.core.api.Assertions.assertThat)1 ExtendWith (org.junit.jupiter.api.extension.ExtendWith)1 EMPTY_LONG_ARRAY (org.neo4j.collection.PrimitiveLongCollections.EMPTY_LONG_ARRAY)1 EntityType (org.neo4j.common.EntityType)1 Config (org.neo4j.configuration.Config)1 DEFAULT_DATABASE_NAME (org.neo4j.configuration.GraphDatabaseSettings.DEFAULT_DATABASE_NAME)1 GraphDatabaseSettings.neo4j_home (org.neo4j.configuration.GraphDatabaseSettings.neo4j_home)1 GraphDatabaseSettings.preallocate_logical_logs (org.neo4j.configuration.GraphDatabaseSettings.preallocate_logical_logs)1 Extractors (org.neo4j.csv.reader.Extractors)1 RecoveryCleanupWorkCollector.immediate (org.neo4j.index.internal.gbptree.RecoveryCleanupWorkCollector.immediate)1