use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.
the class HumanUnderstandableExecutionMonitorIT method shouldReportProgressOfNodeImport.
@Test
void shouldReportProgressOfNodeImport() throws Exception {
// given
CapturingMonitor progress = new CapturingMonitor();
HumanUnderstandableExecutionMonitor monitor = new HumanUnderstandableExecutionMonitor(progress);
IdType idType = IdType.INTEGER;
Input input = new DataGeneratorInput(NODE_COUNT, RELATIONSHIP_COUNT, idType, random.seed(), 0, bareboneNodeHeader(idType, new Extractors(';')), bareboneRelationshipHeader(idType, new Extractors(';')), 1, 1, 0, 0);
Configuration configuration = new Configuration.Overridden(Configuration.DEFAULT) {
@Override
public long pageCacheMemory() {
return mebiBytes(8);
}
};
// when
try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
new ParallelBatchImporter(databaseLayout, fileSystem, NULL, configuration, NullLogService.getInstance(), monitor, EMPTY, defaults(), LATEST_RECORD_FORMATS, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE).doImport(input);
// then
progress.assertAllProgressReachedEnd();
}
}
use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.
the class ForkedProcessorStepTest method shouldBeAbleToProgressUnderStressfulProcessorChanges.
private static void shouldBeAbleToProgressUnderStressfulProcessorChanges(int orderingGuarantees) throws Exception {
// given
int batches = 100;
int processors = Runtime.getRuntime().availableProcessors() * 10;
Configuration config = new Configuration.Overridden(Configuration.DEFAULT) {
@Override
public int maxNumberOfProcessors() {
return processors;
}
};
Stage stage = new StressStage(config, orderingGuarantees, batches);
StageExecution execution = stage.execute();
List<Step<?>> steps = asList(execution.steps());
steps.get(1).processors(processors / 3);
// when
ThreadLocalRandom random = ThreadLocalRandom.current();
while (execution.stillExecuting()) {
steps.get(2).processors(random.nextInt(-2, 5));
Thread.sleep(1);
}
execution.assertHealthy();
// then
assertEquals(batches, steps.get(steps.size() - 1).stats().stat(Keys.done_batches).asLong());
closeAllSilently(steps);
}
use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.
the class DynamicProcessorAssignerTest method shouldMoveProcessorFromOverlyAssignedStep.
@Test
void shouldMoveProcessorFromOverlyAssignedStep() {
// GIVEN
Configuration config = config(10, 5);
// available processors = 2 is enough because it will see the fast step as only using 20% of a processor
// and it rounds down. So there's room for assigning one more.
DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
ControlledStep<?> slowStep = stepWithStats("slow", 0, Keys.avg_processing_time, 6L, Keys.done_batches, 10L).setProcessors(2);
ControlledStep<?> fastStep = stepWithStats("fast", 0, Keys.avg_processing_time, 2L, Keys.done_batches, 10L).setProcessors(3);
StageExecution execution = executionOf(config, slowStep, fastStep);
assigner.start(execution);
// WHEN checking
assigner.check(execution);
// THEN one processor should have been moved from the fast step to the slower step
assertEquals(2, fastStep.processors(0));
assertEquals(3, slowStep.processors(0));
}
use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.
the class DynamicProcessorAssignerTest method shouldMoveProcessorFromNotOnlyFastestStep.
@Test
void shouldMoveProcessorFromNotOnlyFastestStep() {
// The point is that not only the fastest step is subject to have processors removed,
// it's the relationship between all pairs of steps.
// GIVEN
Configuration config = config(10, 5);
DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
Step<?> wayFastest = stepWithStats("wayFastest", 0, Keys.avg_processing_time, 50L, Keys.done_batches, 20L);
Step<?> fast = stepWithStats("fast", 0, Keys.avg_processing_time, 100L, Keys.done_batches, 20L).setProcessors(3);
Step<?> slow = stepWithStats("slow", 2, Keys.avg_processing_time, 220L, Keys.done_batches, 20L);
StageExecution execution = executionOf(config, slow, wayFastest, fast);
assigner.start(execution);
// WHEN
assigner.check(execution);
// THEN
assertEquals(2, fast.processors(0));
assertEquals(2, slow.processors(0));
}
use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.
the class DynamicProcessorAssignerTest method shouldHandleZeroAverage.
@Test
void shouldHandleZeroAverage() {
// GIVEN
Configuration config = config(10, 5);
DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
ControlledStep<?> aStep = stepWithStats("slow", 0, Keys.avg_processing_time, 0L, Keys.done_batches, 0L);
ControlledStep<?> anotherStep = stepWithStats("fast", 0, Keys.avg_processing_time, 0L, Keys.done_batches, 0L);
StageExecution execution = executionOf(config, aStep, anotherStep);
assigner.start(execution);
// WHEN
assigner.check(execution);
// THEN
assertEquals(1, aStep.processors(0));
assertEquals(1, anotherStep.processors(0));
}
Aggregations