Examples with Configuration org.neo4j.internal.batchimport.Configuration used on opensource projects

Example 6 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class DynamicProcessorAssignerTest method shouldNotMoveProcessorFromOverlyAssignedStepIfRemainingPermits.

@Test
void shouldNotMoveProcessorFromOverlyAssignedStepIfRemainingPermits() {
    // GIVEN
    Configuration config = config(10, 10);
    DynamicProcessorAssigner assigner = new DynamicProcessorAssigner(config);
    Step<?> wayFastest = stepWithStats("wayFastest", 0, Keys.avg_processing_time, 50L, Keys.done_batches, 20L);
    Step<?> fast = stepWithStats("fast", 0, Keys.avg_processing_time, 100L, Keys.done_batches, 20L).setProcessors(3);
    Step<?> slow = stepWithStats("slow", 2, Keys.avg_processing_time, 220L, Keys.done_batches, 20L);
    StageExecution execution = executionOf(config, slow, wayFastest, fast);
    assigner.start(execution);
    // WHEN
    assigner.check(execution);
    // THEN no processor have been removed from the fast step
    assertEquals(3, fast.processors(0));
    // although there were some to assign so slow step got one
    assertEquals(2, slow.processors(0));
}

Example 7 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class StageTest method shouldReceiveBatchesInOrder.

@Test
void shouldReceiveBatchesInOrder() {
    // GIVEN
    Configuration config = new Configuration.Overridden(DEFAULT) {

        @Override
        public int batchSize() {
            return 10;
        }
    };
    Stage stage = new Stage("Test stage", null, config, Step.ORDER_SEND_DOWNSTREAM);
    long batches = 1000;
    final long items = batches * config.batchSize();
    stage.add(new PullingProducerStep(stage.control(), config) {

        private final Object theObject = new Object();

        private long i;

        @Override
        protected Object nextBatchOrNull(long ticket, int batchSize) {
            if (i >= items) {
                return null;
            }
            Object[] batch = new Object[batchSize];
            Arrays.fill(batch, theObject);
            i += batchSize;
            return batch;
        }

        @Override
        protected long position() {
            return 0;
        }
    });
    for (int i = 0; i < 3; i++) {
        stage.add(new ReceiveOrderAssertingStep(stage.control(), "Step" + i, config, i, false));
    }
    stage.add(new ReceiveOrderAssertingStep(stage.control(), "Final step", config, 0, true));
    // WHEN
    StageExecution execution = stage.execute();
    for (Step<?> step : execution.steps()) {
        // we start off with two in each step
        step.processors(1);
    }
    new ExecutionSupervisor(INVISIBLE).supervise(execution);
    // THEN
    for (Step<?> step : execution.steps()) {
        assertEquals(batches, step.stats().stat(Keys.done_batches).asLong(), "For " + step);
    }
    stage.close();
}

Example 8 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class StageTest method shouldCloseOnPanic.

@Test
void shouldCloseOnPanic() {
    // given
    // a producer, a processor, a forked processor and a final step
    Configuration configuration = DEFAULT;
    TrackingPanicMonitor panicMonitor = new TrackingPanicMonitor();
    Stage stage = new Stage("test close on panic", null, configuration, random.nextBoolean() ? Step.ORDER_SEND_DOWNSTREAM : 0, ProcessorScheduler.SPAWN_THREAD, panicMonitor) {

        {
            // Producer
            add(new PullingProducerStep(control(), configuration) {

                private volatile long ticket;

                private final ChaosMonkey chaosMonkey = new ChaosMonkey();

                @Override
                protected Object nextBatchOrNull(long ticket, int batchSize) {
                    chaosMonkey.makeChaos();
                    this.ticket = ticket;
                    return new int[batchSize];
                }

                @Override
                protected long position() {
                    return ticket;
                }
            });
            // Processor
            add(new ProcessorStep<>(control(), "processor", configuration, 2, NULL) {

                private final ChaosMonkey chaosMonkey = new ChaosMonkey();

                @Override
                protected void process(Object batch, BatchSender sender, CursorContext cursorContext) {
                    chaosMonkey.makeChaos();
                    sender.send(batch);
                }
            });
            // Forked processor
            add(new ForkedProcessorStep<>(control(), "forked processor", configuration) {

                private final ChaosMonkey chaosMonkey = new ChaosMonkey();

                @Override
                protected void forkedProcess(int id, int processors, Object batch) {
                    chaosMonkey.makeChaos();
                }
            });
            // Final consumer
            add(new ProcessorStep<>(control(), "consumer", configuration, 1, NULL) {

                private final ChaosMonkey chaosMonkey = new ChaosMonkey();

                @Override
                protected void process(Object batch, BatchSender sender, CursorContext cursorContext) throws Throwable {
                    chaosMonkey.makeChaos();
                // don't pass the batch further, i.e. end of the line
                }
            });
        }
    };
    // when/then
    assertThrows(RuntimeException.class, () -> superviseDynamicExecution(stage));
    assertTrue(panicMonitor.hasReceivedPanic());
    assertTrue(panicMonitor.getReceivedPanic().getMessage().contains("Chaos monkey"));
}

Example 9 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class CountsComputer method populateCountStore.

private void populateCountStore(CountsAccessor.Updater countsUpdater) {
    try (NodeLabelsCache cache = new NodeLabelsCache(numberArrayFactory, nodes.getHighId(), highLabelId, memoryTracker)) {
        Configuration configuration = Configuration.defaultConfiguration(databaseLayout.databaseDirectory());
        // Count nodes
        superviseDynamicExecution(new NodeCountsStage(configuration, cache, nodes, highLabelId, countsUpdater, progressMonitor, pageCacheTracer));
        // Count relationships
        superviseDynamicExecution(new RelationshipCountsStage(configuration, cache, relationships, highLabelId, highRelationshipTypeId, countsUpdater, numberArrayFactory, progressMonitor, pageCacheTracer, memoryTracker));
    }
}

Example 10 with Configuration

use of org.neo4j.internal.batchimport.Configuration in project neo4j by neo4j.

the class PropertyAwareEntityStoreScan method run.

@Override
public void run(ExternalUpdatesCheck externalUpdatesCheck) {
    try {
        continueScanning.set(true);
        Configuration config = Configuration.DEFAULT;
        stage = new StoreScanStage<>(dbConfig, config, this::getEntityIdIterator, externalUpdatesCheck, continueScanning, storageReader, entityTokenIdFilter, propertyKeyIdFilter, propertyScanConsumer, tokenScanConsumer, cursorBehaviour, lockFunction, parallelWrite, scheduler, cacheTracer, memoryTracker);
        superviseDynamicExecution(INVISIBLE, stage);
        stage.reportTo(phaseTracker);
    } finally {
        closeAllUnchecked(storageReader);
    }
}