Examples with JobScheduler org.neo4j.scheduler.JobScheduler used on opensource projects

Example 6 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class ImportPanicIT method shouldExitAndThrowExceptionOnPanic.

/**
 * There was this problem where some steps and in particular parallel CSV input parsing that
 * paniced would hang the import entirely.
 */
@Test
void shouldExitAndThrowExceptionOnPanic() throws Exception {
    try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        BatchImporter importer = new ParallelBatchImporter(databaseLayout, testDirectory.getFileSystem(), PageCacheTracer.NULL, Configuration.DEFAULT, NullLogService.getInstance(), ExecutionMonitor.INVISIBLE, AdditionalInitialIds.EMPTY, Config.defaults(), StandardV3_4.RECORD_FORMATS, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE);
        Iterable<DataFactory> nodeData = DataFactories.datas(DataFactories.data(InputEntityDecorators.NO_DECORATOR, fileAsCharReadable(nodeCsvFileWithBrokenEntries())));
        Input brokenCsvInput = new CsvInput(nodeData, DataFactories.defaultFormatNodeFileHeader(), DataFactories.datas(), DataFactories.defaultFormatRelationshipFileHeader(), IdType.ACTUAL, csvConfigurationWithLowBufferSize(), CsvInput.NO_MONITOR, INSTANCE);
        var e = assertThrows(InputException.class, () -> importer.doImport(brokenCsvInput));
        assertTrue(e.getCause() instanceof DataAfterQuoteException);
    }
}

Example 7 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class CsvInputEstimateCalculationIT method shouldCalculateCorrectEstimates.

@Test
void shouldCalculateCorrectEstimates() throws Exception {
    // given a couple of input files of various layouts
    Input input = generateData();
    RecordFormats format = LATEST_RECORD_FORMATS;
    Input.Estimates estimates = input.calculateEstimates(new PropertyValueRecordSizeCalculator(format.property().getRecordSize(NO_STORE_HEADER), GraphDatabaseInternalSettings.string_block_size.defaultValue(), 0, GraphDatabaseInternalSettings.array_block_size.defaultValue(), 0));
    // when
    Config config = Config.defaults();
    FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
    try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        new ParallelBatchImporter(databaseLayout, fs, PageCacheTracer.NULL, PBI_CONFIG, NullLogService.getInstance(), INVISIBLE, EMPTY, config, format, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE).doImport(input);
        // then compare estimates with actual disk sizes
        SingleFilePageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(fs);
        try (PageCache pageCache = new MuninnPageCache(swapperFactory, jobScheduler, MuninnPageCache.config(1000));
            NeoStores stores = new StoreFactory(databaseLayout, config, new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName()), pageCache, fs, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable()).openAllNeoStores()) {
            assertRoughlyEqual(estimates.numberOfNodes(), stores.getNodeStore().getNumberOfIdsInUse());
            assertRoughlyEqual(estimates.numberOfRelationships(), stores.getRelationshipStore().getNumberOfIdsInUse());
            assertRoughlyEqual(estimates.numberOfNodeProperties() + estimates.numberOfRelationshipProperties(), calculateNumberOfProperties(stores));
        }
        long measuredPropertyStorage = propertyStorageSize();
        long estimatedPropertyStorage = estimates.sizeOfNodeProperties() + estimates.sizeOfRelationshipProperties();
        assertThat(estimatedPropertyStorage).as("Estimated property storage size of %s must be within 10%% of the measured size of %s.", bytesToString(estimatedPropertyStorage), bytesToString(measuredPropertyStorage)).isCloseTo(measuredPropertyStorage, withPercentage(10.0));
    }
}

Example 8 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class MultiIndexPopulationConcurrentUpdatesIT method launchCustomIndexPopulation.

private void launchCustomIndexPopulation(GraphDatabaseSettings.SchemaIndex schemaIndex, Map<String, Integer> labelNameIdMap, int propertyId, Runnable customAction) throws Throwable {
    RecordStorageEngine storageEngine = getStorageEngine();
    try (Transaction transaction = db.beginTx()) {
        Config config = Config.defaults();
        KernelTransaction ktx = ((InternalTransaction) transaction).kernelTransaction();
        JobScheduler scheduler = getJobScheduler();
        NullLogProvider nullLogProvider = NullLogProvider.getInstance();
        IndexStoreViewFactory indexStoreViewFactory = mock(IndexStoreViewFactory.class);
        when(indexStoreViewFactory.createTokenIndexStoreView(any())).thenAnswer(invocation -> dynamicIndexStoreViewWrapper(customAction, storageEngine::newReader, invocation.getArgument(0), config, scheduler));
        IndexProviderMap providerMap = getIndexProviderMap();
        indexService = IndexingServiceFactory.createIndexingService(config, scheduler, providerMap, indexStoreViewFactory, ktx.tokenRead(), initialSchemaRulesLoader(storageEngine), nullLogProvider, nullLogProvider, IndexingService.NO_MONITOR, getSchemaState(), mock(IndexStatisticsStore.class), PageCacheTracer.NULL, INSTANCE, "", writable());
        indexService.start();
        rules = createIndexRules(schemaIndex, labelNameIdMap, propertyId);
        schemaCache = new SchemaCache(new StandardConstraintSemantics(), providerMap);
        schemaCache.load(iterable(rules));
        indexService.createIndexes(AUTH_DISABLED, rules);
        transaction.commit();
    }
}

Example 9 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class ConfigurableStandalonePageCacheFactoryTest method mustAutomaticallyStartEvictionThread.

@Test
void mustAutomaticallyStartEvictionThread() throws Exception {
    try (FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
        JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        Path file = testDirectory.homePath().resolve("a").normalize();
        fs.write(file).close();
        try (PageCache cache = ConfigurableStandalonePageCacheFactory.createPageCache(fs, jobScheduler, PageCacheTracer.NULL);
            PagedFile pf = cache.map(file, 4096, DEFAULT_DATABASE_NAME);
            PageCursor cursor = pf.io(0, PagedFile.PF_SHARED_WRITE_LOCK, CursorContext.NULL)) {
            // If the eviction thread has not been started, then this test will block forever.
            for (int i = 0; i < 10_000; i++) {
                assertTrue(cursor.next());
                cursor.putInt(42);
            }
        }
    }
}

Example 10 with JobScheduler

use of org.neo4j.scheduler.JobScheduler in project neo4j by neo4j.

the class GBPTreePartialCreateFuzzIT method main.

static void main(String[] args) throws Exception {
    // Just start and immediately close. The process spawning this subprocess will kill it in the middle of all this
    Path file = Path.of(args[0]);
    try (FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
        JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        SingleFilePageSwapperFactory swapper = new SingleFilePageSwapperFactory(fs);
        try (PageCache pageCache = new MuninnPageCache(swapper, jobScheduler, config(10))) {
            fs.deleteFile(file);
            new GBPTreeBuilder<>(pageCache, file, longLayout().build()).build().close();
        }
    }
}