Examples with StoreFactory org.neo4j.kernel.impl.store.StoreFactory used on opensource projects

Example 16 with StoreFactory

use of org.neo4j.kernel.impl.store.StoreFactory in project neo4j by neo4j.

the class NodeLabelsFieldTest method startUp.

@BeforeEach
void startUp() {
    Config config = Config.defaults(GraphDatabaseInternalSettings.label_block_size, 60);
    StoreFactory storeFactory = new StoreFactory(databaseLayout, config, new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName()), pageCache, fs, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable());
    neoStores = storeFactory.openAllNeoStores(true);
    nodeStore = neoStores.getNodeStore();
}

Example 17 with StoreFactory

use of org.neo4j.kernel.impl.store.StoreFactory in project neo4j by neo4j.

the class CsvInputEstimateCalculationIT method shouldCalculateCorrectEstimates.

@Test
void shouldCalculateCorrectEstimates() throws Exception {
    // given a couple of input files of various layouts
    Input input = generateData();
    RecordFormats format = LATEST_RECORD_FORMATS;
    Input.Estimates estimates = input.calculateEstimates(new PropertyValueRecordSizeCalculator(format.property().getRecordSize(NO_STORE_HEADER), GraphDatabaseInternalSettings.string_block_size.defaultValue(), 0, GraphDatabaseInternalSettings.array_block_size.defaultValue(), 0));
    // when
    Config config = Config.defaults();
    FileSystemAbstraction fs = new DefaultFileSystemAbstraction();
    try (JobScheduler jobScheduler = new ThreadPoolJobScheduler()) {
        new ParallelBatchImporter(databaseLayout, fs, PageCacheTracer.NULL, PBI_CONFIG, NullLogService.getInstance(), INVISIBLE, EMPTY, config, format, ImportLogic.NO_MONITOR, jobScheduler, Collector.EMPTY, LogFilesInitializer.NULL, IndexImporterFactory.EMPTY, EmptyMemoryTracker.INSTANCE).doImport(input);
        // then compare estimates with actual disk sizes
        SingleFilePageSwapperFactory swapperFactory = new SingleFilePageSwapperFactory(fs);
        try (PageCache pageCache = new MuninnPageCache(swapperFactory, jobScheduler, MuninnPageCache.config(1000));
            NeoStores stores = new StoreFactory(databaseLayout, config, new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName()), pageCache, fs, NullLogProvider.getInstance(), PageCacheTracer.NULL, writable()).openAllNeoStores()) {
            assertRoughlyEqual(estimates.numberOfNodes(), stores.getNodeStore().getNumberOfIdsInUse());
            assertRoughlyEqual(estimates.numberOfRelationships(), stores.getRelationshipStore().getNumberOfIdsInUse());
            assertRoughlyEqual(estimates.numberOfNodeProperties() + estimates.numberOfRelationshipProperties(), calculateNumberOfProperties(stores));
        }
        long measuredPropertyStorage = propertyStorageSize();
        long estimatedPropertyStorage = estimates.sizeOfNodeProperties() + estimates.sizeOfRelationshipProperties();
        assertThat(estimatedPropertyStorage).as("Estimated property storage size of %s must be within 10%% of the measured size of %s.", bytesToString(estimatedPropertyStorage), bytesToString(measuredPropertyStorage)).isCloseTo(measuredPropertyStorage, withPercentage(10.0));
    }
}

Example 18 with StoreFactory

use of org.neo4j.kernel.impl.store.StoreFactory in project neo4j by neo4j.

the class RecordStorageMigratorIT method mustMigrateSchemaStoreToNewFormat.

@ParameterizedTest
@MethodSource("versions")
void mustMigrateSchemaStoreToNewFormat(String version, LogPosition expectedLogPosition, Function<TransactionId, Boolean> txIdComparator) throws Exception {
    // Given we have an old store full of random schema rules.
    Path prepare = testDirectory.directory("prepare");
    var fs = testDirectory.getFileSystem();
    MigrationTestUtils.prepareSampleLegacyDatabase(version, fs, databaseLayout.databaseDirectory(), prepare);
    // and a state of the migration saying that it has done the actual migration
    LogService logService = NullLogService.getInstance();
    // Uses this special scan-on-open IGF because when the new IndexedIdGenerator was introduced this test would break
    // when trying to open an older store, before doing migration.
    IdGeneratorFactory igf = new ScanOnOpenOverwritingIdGeneratorFactory(fs, databaseLayout.getDatabaseName());
    LogProvider logProvider = logService.getInternalLogProvider();
    // Prepare all the tokens we'll need.
    StoreFactory legacyStoreFactory = new StoreFactory(databaseLayout, CONFIG, igf, pageCache, fs, StandardV3_4.RECORD_FORMATS, logProvider, PageCacheTracer.NULL, writable(), immutable.empty());
    NeoStores stores = legacyStoreFactory.openNeoStores(false, StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME, StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME);
    createTokens(stores.getLabelTokenStore(), MAX_LABEL_ID);
    createTokens(stores.getRelationshipTypeTokenStore(), MAX_RELATIONSHIP_TYPE_ID);
    createTokens(stores.getPropertyKeyTokenStore(), MAX_PROPERTY_KEY_ID);
    stores.close();
    // Prepare the legacy schema store we'll migrate.
    Path storeFile = databaseLayout.schemaStore();
    Path idFile = databaseLayout.idSchemaStore();
    SchemaStore35 schemaStore35 = new SchemaStore35(storeFile, idFile, CONFIG, IdType.SCHEMA, igf, pageCache, logProvider, StandardV3_4.RECORD_FORMATS, writable(), DEFAULT_DATABASE_NAME, immutable.empty());
    schemaStore35.initialise(false, NULL);
    SplittableRandom rng = new SplittableRandom(randomRule.seed());
    LongHashSet indexes = new LongHashSet();
    LongHashSet constraints = new LongHashSet();
    for (int i = 0; i < 10; i++) {
        long id = schemaStore35.nextId(NULL);
        MutableLongSet target = rng.nextInt(3) < 2 ? indexes : constraints;
        target.add(id);
    }
    List<SchemaRule> generatedRules = new ArrayList<>();
    RealIdsRandomSchema randomSchema = new RealIdsRandomSchema(rng, indexes, constraints);
    while (randomSchema.hasMoreIds()) {
        try {
            SchemaRule schemaRule = randomSchema.nextSchemaRule();
            if (schemaRule instanceof ConstraintDescriptor) {
                ConstraintDescriptor constraint = (ConstraintDescriptor) schemaRule;
                if (constraint.isIndexBackedConstraint() && !constraint.asIndexBackedConstraint().hasOwnedIndexId()) {
                    // Filter out constraints that are supposed to own indexes, but don't, because those are illegal to persist.
                    randomSchema.rollback();
                    continue;
                }
            }
            randomSchema.commit();
            generatedRules.add(schemaRule);
            List<DynamicRecord> dynamicRecords = allocateFrom(schemaStore35, schemaRule, NULL);
            for (DynamicRecord dynamicRecord : dynamicRecords) {
                schemaStore35.updateRecord(dynamicRecord, NULL);
            }
        } catch (NoSuchElementException ignore) {
        // We're starting to run low on ids, but just ignore this and loop as along as there are still some left.
        }
    }
    schemaStore35.flush(NULL);
    schemaStore35.close();
    RecordStoreVersionCheck check = getVersionCheck(pageCache, databaseLayout);
    String versionToMigrateFrom = getVersionToMigrateFrom(check);
    MigrationProgressMonitor progressMonitor = SILENT;
    RecordStorageMigrator migrator = new RecordStorageMigrator(fs, pageCache, CONFIG, logService, jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
    // When we migrate it to the new store format.
    String versionToMigrateTo = getVersionToMigrateTo(check);
    ProgressReporter reporter = progressMonitor.startSection("section");
    migrator.migrate(databaseLayout, migrationLayout, reporter, versionToMigrateFrom, versionToMigrateTo, EMPTY);
    migrator.moveMigratedFiles(migrationLayout, databaseLayout, versionToMigrateFrom, versionToMigrateTo);
    generatedRules.sort(Comparator.comparingLong(SchemaRule::getId));
    // Then the new store should retain an exact representation of the old-format schema rules.
    StoreFactory storeFactory = new StoreFactory(databaseLayout, CONFIG, igf, pageCache, fs, logProvider, PageCacheTracer.NULL, writable());
    try (NeoStores neoStores = storeFactory.openAllNeoStores()) {
        SchemaStore schemaStore = neoStores.getSchemaStore();
        TokenHolders tokenHolders = StoreTokens.readOnlyTokenHolders(neoStores, NULL);
        SchemaStorage storage = new SchemaStorage(schemaStore, tokenHolders, () -> KernelVersion.LATEST);
        List<SchemaRule> migratedRules = new ArrayList<>();
        storage.getAll(NULL).iterator().forEachRemaining(migratedRules::add);
        // Nerf the rule names, since migration may change those around.
        migratedRules = migratedRules.stream().map(r -> r.withName("a")).collect(Collectors.toList());
        generatedRules = generatedRules.stream().map(r -> r.withName("a")).collect(Collectors.toList());
        assertThat(migratedRules).isEqualTo(generatedRules);
    }
}

Example 19 with StoreFactory

use of org.neo4j.kernel.impl.store.StoreFactory in project neo4j by neo4j.

the class RecordStorageMigratorIT method shouldKeepCheckpointLogVersionIfPresentBeforeMigration.

@ParameterizedTest
@MethodSource("versions")
void shouldKeepCheckpointLogVersionIfPresentBeforeMigration(String version, LogPosition expectedLogPosition, Function<TransactionId, Boolean> txIdComparator) throws Exception {
    // given
    Path prepare = testDirectory.directory("prepare");
    var fs = testDirectory.getFileSystem();
    MigrationTestUtils.prepareSampleLegacyDatabase(version, fs, databaseLayout.databaseDirectory(), prepare);
    RecordStoreVersionCheck check = getVersionCheck(pageCache, databaseLayout);
    String versionToMigrateFrom = getVersionToMigrateFrom(check);
    String versionToMigrateTo = getVersionToMigrateTo(check);
    // explicitly set a checkpoint log version into the meta data store
    long checkpointLogVersion = 4;
    MetaDataStore.setRecord(pageCache, databaseLayout.metadataStore(), CHECKPOINT_LOG_VERSION, checkpointLogVersion, databaseLayout.getDatabaseName(), NULL);
    // when
    RecordStorageMigrator migrator = new RecordStorageMigrator(fs, pageCache, CONFIG, NullLogService.getInstance(), jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
    // when
    migrator.migrate(databaseLayout, migrationLayout, SILENT.startSection("section"), versionToMigrateFrom, versionToMigrateTo, EMPTY);
    migrator.moveMigratedFiles(migrationLayout, databaseLayout, versionToMigrateFrom, versionToMigrateTo);
    // then
    try (NeoStores neoStores = new StoreFactory(databaseLayout, Config.defaults(), new DefaultIdGeneratorFactory(fs, immediate(), databaseLayout.getDatabaseName()), pageCache, fs, nullLogProvider(), PageCacheTracer.NULL, writable()).openNeoStores(StoreType.META_DATA)) {
        neoStores.start(NULL);
        assertThat(neoStores.getMetaDataStore().getCheckpointLogVersion()).isEqualTo(checkpointLogVersion);
    }
}

Example 20 with StoreFactory

use of org.neo4j.kernel.impl.store.StoreFactory in project neo4j by neo4j.

the class RecordStorageMigratorIT method shouldBeAbleToMigrateWithoutErrors.

@ParameterizedTest
@MethodSource("versions")
void shouldBeAbleToMigrateWithoutErrors(String version, LogPosition expectedLogPosition, Function<TransactionId, Boolean> txIdComparator) throws Exception {
    // GIVEN a legacy database
    Path prepare = testDirectory.directory("prepare");
    var fs = testDirectory.getFileSystem();
    MigrationTestUtils.prepareSampleLegacyDatabase(version, fs, databaseLayout.databaseDirectory(), prepare);
    AssertableLogProvider logProvider = new AssertableLogProvider(true);
    LogService logService = new SimpleLogService(logProvider, logProvider);
    RecordStoreVersionCheck check = getVersionCheck(pageCache, databaseLayout);
    String versionToMigrateFrom = getVersionToMigrateFrom(check);
    RecordStorageMigrator migrator = new RecordStorageMigrator(fs, pageCache, CONFIG, logService, jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
    // WHEN migrating
    migrator.migrate(databaseLayout, migrationLayout, progressMonitor.startSection("section"), versionToMigrateFrom, getVersionToMigrateTo(check), EMPTY);
    migrator.moveMigratedFiles(migrationLayout, databaseLayout, versionToMigrateFrom, getVersionToMigrateTo(check));
    // THEN starting the new store should be successful
    assertThat(testDirectory.getFileSystem().fileExists(databaseLayout.relationshipGroupDegreesStore())).isTrue();
    GBPTreeRelationshipGroupDegreesStore.DegreesRebuilder noRebuildAssertion = new GBPTreeRelationshipGroupDegreesStore.DegreesRebuilder() {

        @Override
        public void rebuild(RelationshipGroupDegreesStore.Updater updater, CursorContext cursorContext, MemoryTracker memoryTracker) {
            throw new IllegalStateException("Rebuild should not be required");
        }

        @Override
        public long lastCommittedTxId() {
            try {
                return new RecordStorageEngineFactory().readOnlyTransactionIdStore(fs, databaseLayout, pageCache, NULL).getLastCommittedTransactionId();
            } catch (IOException e) {
                throw new UncheckedIOException(e);
            }
        }
    };
    try (GBPTreeRelationshipGroupDegreesStore groupDegreesStore = new GBPTreeRelationshipGroupDegreesStore(pageCache, databaseLayout.relationshipGroupDegreesStore(), testDirectory.getFileSystem(), immediate(), noRebuildAssertion, writable(), PageCacheTracer.NULL, GBPTreeGenericCountsStore.NO_MONITOR, databaseLayout.getDatabaseName(), counts_store_max_cached_entries.defaultValue())) {
        // The rebuild would happen here in start and will throw exception (above) if invoked
        groupDegreesStore.start(NULL, INSTANCE);
        // The store keeps track of committed transactions.
        // It is essential that it starts with the transaction
        // that is the last committed one at the upgrade time.
        assertEquals(TX_ID, groupDegreesStore.txId());
    }
    StoreFactory storeFactory = new StoreFactory(databaseLayout, CONFIG, new ScanOnOpenOverwritingIdGeneratorFactory(fs, databaseLayout.getDatabaseName()), pageCache, fs, logService.getInternalLogProvider(), PageCacheTracer.NULL, writable());
    storeFactory.openAllNeoStores().close();
    assertThat(logProvider).forLevel(ERROR).doesNotHaveAnyLogs();
}