Examples with SchemaStorage org.neo4j.internal.recordstorage.SchemaStorage used on opensource projects

Example 1 with SchemaStorage

use of org.neo4j.internal.recordstorage.SchemaStorage in project neo4j by neo4j.

the class RecordStorageMigratorIT method mustMigrateSchemaStoreToNewFormat.

@ParameterizedTest
@MethodSource("versions")
void mustMigrateSchemaStoreToNewFormat(String version, LogPosition expectedLogPosition, Function<TransactionId, Boolean> txIdComparator) throws Exception {
    // Given we have an old store full of random schema rules.
    Path prepare = testDirectory.directory("prepare");
    var fs = testDirectory.getFileSystem();
    MigrationTestUtils.prepareSampleLegacyDatabase(version, fs, databaseLayout.databaseDirectory(), prepare);
    // and a state of the migration saying that it has done the actual migration
    LogService logService = NullLogService.getInstance();
    // Uses this special scan-on-open IGF because when the new IndexedIdGenerator was introduced this test would break
    // when trying to open an older store, before doing migration.
    IdGeneratorFactory igf = new ScanOnOpenOverwritingIdGeneratorFactory(fs, databaseLayout.getDatabaseName());
    LogProvider logProvider = logService.getInternalLogProvider();
    // Prepare all the tokens we'll need.
    StoreFactory legacyStoreFactory = new StoreFactory(databaseLayout, CONFIG, igf, pageCache, fs, StandardV3_4.RECORD_FORMATS, logProvider, PageCacheTracer.NULL, writable(), immutable.empty());
    NeoStores stores = legacyStoreFactory.openNeoStores(false, StoreType.LABEL_TOKEN, StoreType.LABEL_TOKEN_NAME, StoreType.RELATIONSHIP_TYPE_TOKEN, StoreType.RELATIONSHIP_TYPE_TOKEN_NAME, StoreType.PROPERTY_KEY_TOKEN, StoreType.PROPERTY_KEY_TOKEN_NAME);
    createTokens(stores.getLabelTokenStore(), MAX_LABEL_ID);
    createTokens(stores.getRelationshipTypeTokenStore(), MAX_RELATIONSHIP_TYPE_ID);
    createTokens(stores.getPropertyKeyTokenStore(), MAX_PROPERTY_KEY_ID);
    stores.close();
    // Prepare the legacy schema store we'll migrate.
    Path storeFile = databaseLayout.schemaStore();
    Path idFile = databaseLayout.idSchemaStore();
    SchemaStore35 schemaStore35 = new SchemaStore35(storeFile, idFile, CONFIG, IdType.SCHEMA, igf, pageCache, logProvider, StandardV3_4.RECORD_FORMATS, writable(), DEFAULT_DATABASE_NAME, immutable.empty());
    schemaStore35.initialise(false, NULL);
    SplittableRandom rng = new SplittableRandom(randomRule.seed());
    LongHashSet indexes = new LongHashSet();
    LongHashSet constraints = new LongHashSet();
    for (int i = 0; i < 10; i++) {
        long id = schemaStore35.nextId(NULL);
        MutableLongSet target = rng.nextInt(3) < 2 ? indexes : constraints;
        target.add(id);
    }
    List<SchemaRule> generatedRules = new ArrayList<>();
    RealIdsRandomSchema randomSchema = new RealIdsRandomSchema(rng, indexes, constraints);
    while (randomSchema.hasMoreIds()) {
        try {
            SchemaRule schemaRule = randomSchema.nextSchemaRule();
            if (schemaRule instanceof ConstraintDescriptor) {
                ConstraintDescriptor constraint = (ConstraintDescriptor) schemaRule;
                if (constraint.isIndexBackedConstraint() && !constraint.asIndexBackedConstraint().hasOwnedIndexId()) {
                    // Filter out constraints that are supposed to own indexes, but don't, because those are illegal to persist.
                    randomSchema.rollback();
                    continue;
                }
            }
            randomSchema.commit();
            generatedRules.add(schemaRule);
            List<DynamicRecord> dynamicRecords = allocateFrom(schemaStore35, schemaRule, NULL);
            for (DynamicRecord dynamicRecord : dynamicRecords) {
                schemaStore35.updateRecord(dynamicRecord, NULL);
            }
        } catch (NoSuchElementException ignore) {
        // We're starting to run low on ids, but just ignore this and loop as along as there are still some left.
        }
    }
    schemaStore35.flush(NULL);
    schemaStore35.close();
    RecordStoreVersionCheck check = getVersionCheck(pageCache, databaseLayout);
    String versionToMigrateFrom = getVersionToMigrateFrom(check);
    MigrationProgressMonitor progressMonitor = SILENT;
    RecordStorageMigrator migrator = new RecordStorageMigrator(fs, pageCache, CONFIG, logService, jobScheduler, PageCacheTracer.NULL, batchImporterFactory, INSTANCE);
    // When we migrate it to the new store format.
    String versionToMigrateTo = getVersionToMigrateTo(check);
    ProgressReporter reporter = progressMonitor.startSection("section");
    migrator.migrate(databaseLayout, migrationLayout, reporter, versionToMigrateFrom, versionToMigrateTo, EMPTY);
    migrator.moveMigratedFiles(migrationLayout, databaseLayout, versionToMigrateFrom, versionToMigrateTo);
    generatedRules.sort(Comparator.comparingLong(SchemaRule::getId));
    // Then the new store should retain an exact representation of the old-format schema rules.
    StoreFactory storeFactory = new StoreFactory(databaseLayout, CONFIG, igf, pageCache, fs, logProvider, PageCacheTracer.NULL, writable());
    try (NeoStores neoStores = storeFactory.openAllNeoStores()) {
        SchemaStore schemaStore = neoStores.getSchemaStore();
        TokenHolders tokenHolders = StoreTokens.readOnlyTokenHolders(neoStores, NULL);
        SchemaStorage storage = new SchemaStorage(schemaStore, tokenHolders, () -> KernelVersion.LATEST);
        List<SchemaRule> migratedRules = new ArrayList<>();
        storage.getAll(NULL).iterator().forEachRemaining(migratedRules::add);
        // Nerf the rule names, since migration may change those around.
        migratedRules = migratedRules.stream().map(r -> r.withName("a")).collect(Collectors.toList());
        generatedRules = generatedRules.stream().map(r -> r.withName("a")).collect(Collectors.toList());
        assertThat(migratedRules).isEqualTo(generatedRules);
    }
}

Example 2 with SchemaStorage

use of org.neo4j.internal.recordstorage.SchemaStorage in project neo4j by neo4j.

the class SchemaChecker method checkSchema.

private void checkSchema(MutableIntObjectMap<MutableIntSet> mandatoryNodeProperties, MutableIntObjectMap<MutableIntSet> mandatoryRelationshipProperties, CursorContext cursorContext) {
    long highId = schemaStore.getHighId();
    try (RecordReader<SchemaRecord> schemaReader = new RecordReader<>(schemaStore, true, cursorContext)) {
        Map<Long, SchemaRecord> indexObligations = new HashMap<>();
        Map<Long, SchemaRecord> constraintObligations = new HashMap<>();
        Map<SchemaRuleKey, SchemaRecord> verifiedRulesWithRecords = new HashMap<>();
        // KernelVersion only controls if there will be an injected NLI on non-upgraded stores. That index is not in the store and will
        // not be found when going through the file in performSchemaCheck anyway so we can use latest KernelVersion here.
        // If an injected NLI exist but is not online that will not be reported, but this is an unlikely corner case that we
        // ignore. The index itself will be checked as long as it is online (it is found by IndexAccessors).
        SchemaStorage schemaStorage = new SchemaStorage(schemaStore, tokenHolders, () -> KernelVersion.LATEST);
        // Build map of obligations and such
        buildObligationsMap(highId, schemaReader, schemaStorage, indexObligations, constraintObligations, verifiedRulesWithRecords, cursorContext);
        // Verify all things, now that we have the complete map of obligations back and forth
        performSchemaCheck(highId, schemaReader, indexObligations, constraintObligations, schemaStorage, mandatoryNodeProperties, mandatoryRelationshipProperties, cursorContext);
    }
}

Example 3 with SchemaStorage

use of org.neo4j.internal.recordstorage.SchemaStorage in project neo4j by neo4j.

the class SchemaStorageIT method initStorage.

@BeforeEach
void initStorage() throws Exception {
    try (Transaction transaction = db.beginTx()) {
        TokenWrite tokenWrite = ((InternalTransaction) transaction).kernelTransaction().tokenWrite();
        tokenWrite.propertyKeyGetOrCreateForName(PROP1);
        tokenWrite.propertyKeyGetOrCreateForName(PROP2);
        tokenWrite.labelGetOrCreateForName(LABEL1);
        tokenWrite.labelGetOrCreateForName(LABEL2);
        tokenWrite.relationshipTypeGetOrCreateForName(TYPE1);
        transaction.commit();
    }
    schemaStore = storageEngine.testAccessNeoStores().getSchemaStore();
    storage = new SchemaStorage(schemaStore, tokenHolders, () -> KernelVersion.LATEST);
}

Example 4 with SchemaStorage

use of org.neo4j.internal.recordstorage.SchemaStorage in project neo4j by neo4j.

the class CheckerTestBase method setUpDb.

@BeforeEach
void setUpDb() throws Exception {
    TestDatabaseManagementServiceBuilder builder = new TestDatabaseManagementServiceBuilder(directory.homePath());
    builder.setFileSystem(directory.getFileSystem());
    dbms = builder.build();
    db = (GraphDatabaseAPI) dbms.database(GraphDatabaseSettings.DEFAULT_DATABASE_NAME);
    // Create our tokens
    Kernel kernel = db.getDependencyResolver().resolveDependency(Kernel.class);
    try (KernelTransaction tx = kernel.beginTransaction(KernelTransaction.Type.EXPLICIT, LoginContext.AUTH_DISABLED)) {
        initialData(tx);
        tx.commit();
    }
    DependencyResolver dependencies = db.getDependencyResolver();
    neoStores = dependencies.resolveDependency(RecordStorageEngine.class).testAccessNeoStores();
    nodeStore = neoStores.getNodeStore();
    relationshipGroupStore = neoStores.getRelationshipGroupStore();
    propertyStore = neoStores.getPropertyStore();
    relationshipStore = neoStores.getRelationshipStore();
    schemaStore = neoStores.getSchemaStore();
    tokenHolders = dependencies.resolveDependency(TokenHolders.class);
    schemaStorage = new SchemaStorage(schemaStore, tokenHolders, neoStores.getMetaDataStore());
    cacheAccess = new DefaultCacheAccess(NumberArrayFactories.HEAP.newDynamicByteArray(10_000, new byte[MAX_BYTES], INSTANCE), Counts.NONE, NUMBER_OF_THREADS);
    cacheAccess.setCacheSlotSizes(DEFAULT_SLOT_SIZES);
    pageCache = dependencies.resolveDependency(PageCache.class);
}

Example 5 with SchemaStorage

use of org.neo4j.internal.recordstorage.SchemaStorage in project neo4j by neo4j.

the class FulltextIndexProviderTest method indexWithUnknownAnalyzerWillBeMarkedAsFailedOnStartup.

@Test
void indexWithUnknownAnalyzerWillBeMarkedAsFailedOnStartup() throws Exception {
    // Create a full-text index.
    long indexId;
    try (KernelTransactionImplementation transaction = getKernelTransaction()) {
        int[] propertyIds = { propIdHa };
        SchemaDescriptor schema = SchemaDescriptor.fulltext(EntityType.NODE, new int[] { labelIdHa }, propertyIds);
        IndexPrototype prototype = IndexPrototype.forSchema(schema).withIndexType(FULLTEXT).withName(NAME);
        SchemaWrite schemaWrite = transaction.schemaWrite();
        IndexDescriptor index = schemaWrite.indexCreate(prototype);
        indexId = index.getId();
        transaction.success();
    }
    // Modify the full-text index such that it has an analyzer configured that does not exist.
    controller.restartDbms(builder -> {
        var cacheTracer = NULL;
        FileSystemAbstraction fs = builder.getFileSystem();
        DatabaseLayout databaseLayout = Neo4jLayout.of(builder.getHomeDirectory()).databaseLayout(DEFAULT_DATABASE_NAME);
        DefaultIdGeneratorFactory idGenFactory = new DefaultIdGeneratorFactory(fs, RecoveryCleanupWorkCollector.ignore(), databaseLayout.getDatabaseName());
        try (JobScheduler scheduler = JobSchedulerFactory.createInitialisedScheduler();
            PageCache pageCache = StandalonePageCacheFactory.createPageCache(fs, scheduler, cacheTracer)) {
            StoreFactory factory = new StoreFactory(databaseLayout, Config.defaults(), idGenFactory, pageCache, fs, NullLogProvider.getInstance(), cacheTracer, writable());
            var cursorContext = CursorContext.NULL;
            try (NeoStores neoStores = factory.openAllNeoStores(false)) {
                TokenHolders tokens = StoreTokens.readOnlyTokenHolders(neoStores, CursorContext.NULL);
                SchemaStore schemaStore = neoStores.getSchemaStore();
                SchemaStorage storage = new SchemaStorage(schemaStore, tokens, () -> KernelVersion.LATEST);
                IndexDescriptor index = (IndexDescriptor) storage.loadSingleSchemaRule(indexId, CursorContext.NULL);
                Map<String, Value> indexConfigMap = new HashMap<>(index.getIndexConfig().asMap());
                for (Map.Entry<String, Value> entry : indexConfigMap.entrySet()) {
                    if (entry.getKey().contains("analyzer")) {
                        // This analyzer does not exist!
                        entry.setValue(Values.stringValue("bla-bla-lyzer"));
                    }
                }
                index = index.withIndexConfig(IndexConfig.with(indexConfigMap));
                storage.writeSchemaRule(index, cursorContext, INSTANCE);
                schemaStore.flush(cursorContext);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
        return builder;
    });
    // Verify that the index comes up in a failed state.
    try (Transaction tx = db.beginTx()) {
        IndexDefinition index = tx.schema().getIndexByName(NAME);
        Schema.IndexState indexState = tx.schema().getIndexState(index);
        assertThat(indexState).isEqualTo(Schema.IndexState.FAILED);
        String indexFailure = tx.schema().getIndexFailure(index);
        assertThat(indexFailure).contains("bla-bla-lyzer");
    }
    // Verify that the failed index can be dropped.
    try (Transaction tx = db.beginTx()) {
        tx.schema().getIndexByName(NAME).drop();
        assertThrows(IllegalArgumentException.class, () -> tx.schema().getIndexByName(NAME));
        tx.commit();
    }
    try (Transaction tx = db.beginTx()) {
        assertThrows(IllegalArgumentException.class, () -> tx.schema().getIndexByName(NAME));
    }
    controller.restartDbms();
    try (Transaction tx = db.beginTx()) {
        assertThrows(IllegalArgumentException.class, () -> tx.schema().getIndexByName(NAME));
    }
}