Examples with NeoStores org.neo4j.kernel.impl.store.NeoStores used on opensource projects

Example 46 with NeoStores

use of org.neo4j.kernel.impl.store.NeoStores in project neo4j by neo4j.

the class DumpCountsStoreTest method createNeoStores.

private NeoStores createNeoStores() {
    NeoStores neoStores = mock(NeoStores.class);
    LabelTokenStore labelTokenStore = mock(LabelTokenStore.class);
    RelationshipTypeTokenStore typeTokenStore = mock(RelationshipTypeTokenStore.class);
    PropertyKeyTokenStore propertyKeyTokenStore = mock(PropertyKeyTokenStore.class);
    when(labelTokenStore.getTokens(Integer.MAX_VALUE)).thenReturn(getLabelTokens());
    when(typeTokenStore.getTokens(Integer.MAX_VALUE)).thenReturn(getTypeTokes());
    when(propertyKeyTokenStore.getTokens(Integer.MAX_VALUE)).thenReturn(getPropertyTokens());
    when(neoStores.getLabelTokenStore()).thenReturn(labelTokenStore);
    when(neoStores.getRelationshipTypeTokenStore()).thenReturn(typeTokenStore);
    when(neoStores.getPropertyKeyTokenStore()).thenReturn(propertyKeyTokenStore);
    return neoStores;
}

Example 47 with NeoStores

use of org.neo4j.kernel.impl.store.NeoStores in project neo4j by neo4j.

the class StoreMigrator method migrateWithBatchImporter.

private void migrateWithBatchImporter(File storeDir, File migrationDir, long lastTxId, long lastTxChecksum, long lastTxLogVersion, long lastTxLogByteOffset, MigrationProgressMonitor.Section progressMonitor, RecordFormats oldFormat, RecordFormats newFormat) throws IOException {
    prepareBatchImportMigration(storeDir, migrationDir, oldFormat, newFormat);
    boolean requiresDynamicStoreMigration = !newFormat.dynamic().equals(oldFormat.dynamic());
    boolean requiresPropertyMigration = !newFormat.property().equals(oldFormat.property()) || requiresDynamicStoreMigration;
    File badFile = new File(storeDir, Configuration.BAD_FILE_NAME);
    try (NeoStores legacyStore = instantiateLegacyStore(oldFormat, storeDir);
        RecordCursors nodeInputCursors = new RecordCursors(legacyStore);
        RecordCursors relationshipInputCursors = new RecordCursors(legacyStore);
        OutputStream badOutput = new BufferedOutputStream(new FileOutputStream(badFile, false))) {
        Configuration importConfig = new Configuration.Overridden(config);
        AdditionalInitialIds additionalInitialIds = readAdditionalIds(lastTxId, lastTxChecksum, lastTxLogVersion, lastTxLogByteOffset);
        // We have to make sure to keep the token ids if we're migrating properties/labels
        BatchImporter importer = new ParallelBatchImporter(migrationDir.getAbsoluteFile(), fileSystem, pageCache, importConfig, logService, withDynamicProcessorAssignment(migrationBatchImporterMonitor(legacyStore, progressMonitor, importConfig), importConfig), additionalInitialIds, config, newFormat);
        InputIterable<InputNode> nodes = legacyNodesAsInput(legacyStore, requiresPropertyMigration, nodeInputCursors);
        InputIterable<InputRelationship> relationships = legacyRelationshipsAsInput(legacyStore, requiresPropertyMigration, relationshipInputCursors);
        importer.doImport(Inputs.input(nodes, relationships, IdMappers.actual(), IdGenerators.fromInput(), Collectors.badCollector(badOutput, 0)));
        // During migration the batch importer doesn't necessarily writes all entities, depending on
        // which stores needs migration. Node, relationship, relationship group stores are always written
        // anyways and cannot be avoided with the importer, but delete the store files that weren't written
        // (left empty) so that we don't overwrite those in the real store directory later.
        Collection<StoreFile> storesToDeleteFromMigratedDirectory = new ArrayList<>();
        storesToDeleteFromMigratedDirectory.add(StoreFile.NEO_STORE);
        if (!requiresPropertyMigration) {
            // We didn't migrate properties, so the property stores in the migrated store are just empty/bogus
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.PROPERTY_STORE, StoreFile.PROPERTY_STRING_STORE, StoreFile.PROPERTY_ARRAY_STORE));
        }
        if (!requiresDynamicStoreMigration) {
            // We didn't migrate labels (dynamic node labels) or any other dynamic store
            storesToDeleteFromMigratedDirectory.addAll(asList(StoreFile.NODE_LABEL_STORE, StoreFile.LABEL_TOKEN_STORE, StoreFile.LABEL_TOKEN_NAMES_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_STORE, StoreFile.RELATIONSHIP_TYPE_TOKEN_NAMES_STORE, StoreFile.PROPERTY_KEY_TOKEN_STORE, StoreFile.PROPERTY_KEY_TOKEN_NAMES_STORE, StoreFile.SCHEMA_STORE));
        }
        StoreFile.fileOperation(DELETE, fileSystem, migrationDir, null, storesToDeleteFromMigratedDirectory, true, null, StoreFileType.values());
        // When migrating on a block device there might be some files only accessible via the page cache.
        try {
            Predicate<FileHandle> fileHandlePredicate = fileHandle -> storesToDeleteFromMigratedDirectory.stream().anyMatch(storeFile -> storeFile.fileName(StoreFileType.STORE).equals(fileHandle.getFile().getName()));
            pageCache.streamFilesRecursive(migrationDir).filter(fileHandlePredicate).forEach(FileHandle.HANDLE_DELETE);
        } catch (NoSuchFileException e) {
        // This means that we had no files only present in the page cache, this is fine.
        }
    }
}

Example 48 with NeoStores

use of org.neo4j.kernel.impl.store.NeoStores in project neo4j by neo4j.

the class PropertyDeduplicator method deduplicateProperties.

public void deduplicateProperties() throws IOException {
    StoreFactory factory = new StoreFactory(workingDir, pageCache, fileSystem, NullLogProvider.getInstance());
    try (NeoStores neoStores = factory.openNeoStores(StoreType.PROPERTY, StoreType.NODE, StoreType.SCHEMA)) {
        PropertyStore propertyStore = neoStores.getPropertyStore();
        NodeStore nodeStore = neoStores.getNodeStore();
        SchemaStore schemaStore = neoStores.getSchemaStore();
        PrimitiveLongObjectMap<List<DuplicateCluster>> duplicateClusters = collectConflictingProperties(propertyStore);
        resolveConflicts(duplicateClusters, propertyStore, nodeStore, schemaStore, neoStores.getStoreDir());
    }
}

Example 49 with NeoStores

use of org.neo4j.kernel.impl.store.NeoStores in project neo4j by neo4j.

the class RecoveryIT method idGeneratorsRebuildAfterRecovery.

@Test
public void idGeneratorsRebuildAfterRecovery() throws IOException {
    GraphDatabaseService database = startDatabase(directory.graphDbDir());
    int numberOfNodes = 10;
    try (Transaction transaction = database.beginTx()) {
        for (int nodeIndex = 0; nodeIndex < numberOfNodes; nodeIndex++) {
            database.createNode();
        }
        transaction.success();
    }
    // copying only transaction log simulate non clean shutdown db that should be able to recover just from logs
    File restoreDbStoreDir = copyTransactionLogs();
    GraphDatabaseService recoveredDatabase = startDatabase(restoreDbStoreDir);
    NeoStores neoStore = ((GraphDatabaseAPI) recoveredDatabase).getDependencyResolver().resolveDependency(RecordStorageEngine.class).testAccessNeoStores();
    assertEquals(numberOfNodes, neoStore.getNodeStore().getHighId());
    // Make sure id generator has been rebuilt so this doesn't throw null pointer exception
    assertTrue(neoStore.getNodeStore().nextId() > 0);
    database.shutdown();
    recoveredDatabase.shutdown();
}

Example 50 with NeoStores

use of org.neo4j.kernel.impl.store.NeoStores in project neo4j by neo4j.

the class DetectAllRelationshipInconsistenciesIT method shouldDetectSabotagedRelationshipWhereEverItIs.

@Test
public void shouldDetectSabotagedRelationshipWhereEverItIs() throws Exception {
    // GIVEN a database which lots of relationships
    GraphDatabaseAPI db = getGraphDatabaseAPI();
    Sabotage sabotage;
    try {
        Node[] nodes = new Node[1_000];
        Relationship[] relationships = new Relationship[10_000];
        try (Transaction tx = db.beginTx()) {
            for (int i = 0; i < nodes.length; i++) {
                nodes[i] = db.createNode(label("Foo"));
            }
            for (int i = 0; i < 10_000; i++) {
                relationships[i] = random.among(nodes).createRelationshipTo(random.among(nodes), MyRelTypes.TEST);
            }
            tx.success();
        }
        // WHEN sabotaging a random relationship
        DependencyResolver resolver = db.getDependencyResolver();
        PageCache pageCache = resolver.resolveDependency(PageCache.class);
        StoreFactory storeFactory = newStoreFactory(pageCache);
        try (NeoStores neoStores = storeFactory.openNeoStores(false, StoreType.RELATIONSHIP)) {
            RelationshipStore relationshipStore = neoStores.getRelationshipStore();
            Relationship sabotagedRelationships = random.among(relationships);
            sabotage = sabotage(relationshipStore, sabotagedRelationships.getId());
        }
    } finally {
        db.shutdown();
    }
    // THEN the checker should find it, where ever it is in the store
    db = getGraphDatabaseAPI();
    try {
        DependencyResolver resolver = db.getDependencyResolver();
        PageCache pageCache = resolver.resolveDependency(PageCache.class);
        StoreFactory storeFactory = newStoreFactory(pageCache);
        try (NeoStores neoStores = storeFactory.openAllNeoStores()) {
            StoreAccess storeAccess = new StoreAccess(neoStores).initialize();
            DirectStoreAccess directStoreAccess = new DirectStoreAccess(storeAccess, db.getDependencyResolver().resolveDependency(LabelScanStore.class), db.getDependencyResolver().resolveDependency(SchemaIndexProvider.class));
            int threads = random.intBetween(2, 10);
            FullCheck checker = new FullCheck(getTuningConfiguration(), ProgressMonitorFactory.NONE, Statistics.NONE, threads);
            AssertableLogProvider logProvider = new AssertableLogProvider(true);
            ConsistencySummaryStatistics summary = checker.execute(directStoreAccess, logProvider.getLog(FullCheck.class));
            int relationshipInconsistencies = summary.getInconsistencyCountForRecordType(RecordType.RELATIONSHIP);
            assertTrue("Couldn't detect sabotaged relationship " + sabotage, relationshipInconsistencies > 0);
            logProvider.assertContainsLogCallContaining(sabotage.after.toString());
        }
    } finally {
        db.shutdown();
    }
}