Examples with StorageEngine org.neo4j.storageengine.api.StorageEngine used on opensource projects

Example 6 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class TransactionRepresentationCommitProcessTest method shouldFailWithProperMessageOnAppendException.

@Test
public void shouldFailWithProperMessageOnAppendException() throws Exception {
    // GIVEN
    TransactionAppender appender = mock(TransactionAppender.class);
    IOException rootCause = new IOException("Mock exception");
    doThrow(new IOException(rootCause)).when(appender).append(any(TransactionToApply.class), any(LogAppendEvent.class));
    StorageEngine storageEngine = mock(StorageEngine.class);
    TransactionCommitProcess commitProcess = new TransactionRepresentationCommitProcess(appender, storageEngine);
    // WHEN
    try {
        commitProcess.commit(mockedTransaction(), commitEvent, INTERNAL);
        fail("Should have failed, something is wrong with the mocking in this test");
    } catch (TransactionFailureException e) {
        assertThat(e.getMessage(), containsString("Could not append transaction representation to log"));
        assertTrue(contains(e, rootCause.getMessage(), rootCause.getClass()));
    }
}

Example 7 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class NeoStoreFileListingTest method shouldCloseIndexAndLabelScanSnapshots.

@Test
public void shouldCloseIndexAndLabelScanSnapshots() throws Exception {
    // Given
    LabelScanStore labelScanStore = mock(LabelScanStore.class);
    IndexingService indexingService = mock(IndexingService.class);
    LegacyIndexProviderLookup legacyIndexes = mock(LegacyIndexProviderLookup.class);
    when(legacyIndexes.all()).thenReturn(Collections.emptyList());
    File storeDir = mock(File.class);
    filesInStoreDirAre(storeDir, STANDARD_STORE_DIR_FILES, STANDARD_STORE_DIR_DIRECTORIES);
    StorageEngine storageEngine = mock(StorageEngine.class);
    NeoStoreFileListing fileListing = new NeoStoreFileListing(storeDir, labelScanStore, indexingService, legacyIndexes, storageEngine);
    ResourceIterator<File> scanSnapshot = scanStoreFilesAre(labelScanStore, new String[] { "blah/scan.store", "scan.more" });
    ResourceIterator<File> indexSnapshot = indexFilesAre(indexingService, new String[] { "schema/index/my.index" });
    ResourceIterator<StoreFileMetadata> result = fileListing.listStoreFiles(false);
    // When
    result.close();
    // Then
    verify(scanSnapshot).close();
    verify(indexSnapshot).close();
}

Example 8 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class RecoveryTest method shouldRecoverExistingData.

@Test
public void shouldRecoverExistingData() throws Exception {
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(directory.directory(), "log", fileSystemRule.get());
    File file = logFiles.getLogFileForVersion(logVersion);
    writeSomeData(file, new Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException>() {

        @Override
        public boolean visit(Pair<LogEntryWriter, Consumer<LogPositionMarker>> pair) throws IOException {
            LogEntryWriter writer = pair.first();
            Consumer<LogPositionMarker> consumer = pair.other();
            LogPositionMarker marker = new LogPositionMarker();
            // last committed tx
            consumer.accept(marker);
            LogPosition lastCommittedTxPosition = marker.newPosition();
            writer.writeStartEntry(0, 1, 2L, 3L, new byte[0]);
            lastCommittedTxStartEntry = new LogEntryStart(0, 1, 2L, 3L, new byte[0], lastCommittedTxPosition);
            writer.writeCommitEntry(4L, 5L);
            lastCommittedTxCommitEntry = new OnePhaseCommit(4L, 5L);
            // check point pointing to the previously committed transaction
            writer.writeCheckPointEntry(lastCommittedTxPosition);
            expectedCheckPointEntry = new CheckPoint(lastCommittedTxPosition);
            // tx committed after checkpoint
            consumer.accept(marker);
            writer.writeStartEntry(0, 1, 6L, 4L, new byte[0]);
            expectedStartEntry = new LogEntryStart(0, 1, 6L, 4L, new byte[0], marker.newPosition());
            writer.writeCommitEntry(5L, 7L);
            expectedCommitEntry = new OnePhaseCommit(5L, 7L);
            return true;
        }
    });
    LifeSupport life = new LifeSupport();
    Recovery.Monitor monitor = mock(Recovery.Monitor.class);
    final AtomicBoolean recoveryRequired = new AtomicBoolean();
    try {
        StorageEngine storageEngine = mock(StorageEngine.class);
        final LogEntryReader<ReadableClosablePositionAwareChannel> reader = new VersionAwareLogEntryReader<>();
        LatestCheckPointFinder finder = new LatestCheckPointFinder(logFiles, fileSystemRule.get(), reader);
        LogHeaderCache logHeaderCache = new LogHeaderCache(10);
        TransactionMetadataCache metadataCache = new TransactionMetadataCache(100);
        LogFile logFile = life.add(new PhysicalLogFile(fileSystemRule.get(), logFiles, 50, () -> transactionIdStore.getLastCommittedTransactionId(), logVersionRepository, mock(PhysicalLogFile.Monitor.class), logHeaderCache));
        LogicalTransactionStore txStore = new PhysicalLogicalTransactionStore(logFile, metadataCache, reader);
        life.add(new Recovery(new DefaultRecoverySPI(storageEngine, logFiles, fileSystemRule.get(), logVersionRepository, finder, transactionIdStore, txStore, NO_MONITOR) {

            private int nr = 0;

            @Override
            public Visitor<CommittedTransactionRepresentation, Exception> startRecovery() {
                recoveryRequired.set(true);
                final Visitor<CommittedTransactionRepresentation, Exception> actual = super.startRecovery();
                return new Visitor<CommittedTransactionRepresentation, Exception>() {

                    @Override
                    public boolean visit(CommittedTransactionRepresentation tx) throws Exception {
                        actual.visit(tx);
                        switch(nr++) {
                            case 0:
                                assertEquals(lastCommittedTxStartEntry, tx.getStartEntry());
                                assertEquals(lastCommittedTxCommitEntry, tx.getCommitEntry());
                                break;
                            case 1:
                                assertEquals(expectedStartEntry, tx.getStartEntry());
                                assertEquals(expectedCommitEntry, tx.getCommitEntry());
                                break;
                            default:
                                fail("Too many recovered transactions");
                        }
                        return false;
                    }
                };
            }
        }, monitor));
        life.start();
        InOrder order = inOrder(monitor);
        order.verify(monitor, times(1)).recoveryRequired(any(LogPosition.class));
        order.verify(monitor, times(1)).recoveryCompleted(2);
        assertTrue(recoveryRequired.get());
    } finally {
        life.shutdown();
    }
}

Example 9 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class NeoStoreDataSource method start.

@Override
public void start() throws IOException {
    dependencies = new Dependencies();
    life = new LifeSupport();
    schemaIndexProvider = dependencyResolver.resolveDependency(SchemaIndexProvider.class, HighestSelectionStrategy.getInstance());
    labelScanStoreProvider = dependencyResolver.resolveDependency(LabelScanStoreProvider.class, new NamedLabelScanStoreSelectionStrategy(config));
    dependencyResolver.resolveDependency(LabelScanStoreProvider.class, new DeleteStoresFromOtherLabelScanStoreProviders(labelScanStoreProvider));
    IndexConfigStore indexConfigStore = new IndexConfigStore(storeDir, fs);
    dependencies.satisfyDependency(lockService);
    dependencies.satisfyDependency(indexConfigStore);
    life.add(indexConfigStore);
    // Monitor listeners
    LoggingLogFileMonitor loggingLogMonitor = new LoggingLogFileMonitor(msgLog);
    monitors.addMonitorListener(loggingLogMonitor);
    life.add(new Delegate(Lifecycles.multiple(indexProviders.values())));
    // Upgrade the store before we begin
    RecordFormats formats = selectStoreFormats(config, storeDir, fs, pageCache, logService);
    upgradeStore(formats);
    // Build all modules and their services
    StorageEngine storageEngine = null;
    try {
        UpdateableSchemaState updateableSchemaState = new KernelSchemaStateStore(logProvider);
        SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering = new SynchronizedArrayIdOrderingQueue(20);
        storageEngine = buildStorageEngine(propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, legacyIndexProviderLookup, indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering);
        LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader<>(storageEngine.commandReaderFactory());
        TransactionIdStore transactionIdStore = dependencies.resolveDependency(TransactionIdStore.class);
        LogVersionRepository logVersionRepository = dependencies.resolveDependency(LogVersionRepository.class);
        NeoStoreTransactionLogModule transactionLogModule = buildTransactionLogs(storeDir, config, logProvider, scheduler, fs, storageEngine, logEntryReader, legacyIndexTransactionOrdering, transactionIdStore, logVersionRepository);
        transactionLogModule.satisfyDependencies(dependencies);
        buildRecovery(fs, transactionIdStore, logVersionRepository, monitors.newMonitor(Recovery.Monitor.class), monitors.newMonitor(PositionToRecoverFrom.Monitor.class), transactionLogModule.logFiles(), startupStatistics, storageEngine, logEntryReader, transactionLogModule.logicalTransactionStore());
        // At the time of writing this comes from the storage engine (IndexStoreView)
        PropertyAccessor propertyAccessor = dependencies.resolveDependency(PropertyAccessor.class);
        final NeoStoreKernelModule kernelModule = buildKernel(transactionLogModule.transactionAppender(), dependencies.resolveDependency(IndexingService.class), storageEngine.storeReadLayer(), updateableSchemaState, dependencies.resolveDependency(LabelScanStore.class), storageEngine, indexConfigStore, transactionIdStore, availabilityGuard, clock, propertyAccessor);
        kernelModule.satisfyDependencies(dependencies);
        // Do these assignments last so that we can ensure no cyclical dependencies exist
        this.storageEngine = storageEngine;
        this.transactionLogModule = transactionLogModule;
        this.kernelModule = kernelModule;
        dependencies.satisfyDependency(this);
        dependencies.satisfyDependency(updateableSchemaState);
        dependencies.satisfyDependency(storageEngine.storeReadLayer());
        dependencies.satisfyDependency(logEntryReader);
        dependencies.satisfyDependency(storageEngine);
    } catch (Throwable e) {
        // Something unexpected happened during startup
        msgLog.warn("Exception occurred while setting up store modules. Attempting to close things down.", e);
        try {
            // Close the neostore, so that locks are released properly
            if (storageEngine != null) {
                storageEngine.forceClose();
            }
        } catch (Exception closeException) {
            msgLog.error("Couldn't close neostore after startup failure", closeException);
        }
        throw Exceptions.launderedException(e);
    }
    // NOTE: please make sure this is performed after having added everything to the life, in fact we would like
    // to perform the checkpointing as first step when the life is shutdown.
    life.add(lifecycleToTriggerCheckPointOnShutdown());
    try {
        life.start();
    } catch (Throwable e) {
        // Something unexpected happened during startup
        msgLog.warn("Exception occurred while starting the datasource. Attempting to close things down.", e);
        try {
            life.shutdown();
            // Close the neostore, so that locks are released properly
            storageEngine.forceClose();
        } catch (Exception closeException) {
            msgLog.error("Couldn't close neostore after startup failure", closeException);
        }
        throw Exceptions.launderedException(e);
    }
    /*
         * At this point recovery has completed and the datasource is ready for use. Whatever panic might have
         * happened before has been healed. So we can safely set the kernel health to ok.
         * This right now has any real effect only in the case of internal restarts (for example, after a store copy
         * in the case of HA). Standalone instances will have to be restarted by the user, as is proper for all
         * kernel panics.
         */
    databaseHealth.healed();
}

Example 10 with StorageEngine

use of org.neo4j.storageengine.api.StorageEngine in project neo4j by neo4j.

the class ReplicatedTokenHolderTest method mockedStorageEngine.

@SuppressWarnings("unchecked")
private StorageEngine mockedStorageEngine() throws Exception {
    StorageEngine storageEngine = mock(StorageEngine.class);
    doAnswer(invocation -> {
        Collection<StorageCommand> target = invocation.getArgumentAt(0, Collection.class);
        ReadableTransactionState txState = invocation.getArgumentAt(1, ReadableTransactionState.class);
        txState.accept(new TxStateVisitor.Adapter() {

            @Override
            public void visitCreatedLabelToken(String name, int id) {
                LabelTokenRecord before = new LabelTokenRecord(id);
                LabelTokenRecord after = before.clone();
                after.setInUse(true);
                target.add(new Command.LabelTokenCommand(before, after));
            }
        });
        return null;
    }).when(storageEngine).createCommands(anyCollection(), any(ReadableTransactionState.class), any(StorageStatement.class), any(ResourceLocker.class), anyLong());
    StoreReadLayer readLayer = mock(StoreReadLayer.class);
    when(readLayer.newStatement()).thenReturn(mock(StorageStatement.class));
    when(storageEngine.storeReadLayer()).thenReturn(readLayer);
    return storageEngine;
}