Examples with FileSystemAbstraction org.neo4j.io.fs.FileSystemAbstraction used on opensource projects

Example 51 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class DumpCommandTest method shouldReportAHelpfulErrorIfWeDontHaveWritePermissionsForLock.

@Test
public void shouldReportAHelpfulErrorIfWeDontHaveWritePermissionsForLock() throws Exception {
    assumeFalse("We haven't found a way to reliably tests permissions on Windows", SystemUtils.IS_OS_WINDOWS);
    try (FileSystemAbstraction fileSystem = new DefaultFileSystemAbstraction();
        StoreLocker storeLocker = new StoreLocker(fileSystem)) {
        storeLocker.checkLock(databaseDirectory.toFile());
        try (Closeable ignored = withPermissions(databaseDirectory.resolve(StoreLocker.STORE_LOCK_FILENAME), emptySet())) {
            execute("foo.db");
            fail("expected exception");
        } catch (CommandFailed e) {
            assertThat(e.getMessage(), equalTo("you do not have permission to dump the database -- is Neo4j " + "running as a different user?"));
        }
    }
}

Example 52 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}

Example 53 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class MuninnPageCacheTest method mustUnblockPageFaultersWhenEvictionGetsException.

@Test(timeout = SEMI_LONG_TIMEOUT_MILLIS)
public void mustUnblockPageFaultersWhenEvictionGetsException() throws Exception {
    writeInitialDataTo(file("a"));
    FileSystemAbstraction fs = new DelegatingFileSystemAbstraction(this.fs) {

        @Override
        public StoreChannel open(File fileName, String mode) throws IOException {
            return new DelegatingStoreChannel(super.open(fileName, mode)) {

                @Override
                public void writeAll(ByteBuffer src, long position) throws IOException {
                    throw new IOException("uh-oh...");
                }
            };
        }
    };
    MuninnPageCache pageCache = createPageCache(fs, 2, 8, PageCacheTracer.NULL, DefaultPageCursorTracerSupplier.INSTANCE);
    final PagedFile pagedFile = pageCache.map(file("a"), 8);
    // all writes.
    try (PageCursor cursor = pagedFile.io(0, PF_SHARED_WRITE_LOCK)) {
        for (int i = 0; i < 1000; i++) {
            assertTrue(cursor.next());
        }
        fail("Expected an exception at this point");
    } catch (IOException ignore) {
    // Good.
    }
}

Example 54 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class BatchingTransactionAppenderConcurrencyTest method shouldHaveAllConcurrentAppendersSeePanic.

/*
     * There was an issue where if multiple concurrent appending threads did append and they moved on
     * to await a force, where the force would fail and the one doing the force would raise a panic...
     * the other threads may not notice the panic and move on to mark those transactions as committed
     * and notice the panic later (which would be too late).
     */
@Test
public void shouldHaveAllConcurrentAppendersSeePanic() throws Throwable {
    // GIVEN
    Adversary adversary = new ClassGuardedAdversary(new CountingAdversary(1, true), failMethod(BatchingTransactionAppender.class, "force"));
    EphemeralFileSystemAbstraction efs = new EphemeralFileSystemAbstraction();
    File directory = new File("dir").getCanonicalFile();
    efs.mkdirs(directory);
    FileSystemAbstraction fs = new AdversarialFileSystemAbstraction(adversary, efs);
    life.add(new FileSystemLifecycleAdapter(fs));
    DatabaseHealth databaseHealth = new DatabaseHealth(mock(DatabasePanicEventGenerator.class), NullLog.getInstance());
    PhysicalLogFiles logFiles = new PhysicalLogFiles(directory, fs);
    LogFile logFile = life.add(new PhysicalLogFile(fs, logFiles, kibiBytes(10), transactionIdStore::getLastCommittedTransactionId, new DeadSimpleLogVersionRepository(0), new PhysicalLogFile.Monitor.Adapter(), logHeaderCache));
    final BatchingTransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    life.start();
    // WHEN
    int numberOfAppenders = 10;
    final CountDownLatch trap = new CountDownLatch(numberOfAppenders);
    final LogAppendEvent beforeForceTrappingEvent = new LogAppendEvent.Empty() {

        @Override
        public LogForceWaitEvent beginLogForceWait() {
            trap.countDown();
            awaitLatch(trap);
            return super.beginLogForceWait();
        }
    };
    Race race = new Race();
    for (int i = 0; i < numberOfAppenders; i++) {
        race.addContestant(() -> {
            try {
                // Append to the log, the LogAppenderEvent will have all of the appending threads
                // do wait for all of the other threads to start the force thing
                appender.append(tx(), beforeForceTrappingEvent);
                fail("No transaction should be considered appended");
            } catch (IOException e) {
            // Good, we know that this test uses an adversarial file system which will throw
            // an exception in BatchingTransactionAppender#force, and since all these transactions
            // will append and be forced in the same batch, where the force will fail then
            // all these transactions should fail. If there's any transaction not failing then
            // it just didn't notice the panic, which would be potentially hazardous.
            }
        });
    }
    // THEN perform the race. The relevant assertions are made inside the contestants.
    race.go();
}

Example 55 with FileSystemAbstraction

use of org.neo4j.io.fs.FileSystemAbstraction in project neo4j by neo4j.

the class PhysicalLogFileTest method shouldCloseChannelInFailedAttemptToReadHeaderAfterOpen.

@Test
public void shouldCloseChannelInFailedAttemptToReadHeaderAfterOpen() throws Exception {
    // GIVEN a file which returns 1/2 log header size worth of bytes
    File directory = new File("/dir");
    FileSystemAbstraction fs = mock(FileSystemAbstraction.class);
    PhysicalLogFiles logFiles = new PhysicalLogFiles(directory, fs);
    int logVersion = 0;
    File logFile = logFiles.getLogFileForVersion(logVersion);
    StoreChannel channel = mock(StoreChannel.class);
    when(channel.read(any(ByteBuffer.class))).thenReturn(LogHeader.LOG_HEADER_SIZE / 2);
    when(fs.fileExists(logFile)).thenReturn(true);
    when(fs.open(eq(logFile), anyString())).thenReturn(channel);
    // WHEN
    try {
        PhysicalLogFile.openForVersion(logFiles, fs, logVersion, false);
        fail("Should have failed");
    } catch (IncompleteLogHeaderException e) {
        // THEN good
        verify(channel).close();
    }
}