Examples with Directories org.apache.cassandra.db.Directories used on opensource projects

Example 6 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testCommitMultipleFolders.

@Test
public void testCommitMultipleFolders() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());
    SSTableReader[] sstables = { sstable(dataFolder1, cfs, 0, 128), sstable(dataFolder1, cfs, 1, 128), sstable(dataFolder2, cfs, 2, 128), sstable(dataFolder2, cfs, 3, 128) };
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);
    log.finish();
    sstables[0].markObsolete(tidiers[0]);
    sstables[2].markObsolete(tidiers[1]);
    Arrays.stream(sstables).forEach(s -> s.selfRef().release());
    LogTransaction.waitForDeletions();
    assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[1].getAllFilePaths()));
    assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[3].getAllFilePaths()));
}

Example 7 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testRemoveUnfinishedLeftovers_abort.

@Test
public void testRemoveUnfinishedLeftovers_abort() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
    // simulate tracking sstables with a failed transaction (new log file NOT deleted)
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    log.trackNew(sstableNew);
    LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
    Set<File> tmpFiles = sstableNew.getAllFilePaths().stream().map(File::new).collect(Collectors.toSet());
    sstableNew.selfRef().release();
    sstableOld.selfRef().release();
    Assert.assertEquals(tmpFiles, getTemporaryFiles(sstableNew.descriptor.directory));
    // normally called at startup
    LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
    // sstableOld should be only table left
    Directories directories = new Directories(cfs.metadata());
    Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
    assertEquals(1, sstables.size());
    assertFiles(dataFolder.getPath(), new HashSet<>(sstableOld.getAllFilePaths()));
    // complete the transaction before releasing files
    tidier.run();
    log.close();
}

Example 8 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testCommitOnlyOld.

@Test
public void testCommitOnlyOld() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    LogTransaction.SSTableTidier tidier = log.obsoleted(sstable);
    assertNotNull(tidier);
    log.finish();
    sstable.markObsolete(tidier);
    sstable.selfRef().release();
    assertFiles(dataFolder.getPath(), new HashSet<>());
}

Example 9 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testRemoveUnfinishedLeftovers_commit.

@Test
public void testRemoveUnfinishedLeftovers_commit() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
    // simulate tracking sstables with a committed transaction (new log file deleted)
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    log.trackNew(sstableNew);
    LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
    //Fake a commit
    log.txnFile().commit();
    Set<File> tmpFiles = sstableOld.getAllFilePaths().stream().map(File::new).collect(Collectors.toSet());
    sstableNew.selfRef().release();
    sstableOld.selfRef().release();
    Assert.assertEquals(tmpFiles, getTemporaryFiles(sstableOld.descriptor.directory));
    // normally called at startup
    LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
    // sstableNew should be only table left
    Directories directories = new Directories(cfs.metadata());
    Map<Descriptor, Set<Component>> sstables = directories.sstableLister(Directories.OnTxnErr.THROW).list();
    assertEquals(1, sstables.size());
    assertFiles(dataFolder.getPath(), new HashSet<>(sstableNew.getAllFilePaths()));
    // complete the transaction to avoid LEAK errors
    tidier.run();
    assertNull(log.complete(null));
}

Example 10 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testCorruptRecord.

private static void testCorruptRecord(BiConsumer<LogTransaction, SSTableReader> modifier, boolean isRecoverable) throws IOException {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    SSTableReader sstableOld = sstable(dataFolder, cfs, 0, 128);
    SSTableReader sstableNew = sstable(dataFolder, cfs, 1, 128);
    // simulate tracking sstables with a committed transaction except the checksum will be wrong
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    log.trackNew(sstableNew);
    LogTransaction.SSTableTidier tidier = log.obsoleted(sstableOld);
    // Modify the transaction log or disk state for sstableOld
    modifier.accept(log, sstableOld);
    // Sync the folder to make sure that later on removeUnfinishedLeftovers picks up
    // any changes to the txn files done by the modifier
    assertNull(log.txnFile().syncDirectory(null));
    assertNull(log.complete(null));
    sstableOld.selfRef().release();
    sstableNew.selfRef().release();
    // The files on disk, for old files make sure to exclude the files that were deleted by the modifier
    Set<String> newFiles = sstableNew.getAllFilePaths().stream().collect(Collectors.toSet());
    Set<String> oldFiles = sstableOld.getAllFilePaths().stream().filter(p -> new File(p).exists()).collect(Collectors.toSet());
    //This should filter as in progress since the last record is corrupt
    assertFiles(newFiles, getTemporaryFiles(dataFolder));
    assertFiles(oldFiles, getFinalFiles(dataFolder));
    if (isRecoverable) {
        // the corruption is recoverable but the commit record is unreadable so the transaction is still in progress
        //This should remove new files
        LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
        // make sure to exclude the old files that were deleted by the modifier
        assertFiles(dataFolder.getPath(), oldFiles);
    } else {
        // if an intermediate line was also modified, it should ignore the tx log file
        //This should not remove any files
        LogTransaction.removeUnfinishedLeftovers(cfs.metadata());
        assertFiles(dataFolder.getPath(), Sets.newHashSet(Iterables.concat(newFiles, oldFiles, log.logFilePaths())));
    }
    // make sure to run the tidier to avoid any leaks in the logs
    tidier.run();
}