Search in sources :

Example 1 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testGetTemporaryFilesThrowsIfCompletingAfterObsoletion.

@Test
public void testGetTemporaryFilesThrowsIfCompletingAfterObsoletion() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
    LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(logs);
    LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);
    sstable.markObsolete(tidier);
    sstable.selfRef().release();
    LogTransaction.waitForDeletions();
    try {
        // This should race with the asynchronous deletion of txn log files
        // it should throw because we are violating the requirement that a transaction must
        // finish before deleting files (i.e. releasing sstables)
        getTemporaryFiles(dataFolder);
        fail("Expected runtime exception");
    } catch (RuntimeException e) {
        //pass as long as the cause is not an assertion
        assertFalse(e.getCause() instanceof AssertionError);
    }
    logs.finish();
}
Also used : Directories(org.apache.cassandra.db.Directories) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) ColumnFamilyStore(org.apache.cassandra.db.ColumnFamilyStore) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) Test(org.junit.Test) AbstractTransactionalTest(org.apache.cassandra.utils.concurrent.AbstractTransactionalTest)

Example 2 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testRemoveUnfinishedLeftovers_abort_multipleFolders.

@Test
public void testRemoveUnfinishedLeftovers_abort_multipleFolders() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());
    SSTableReader[] sstables = { sstable(dataFolder1, cfs, 0, 128), sstable(dataFolder1, cfs, 1, 128), sstable(dataFolder2, cfs, 2, 128), sstable(dataFolder2, cfs, 3, 128) };
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);
    Collection<File> logFiles = log.logFiles();
    Assert.assertEquals(2, logFiles.size());
    // fake an abort
    log.txnFile().abort();
    Arrays.stream(sstables).forEach(s -> s.selfRef().release());
    // test listing
    Assert.assertEquals(sstables[1].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()), getTemporaryFiles(dataFolder1));
    Assert.assertEquals(sstables[3].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()), getTemporaryFiles(dataFolder2));
    // normally called at startup
    assertTrue(LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
    // old tables should be only table left
    assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[0].getAllFilePaths()));
    assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[2].getAllFilePaths()));
    // complete the transaction to avoid LEAK errors
    Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
    assertNull(log.complete(null));
}
Also used : Directories(org.apache.cassandra.db.Directories) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) ColumnFamilyStore(org.apache.cassandra.db.ColumnFamilyStore) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) Test(org.junit.Test) AbstractTransactionalTest(org.apache.cassandra.utils.concurrent.AbstractTransactionalTest)

Example 3 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testRemoveUnfinishedLeftovers_commit_multipleFolders.

@Test
public void testRemoveUnfinishedLeftovers_commit_multipleFolders() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());
    SSTableReader[] sstables = { sstable(dataFolder1, cfs, 0, 128), sstable(dataFolder1, cfs, 1, 128), sstable(dataFolder2, cfs, 2, 128), sstable(dataFolder2, cfs, 3, 128) };
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);
    Collection<File> logFiles = log.logFiles();
    Assert.assertEquals(2, logFiles.size());
    // fake a commit
    log.txnFile().commit();
    Arrays.stream(sstables).forEach(s -> s.selfRef().release());
    // test listing
    Assert.assertEquals(sstables[0].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()), getTemporaryFiles(dataFolder1));
    Assert.assertEquals(sstables[2].getAllFilePaths().stream().map(File::new).collect(Collectors.toSet()), getTemporaryFiles(dataFolder2));
    // normally called at startup
    assertTrue(LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
    // new tables should be only table left
    assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[1].getAllFilePaths()));
    assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[3].getAllFilePaths()));
    // complete the transaction to avoid LEAK errors
    Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
    assertNull(log.complete(null));
}
Also used : Directories(org.apache.cassandra.db.Directories) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) ColumnFamilyStore(org.apache.cassandra.db.ColumnFamilyStore) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) Test(org.junit.Test) AbstractTransactionalTest(org.apache.cassandra.utils.concurrent.AbstractTransactionalTest)

Example 4 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testGetTemporaryFilesSafeAfterObsoletion.

@Test
public void testGetTemporaryFilesSafeAfterObsoletion() throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File dataFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    SSTableReader sstable = sstable(dataFolder, cfs, 0, 128);
    LogTransaction logs = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(logs);
    LogTransaction.SSTableTidier tidier = logs.obsoleted(sstable);
    logs.finish();
    sstable.markObsolete(tidier);
    sstable.selfRef().release();
    // was completed before deleting files (i.e. releasing sstables)
    for (int i = 0; i < 200; i++) getTemporaryFiles(dataFolder);
}
Also used : Directories(org.apache.cassandra.db.Directories) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) ColumnFamilyStore(org.apache.cassandra.db.ColumnFamilyStore) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File) Test(org.junit.Test) AbstractTransactionalTest(org.apache.cassandra.utils.concurrent.AbstractTransactionalTest)

Example 5 with Directories

use of org.apache.cassandra.db.Directories in project cassandra by apache.

the class LogTransactionTest method testRemoveUnfinishedLeftovers_multipleFolders_errorConditions.

private static void testRemoveUnfinishedLeftovers_multipleFolders_errorConditions(Consumer<LogTransaction> modifier, boolean shouldCommit) throws Throwable {
    ColumnFamilyStore cfs = MockSchema.newCFS(KEYSPACE);
    File origiFolder = new Directories(cfs.metadata()).getDirectoryForNewSSTables();
    File dataFolder1 = new File(origiFolder, "1");
    File dataFolder2 = new File(origiFolder, "2");
    Files.createDirectories(dataFolder1.toPath());
    Files.createDirectories(dataFolder2.toPath());
    SSTableReader[] sstables = { sstable(dataFolder1, cfs, 0, 128), sstable(dataFolder1, cfs, 1, 128), sstable(dataFolder2, cfs, 2, 128), sstable(dataFolder2, cfs, 3, 128) };
    LogTransaction log = new LogTransaction(OperationType.COMPACTION);
    assertNotNull(log);
    LogTransaction.SSTableTidier[] tidiers = { log.obsoleted(sstables[0]), log.obsoleted(sstables[2]) };
    log.trackNew(sstables[1]);
    log.trackNew(sstables[3]);
    // fake some error condition on the txn logs
    modifier.accept(log);
    Arrays.stream(sstables).forEach(s -> s.selfRef().release());
    // if shouldCommit is true then it should remove the leftovers and return true, false otherwise
    assertEquals(shouldCommit, LogTransaction.removeUnfinishedLeftovers(Arrays.asList(dataFolder1, dataFolder2)));
    LogTransaction.waitForDeletions();
    if (shouldCommit) {
        // only new sstables should still be there
        assertFiles(dataFolder1.getPath(), new HashSet<>(sstables[1].getAllFilePaths()));
        assertFiles(dataFolder2.getPath(), new HashSet<>(sstables[3].getAllFilePaths()));
    } else {
        // all files should still be there
        assertFiles(dataFolder1.getPath(), Sets.newHashSet(Iterables.concat(sstables[0].getAllFilePaths(), sstables[1].getAllFilePaths(), Collections.singleton(log.logFilePaths().get(0)))));
        assertFiles(dataFolder2.getPath(), Sets.newHashSet(Iterables.concat(sstables[2].getAllFilePaths(), sstables[3].getAllFilePaths(), Collections.singleton(log.logFilePaths().get(1)))));
    }
    // complete the transaction to avoid LEAK errors
    Arrays.stream(tidiers).forEach(LogTransaction.SSTableTidier::run);
    // just anything to make sure transaction tidier will finish
    log.txnFile().commit();
    assertNull(log.complete(null));
}
Also used : Directories(org.apache.cassandra.db.Directories) SSTableReader(org.apache.cassandra.io.sstable.format.SSTableReader) ColumnFamilyStore(org.apache.cassandra.db.ColumnFamilyStore) RandomAccessFile(java.io.RandomAccessFile) File(java.io.File)

Aggregations

File (java.io.File)19 RandomAccessFile (java.io.RandomAccessFile)19 ColumnFamilyStore (org.apache.cassandra.db.ColumnFamilyStore)19 Directories (org.apache.cassandra.db.Directories)19 SSTableReader (org.apache.cassandra.io.sstable.format.SSTableReader)19 AbstractTransactionalTest (org.apache.cassandra.utils.concurrent.AbstractTransactionalTest)17 Test (org.junit.Test)17 ImmutableSet (com.google.common.collect.ImmutableSet)4 Iterables (com.google.common.collect.Iterables)1 Sets (com.google.common.collect.Sets)1 IOError (java.io.IOError)1 IOException (java.io.IOException)1 Files (java.nio.file.Files)1 java.util (java.util)1 BiConsumer (java.util.function.BiConsumer)1 Consumer (java.util.function.Consumer)1 Collectors (java.util.stream.Collectors)1 Assert (junit.framework.Assert)1 Assert.assertNotNull (junit.framework.Assert.assertNotNull)1 Assert.assertNull (junit.framework.Assert.assertNull)1