Examples with SSTableWriter org.bboxdb.storage.sstable.SSTableWriter used on opensource projects

Example 6 with SSTableWriter

use of org.bboxdb.storage.sstable.SSTableWriter in project bboxdb by jnidzwetzki.

the class TestSSTable method testWrittenFilesWithExceiton.

/**
 * Test written files
 * @throws Exception
 */
@Test(timeout = 60000)
public void testWrittenFilesWithExceiton() throws Exception {
    final String relationDirectory = SSTableHelper.getSSTableDir(STORAGE_DIRECTORY, TEST_RELATION);
    final File relationDirectoryFile = new File(relationDirectory);
    FileUtil.deleteRecursive(relationDirectoryFile.toPath());
    Assert.assertFalse(relationDirectoryFile.exists());
    relationDirectoryFile.mkdirs();
    final List<Tuple> tupleList = createTupleList();
    final SSTableWriter ssTableWriter = new SSTableWriter(STORAGE_DIRECTORY, TEST_RELATION, 1, EXPECTED_TUPLES);
    ssTableWriter.open();
    ssTableWriter.addData(tupleList);
    final File sstableFile = ssTableWriter.getSstableFile();
    final File sstableIndexFile = ssTableWriter.getSstableIndexFile();
    Assert.assertFalse(ssTableWriter.isErrorFlagSet());
    ssTableWriter.setErrorFlag();
    Assert.assertTrue(ssTableWriter.isErrorFlagSet());
    ssTableWriter.close();
    Assert.assertFalse(sstableFile.exists());
    Assert.assertFalse(sstableIndexFile.exists());
}

Example 7 with SSTableWriter

use of org.bboxdb.storage.sstable.SSTableWriter in project bboxdb by jnidzwetzki.

the class TestSSTable method testDelayedDeletion.

/**
 * Test delayed deletion
 * @throws Exception
 */
@Test(timeout = 60000)
public void testDelayedDeletion() throws Exception {
    final String relationDirectory = SSTableHelper.getSSTableDir(STORAGE_DIRECTORY, TEST_RELATION);
    final File relationDirectoryFile = new File(relationDirectory);
    FileUtil.deleteRecursive(relationDirectoryFile.toPath());
    // Directory should be empty
    Assert.assertFalse(relationDirectoryFile.isDirectory());
    relationDirectoryFile.mkdirs();
    final List<Tuple> tupleList = createTupleList();
    final SSTableWriter ssTableWriter = new SSTableWriter(STORAGE_DIRECTORY, TEST_RELATION, 1, EXPECTED_TUPLES);
    ssTableWriter.open();
    ssTableWriter.addData(tupleList);
    final File sstableFile = ssTableWriter.getSstableFile();
    final File sstableIndexFile = ssTableWriter.getSstableIndexFile();
    ssTableWriter.close();
    Assert.assertTrue(sstableFile.exists());
    Assert.assertTrue(sstableIndexFile.exists());
    final ReadOnlyTupleStore ssTableFacade = new SSTableFacade(STORAGE_DIRECTORY, TEST_RELATION, 1, 0);
    ssTableFacade.acquire();
    ssTableFacade.deleteOnClose();
    // Directory should contain at least the sstable, the key index and the meta data file
    Assert.assertTrue(relationDirectoryFile.listFiles().length >= 3);
    Assert.assertTrue(sstableFile.exists());
    Assert.assertTrue(sstableIndexFile.exists());
    // After calling release, the files can be deleted
    ssTableFacade.release();
    Assert.assertFalse(sstableFile.exists());
    Assert.assertFalse(sstableIndexFile.exists());
    // Directory should be empty
    Assert.assertEquals(0, relationDirectoryFile.listFiles().length);
}

Example 8 with SSTableWriter

use of org.bboxdb.storage.sstable.SSTableWriter in project bboxdb by jnidzwetzki.

the class TestTableCompactor method testCompactTestFileCreation.

@Test(timeout = 60000)
public void testCompactTestFileCreation() throws StorageManagerException {
    final List<Tuple> tupleList1 = new ArrayList<Tuple>();
    tupleList1.add(new Tuple("1", BoundingBox.FULL_SPACE, "abc".getBytes()));
    final SSTableKeyIndexReader reader1 = addTuplesToFileAndGetReader(tupleList1, 1);
    final List<Tuple> tupleList2 = new ArrayList<Tuple>();
    tupleList2.add(new Tuple("2", BoundingBox.FULL_SPACE, "def".getBytes()));
    final SSTableKeyIndexReader reader2 = addTuplesToFileAndGetReader(tupleList2, 2);
    storageRegistry.deleteTable(TEST_RELATION, true);
    storageRegistry.createTable(TEST_RELATION, new TupleStoreConfiguration());
    final TupleStoreManager storageManager = storageRegistry.getTupleStoreManager(TEST_RELATION);
    final SSTableCompactor compactor = new SSTableCompactor(storageManager, Arrays.asList(reader1, reader2));
    compactor.executeCompactation();
    final List<SSTableWriter> resultWriter = compactor.getResultList();
    Assert.assertEquals(1, resultWriter.size());
    Assert.assertEquals(2, compactor.getReadTuples());
    Assert.assertEquals(2, compactor.getWrittenTuples());
    for (final SSTableWriter writer : resultWriter) {
        Assert.assertTrue(writer.getSstableFile().exists());
        Assert.assertTrue(writer.getSstableIndexFile().exists());
        writer.close();
    }
}

Example 9 with SSTableWriter

use of org.bboxdb.storage.sstable.SSTableWriter in project bboxdb by jnidzwetzki.

the class SSTableCompactor method openNewSSTableWriter.

/**
 * Open a new SSTable writer
 * @param resultList
 * @return
 * @throws StorageManagerException
 */
protected SSTableWriter openNewSSTableWriter() throws StorageManagerException {
    final long estimatedMaxNumberOfEntries = calculateNumberOfEntries(sstableIndexReader);
    final String directory = sstableIndexReader.get(0).getDirectory();
    final int tablenumber = tupleStoreManager.increaseTableNumber();
    final SSTableWriter sstableWriter = new SSTableWriter(directory, tupleStoreManager.getTupleStoreName(), tablenumber, estimatedMaxNumberOfEntries);
    sstableWriter.open();
    resultList.add(sstableWriter);
    logger.info("Output file for compact: {}", sstableWriter.getSstableFile());
    return sstableWriter;
}

Example 10 with SSTableWriter

use of org.bboxdb.storage.sstable.SSTableWriter in project bboxdb by jnidzwetzki.

the class MemtableWriterRunnable method writeMemtable.

/**
 * Write a memtable to disk and return the file handle of the table
 * @param dataDirectory
 *
 * @param memtable
 * @param sstableManager
 * @return
 * @throws Exception
 */
protected int writeMemtable(final String dataDirectory, final Memtable memtable, final TupleStoreManager sstableManager) throws Exception {
    final int tableNumber = sstableManager.increaseTableNumber();
    logger.info("Writing memtable number {} with {} entries and a size of {}", tableNumber, memtable.getNumberOfTuples(), FileSizeHelper.readableFileSize(memtable.getSize()));
    try (final SSTableWriter ssTableWriter = new SSTableWriter(dataDirectory, sstableManager.getTupleStoreName(), tableNumber, memtable.getMaxEntries())) {
        ssTableWriter.open();
        ssTableWriter.addData(memtable.getSortedTupleList());
        return tableNumber;
    } catch (Exception e) {
        throw e;
    }
}