use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.
the class TestSSTable method testDelayedDeletion.
/**
* Test delayed deletion
* @throws Exception
*/
@Test(timeout = 60000)
public void testDelayedDeletion() throws Exception {
final String relationDirectory = SSTableHelper.getSSTableDir(STORAGE_DIRECTORY, TEST_RELATION);
final File relationDirectoryFile = new File(relationDirectory);
FileUtil.deleteRecursive(relationDirectoryFile.toPath());
// Directory should be empty
Assert.assertFalse(relationDirectoryFile.isDirectory());
relationDirectoryFile.mkdirs();
final List<Tuple> tupleList = createTupleList();
final SSTableWriter ssTableWriter = new SSTableWriter(STORAGE_DIRECTORY, TEST_RELATION, 1, EXPECTED_TUPLES);
ssTableWriter.open();
ssTableWriter.addData(tupleList);
final File sstableFile = ssTableWriter.getSstableFile();
final File sstableIndexFile = ssTableWriter.getSstableIndexFile();
ssTableWriter.close();
Assert.assertTrue(sstableFile.exists());
Assert.assertTrue(sstableIndexFile.exists());
final ReadOnlyTupleStore ssTableFacade = new SSTableFacade(STORAGE_DIRECTORY, TEST_RELATION, 1, 0);
ssTableFacade.acquire();
ssTableFacade.deleteOnClose();
// Directory should contain at least the sstable, the key index and the meta data file
Assert.assertTrue(relationDirectoryFile.listFiles().length >= 3);
Assert.assertTrue(sstableFile.exists());
Assert.assertTrue(sstableIndexFile.exists());
// After calling release, the files can be deleted
ssTableFacade.release();
Assert.assertFalse(sstableFile.exists());
Assert.assertFalse(sstableIndexFile.exists());
// Directory should be empty
Assert.assertEquals(0, relationDirectoryFile.listFiles().length);
}
use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.
the class TupleStoreManager method get.
/**
* Search for the most recent version of the tuple
* @param key
* @return The tuple or null
* @throws StorageManagerException
*/
public List<Tuple> get(final String key) throws StorageManagerException {
if (!serviceState.isInRunningState()) {
throw new StorageManagerException("Storage manager is not ready: " + tupleStoreName.getFullname() + " state: " + serviceState);
}
final Summary.Timer requestTimer = getRequestLatency.startTimer();
final List<ReadOnlyTupleStore> aquiredStorages = new ArrayList<>();
final List<Tuple> tupleList = new ArrayList<>();
try {
aquiredStorages.addAll(aquireStorage());
for (final ReadOnlyTupleStore tupleStorage : aquiredStorages) {
final List<Tuple> resultTuples = tupleStorage.get(key);
tupleList.addAll(resultTuples);
}
} catch (Exception e) {
throw e;
} finally {
releaseStorage(aquiredStorages);
requestTimer.observeDuration();
}
final DuplicateResolver<Tuple> resolver = TupleDuplicateResolverFactory.build(tupleStoreConfiguration);
resolver.removeDuplicates(tupleList);
return tupleList;
}
use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.
the class TupleStoreManager method aquireStorage.
/**
* Try to acquire all needed tables
* @return
* @throws StorageManagerException
*/
public List<ReadOnlyTupleStore> aquireStorage() throws StorageManagerException {
for (int execution = 0; execution < Const.OPERATION_RETRY; execution++) {
final List<ReadOnlyTupleStore> aquiredStorages = new ArrayList<>();
final List<ReadOnlyTupleStore> knownStorages = tupleStoreInstances.getAllTupleStorages();
for (final ReadOnlyTupleStore tupleStorage : knownStorages) {
final boolean canBeUsed = tupleStorage.acquire();
if (!canBeUsed) {
if (execution == Const.OPERATION_RETRY - 1) {
logger.error("Unable to aquire: {} with {} retries", tupleStorage, execution);
}
break;
} else {
aquiredStorages.add(tupleStorage);
}
}
if (knownStorages.size() == aquiredStorages.size()) {
return aquiredStorages;
} else {
// one or more storages could not be acquired
// release storages and retry
releaseStorage(aquiredStorages);
try {
// Wait some time and try again
Thread.sleep(10);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
}
}
throw new StorageManagerException("Unable to aquire all sstables in " + Const.OPERATION_RETRY + " retries");
}
use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.
the class SSTableCheckpointRunnable method isCheckpointNeeded.
/**
* Decide if a new checkpoint is needed
* @return
*/
protected boolean isCheckpointNeeded(final TupleStoreManager ssTableManager) {
final List<ReadOnlyTupleStore> inMemoryStores = ssTableManager.getAllInMemoryStorages();
if (inMemoryStores.isEmpty()) {
return false;
}
final long currentTime = System.currentTimeMillis();
// The checkpoint predicate
final LongPredicate checkpointPredicate = m -> {
final long checkpointThreshold = TimeUnit.MICROSECONDS.toMillis(m) + maxUncheckpointedMiliseconds;
return checkpointThreshold < currentTime;
};
final boolean checkpointNeeded = inMemoryStores.stream().filter(Objects::nonNull).mapToLong(m -> m.getOldestTupleVersionTimestamp()).anyMatch(checkpointPredicate);
logger.debug("Checkpoint for {} needed {}", ssTableManager.getTupleStoreName().getFullname(), checkpointNeeded);
return checkpointNeeded;
}
use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.
the class RegionMerger method mergeDataByLocalRead.
/**
* Merge data by local data read
*
* @param region
* @param tupleStoreName
* @param tupleRedistributor
* @param childRegion
* @throws StorageManagerException
* @throws TupleStoreManagerRegistry
*/
private void mergeDataByLocalRead(final DistributionRegion region, final TupleStoreName tupleStoreName, final TupleRedistributor tupleRedistributor, final DistributionRegion childRegion) throws StorageManagerException {
final long regionId = region.getRegionId();
final TupleStoreName childRegionName = tupleStoreName.cloneWithDifferntRegionId(regionId);
final TupleStoreManager tupleStoreManager = registry.getTupleStoreManager(childRegionName);
final List<ReadOnlyTupleStore> storages = new ArrayList<>();
try {
storages.addAll(tupleStoreManager.aquireStorage());
for (final ReadOnlyTupleStore storage : storages) {
for (final Tuple tuple : storage) {
tupleRedistributor.redistributeTuple(tuple);
}
}
} catch (Exception e) {
throw e;
} finally {
tupleStoreManager.releaseStorage(storages);
}
}
Aggregations