Examples with ReadOnlyTupleStore org.bboxdb.storage.tuplestore.ReadOnlyTupleStore used on opensource projects

Example 6 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class TestSSTable method testDelayedDeletion.

/**
 * Test delayed deletion
 * @throws Exception
 */
@Test(timeout = 60000)
public void testDelayedDeletion() throws Exception {
    final String relationDirectory = SSTableHelper.getSSTableDir(STORAGE_DIRECTORY, TEST_RELATION);
    final File relationDirectoryFile = new File(relationDirectory);
    FileUtil.deleteRecursive(relationDirectoryFile.toPath());
    // Directory should be empty
    Assert.assertFalse(relationDirectoryFile.isDirectory());
    relationDirectoryFile.mkdirs();
    final List<Tuple> tupleList = createTupleList();
    final SSTableWriter ssTableWriter = new SSTableWriter(STORAGE_DIRECTORY, TEST_RELATION, 1, EXPECTED_TUPLES);
    ssTableWriter.open();
    ssTableWriter.addData(tupleList);
    final File sstableFile = ssTableWriter.getSstableFile();
    final File sstableIndexFile = ssTableWriter.getSstableIndexFile();
    ssTableWriter.close();
    Assert.assertTrue(sstableFile.exists());
    Assert.assertTrue(sstableIndexFile.exists());
    final ReadOnlyTupleStore ssTableFacade = new SSTableFacade(STORAGE_DIRECTORY, TEST_RELATION, 1, 0);
    ssTableFacade.acquire();
    ssTableFacade.deleteOnClose();
    // Directory should contain at least the sstable, the key index and the meta data file
    Assert.assertTrue(relationDirectoryFile.listFiles().length >= 3);
    Assert.assertTrue(sstableFile.exists());
    Assert.assertTrue(sstableIndexFile.exists());
    // After calling release, the files can be deleted
    ssTableFacade.release();
    Assert.assertFalse(sstableFile.exists());
    Assert.assertFalse(sstableIndexFile.exists());
    // Directory should be empty
    Assert.assertEquals(0, relationDirectoryFile.listFiles().length);
}

Example 7 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class TupleStoreManager method get.

/**
 * Search for the most recent version of the tuple
 * @param key
 * @return The tuple or null
 * @throws StorageManagerException
 */
public List<Tuple> get(final String key) throws StorageManagerException {
    if (!serviceState.isInRunningState()) {
        throw new StorageManagerException("Storage manager is not ready: " + tupleStoreName.getFullname() + " state: " + serviceState);
    }
    final Summary.Timer requestTimer = getRequestLatency.startTimer();
    final List<ReadOnlyTupleStore> aquiredStorages = new ArrayList<>();
    final List<Tuple> tupleList = new ArrayList<>();
    try {
        aquiredStorages.addAll(aquireStorage());
        for (final ReadOnlyTupleStore tupleStorage : aquiredStorages) {
            final List<Tuple> resultTuples = tupleStorage.get(key);
            tupleList.addAll(resultTuples);
        }
    } catch (Exception e) {
        throw e;
    } finally {
        releaseStorage(aquiredStorages);
        requestTimer.observeDuration();
    }
    final DuplicateResolver<Tuple> resolver = TupleDuplicateResolverFactory.build(tupleStoreConfiguration);
    resolver.removeDuplicates(tupleList);
    return tupleList;
}

Example 8 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class TupleStoreManager method aquireStorage.

/**
 * Try to acquire all needed tables
 * @return
 * @throws StorageManagerException
 */
public List<ReadOnlyTupleStore> aquireStorage() throws StorageManagerException {
    for (int execution = 0; execution < Const.OPERATION_RETRY; execution++) {
        final List<ReadOnlyTupleStore> aquiredStorages = new ArrayList<>();
        final List<ReadOnlyTupleStore> knownStorages = tupleStoreInstances.getAllTupleStorages();
        for (final ReadOnlyTupleStore tupleStorage : knownStorages) {
            final boolean canBeUsed = tupleStorage.acquire();
            if (!canBeUsed) {
                if (execution == Const.OPERATION_RETRY - 1) {
                    logger.error("Unable to aquire: {} with {} retries", tupleStorage, execution);
                }
                break;
            } else {
                aquiredStorages.add(tupleStorage);
            }
        }
        if (knownStorages.size() == aquiredStorages.size()) {
            return aquiredStorages;
        } else {
            // one or more storages could not be acquired
            // release storages and retry
            releaseStorage(aquiredStorages);
            try {
                // Wait some time and try again
                Thread.sleep(10);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
        }
    }
    throw new StorageManagerException("Unable to aquire all sstables in " + Const.OPERATION_RETRY + " retries");
}

Example 9 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class SSTableCheckpointRunnable method isCheckpointNeeded.

/**
 * Decide if a new checkpoint is needed
 * @return
 */
protected boolean isCheckpointNeeded(final TupleStoreManager ssTableManager) {
    final List<ReadOnlyTupleStore> inMemoryStores = ssTableManager.getAllInMemoryStorages();
    if (inMemoryStores.isEmpty()) {
        return false;
    }
    final long currentTime = System.currentTimeMillis();
    // The checkpoint predicate
    final LongPredicate checkpointPredicate = m -> {
        final long checkpointThreshold = TimeUnit.MICROSECONDS.toMillis(m) + maxUncheckpointedMiliseconds;
        return checkpointThreshold < currentTime;
    };
    final boolean checkpointNeeded = inMemoryStores.stream().filter(Objects::nonNull).mapToLong(m -> m.getOldestTupleVersionTimestamp()).anyMatch(checkpointPredicate);
    logger.debug("Checkpoint for {} needed {}", ssTableManager.getTupleStoreName().getFullname(), checkpointNeeded);
    return checkpointNeeded;
}

Example 10 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class RegionMerger method mergeDataByLocalRead.

/**
 * Merge data by local data read
 *
 * @param region
 * @param tupleStoreName
 * @param tupleRedistributor
 * @param childRegion
 * @throws StorageManagerException
 * @throws TupleStoreManagerRegistry
 */
private void mergeDataByLocalRead(final DistributionRegion region, final TupleStoreName tupleStoreName, final TupleRedistributor tupleRedistributor, final DistributionRegion childRegion) throws StorageManagerException {
    final long regionId = region.getRegionId();
    final TupleStoreName childRegionName = tupleStoreName.cloneWithDifferntRegionId(regionId);
    final TupleStoreManager tupleStoreManager = registry.getTupleStoreManager(childRegionName);
    final List<ReadOnlyTupleStore> storages = new ArrayList<>();
    try {
        storages.addAll(tupleStoreManager.aquireStorage());
        for (final ReadOnlyTupleStore storage : storages) {
            for (final Tuple tuple : storage) {
                tupleRedistributor.redistributeTuple(tuple);
            }
        }
    } catch (Exception e) {
        throw e;
    } finally {
        tupleStoreManager.releaseStorage(storages);
    }
}