Search in sources :

Example 6 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class TestSSTable method testDelayedDeletion.

/**
 * Test delayed deletion
 * @throws Exception
 */
@Test(timeout = 60000)
public void testDelayedDeletion() throws Exception {
    final String relationDirectory = SSTableHelper.getSSTableDir(STORAGE_DIRECTORY, TEST_RELATION);
    final File relationDirectoryFile = new File(relationDirectory);
    FileUtil.deleteRecursive(relationDirectoryFile.toPath());
    // Directory should be empty
    Assert.assertFalse(relationDirectoryFile.isDirectory());
    relationDirectoryFile.mkdirs();
    final List<Tuple> tupleList = createTupleList();
    final SSTableWriter ssTableWriter = new SSTableWriter(STORAGE_DIRECTORY, TEST_RELATION, 1, EXPECTED_TUPLES);
    ssTableWriter.open();
    ssTableWriter.addData(tupleList);
    final File sstableFile = ssTableWriter.getSstableFile();
    final File sstableIndexFile = ssTableWriter.getSstableIndexFile();
    ssTableWriter.close();
    Assert.assertTrue(sstableFile.exists());
    Assert.assertTrue(sstableIndexFile.exists());
    final ReadOnlyTupleStore ssTableFacade = new SSTableFacade(STORAGE_DIRECTORY, TEST_RELATION, 1, 0);
    ssTableFacade.acquire();
    ssTableFacade.deleteOnClose();
    // Directory should contain at least the sstable, the key index and the meta data file
    Assert.assertTrue(relationDirectoryFile.listFiles().length >= 3);
    Assert.assertTrue(sstableFile.exists());
    Assert.assertTrue(sstableIndexFile.exists());
    // After calling release, the files can be deleted
    ssTableFacade.release();
    Assert.assertFalse(sstableFile.exists());
    Assert.assertFalse(sstableIndexFile.exists());
    // Directory should be empty
    Assert.assertEquals(0, relationDirectoryFile.listFiles().length);
}
Also used : SSTableWriter(org.bboxdb.storage.sstable.SSTableWriter) SSTableFacade(org.bboxdb.storage.sstable.reader.SSTableFacade) ReadOnlyTupleStore(org.bboxdb.storage.tuplestore.ReadOnlyTupleStore) File(java.io.File) Tuple(org.bboxdb.storage.entity.Tuple) DeletedTuple(org.bboxdb.storage.entity.DeletedTuple) Test(org.junit.Test)

Example 7 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class TupleStoreManager method get.

/**
 * Search for the most recent version of the tuple
 * @param key
 * @return The tuple or null
 * @throws StorageManagerException
 */
public List<Tuple> get(final String key) throws StorageManagerException {
    if (!serviceState.isInRunningState()) {
        throw new StorageManagerException("Storage manager is not ready: " + tupleStoreName.getFullname() + " state: " + serviceState);
    }
    final Summary.Timer requestTimer = getRequestLatency.startTimer();
    final List<ReadOnlyTupleStore> aquiredStorages = new ArrayList<>();
    final List<Tuple> tupleList = new ArrayList<>();
    try {
        aquiredStorages.addAll(aquireStorage());
        for (final ReadOnlyTupleStore tupleStorage : aquiredStorages) {
            final List<Tuple> resultTuples = tupleStorage.get(key);
            tupleList.addAll(resultTuples);
        }
    } catch (Exception e) {
        throw e;
    } finally {
        releaseStorage(aquiredStorages);
        requestTimer.observeDuration();
    }
    final DuplicateResolver<Tuple> resolver = TupleDuplicateResolverFactory.build(tupleStoreConfiguration);
    resolver.removeDuplicates(tupleList);
    return tupleList;
}
Also used : ArrayList(java.util.ArrayList) Summary(io.prometheus.client.Summary) StorageManagerException(org.bboxdb.storage.StorageManagerException) ReadOnlyTupleStore(org.bboxdb.storage.tuplestore.ReadOnlyTupleStore) Tuple(org.bboxdb.storage.entity.Tuple) RejectedException(org.bboxdb.commons.RejectedException) ZookeeperNotFoundException(org.bboxdb.distribution.zookeeper.ZookeeperNotFoundException) ZookeeperException(org.bboxdb.distribution.zookeeper.ZookeeperException) StorageManagerException(org.bboxdb.storage.StorageManagerException) IOException(java.io.IOException) BBoxDBException(org.bboxdb.misc.BBoxDBException)

Example 8 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class TupleStoreManager method aquireStorage.

/**
 * Try to acquire all needed tables
 * @return
 * @throws StorageManagerException
 */
public List<ReadOnlyTupleStore> aquireStorage() throws StorageManagerException {
    for (int execution = 0; execution < Const.OPERATION_RETRY; execution++) {
        final List<ReadOnlyTupleStore> aquiredStorages = new ArrayList<>();
        final List<ReadOnlyTupleStore> knownStorages = tupleStoreInstances.getAllTupleStorages();
        for (final ReadOnlyTupleStore tupleStorage : knownStorages) {
            final boolean canBeUsed = tupleStorage.acquire();
            if (!canBeUsed) {
                if (execution == Const.OPERATION_RETRY - 1) {
                    logger.error("Unable to aquire: {} with {} retries", tupleStorage, execution);
                }
                break;
            } else {
                aquiredStorages.add(tupleStorage);
            }
        }
        if (knownStorages.size() == aquiredStorages.size()) {
            return aquiredStorages;
        } else {
            // one or more storages could not be acquired
            // release storages and retry
            releaseStorage(aquiredStorages);
            try {
                // Wait some time and try again
                Thread.sleep(10);
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                break;
            }
        }
    }
    throw new StorageManagerException("Unable to aquire all sstables in " + Const.OPERATION_RETRY + " retries");
}
Also used : ArrayList(java.util.ArrayList) StorageManagerException(org.bboxdb.storage.StorageManagerException) ReadOnlyTupleStore(org.bboxdb.storage.tuplestore.ReadOnlyTupleStore)

Example 9 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class SSTableCheckpointRunnable method isCheckpointNeeded.

/**
 * Decide if a new checkpoint is needed
 * @return
 */
protected boolean isCheckpointNeeded(final TupleStoreManager ssTableManager) {
    final List<ReadOnlyTupleStore> inMemoryStores = ssTableManager.getAllInMemoryStorages();
    if (inMemoryStores.isEmpty()) {
        return false;
    }
    final long currentTime = System.currentTimeMillis();
    // The checkpoint predicate
    final LongPredicate checkpointPredicate = m -> {
        final long checkpointThreshold = TimeUnit.MICROSECONDS.toMillis(m) + maxUncheckpointedMiliseconds;
        return checkpointThreshold < currentTime;
    };
    final boolean checkpointNeeded = inMemoryStores.stream().filter(Objects::nonNull).mapToLong(m -> m.getOldestTupleVersionTimestamp()).anyMatch(checkpointPredicate);
    logger.debug("Checkpoint for {} needed {}", ssTableManager.getTupleStoreName().getFullname(), checkpointNeeded);
    return checkpointNeeded;
}
Also used : Logger(org.slf4j.Logger) ExceptionSafeRunnable(org.bboxdb.commons.concurrent.ExceptionSafeRunnable) LoggerFactory(org.slf4j.LoggerFactory) Const(org.bboxdb.misc.Const) ReadOnlyTupleStore(org.bboxdb.storage.tuplestore.ReadOnlyTupleStore) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) TupleStoreName(org.bboxdb.storage.entity.TupleStoreName) TupleStoreManagerRegistry(org.bboxdb.storage.tuplestore.manager.TupleStoreManagerRegistry) SystemInfo(org.bboxdb.commons.SystemInfo) List(java.util.List) TupleStoreManager(org.bboxdb.storage.tuplestore.manager.TupleStoreManager) LongPredicate(java.util.function.LongPredicate) StorageManagerException(org.bboxdb.storage.StorageManagerException) DiskStorage(org.bboxdb.storage.tuplestore.DiskStorage) Objects(java.util.Objects) ReadOnlyTupleStore(org.bboxdb.storage.tuplestore.ReadOnlyTupleStore) LongPredicate(java.util.function.LongPredicate)

Example 10 with ReadOnlyTupleStore

use of org.bboxdb.storage.tuplestore.ReadOnlyTupleStore in project bboxdb by jnidzwetzki.

the class RegionMerger method mergeDataByLocalRead.

/**
 * Merge data by local data read
 *
 * @param region
 * @param tupleStoreName
 * @param tupleRedistributor
 * @param childRegion
 * @throws StorageManagerException
 * @throws TupleStoreManagerRegistry
 */
private void mergeDataByLocalRead(final DistributionRegion region, final TupleStoreName tupleStoreName, final TupleRedistributor tupleRedistributor, final DistributionRegion childRegion) throws StorageManagerException {
    final long regionId = region.getRegionId();
    final TupleStoreName childRegionName = tupleStoreName.cloneWithDifferntRegionId(regionId);
    final TupleStoreManager tupleStoreManager = registry.getTupleStoreManager(childRegionName);
    final List<ReadOnlyTupleStore> storages = new ArrayList<>();
    try {
        storages.addAll(tupleStoreManager.aquireStorage());
        for (final ReadOnlyTupleStore storage : storages) {
            for (final Tuple tuple : storage) {
                tupleRedistributor.redistributeTuple(tuple);
            }
        }
    } catch (Exception e) {
        throw e;
    } finally {
        tupleStoreManager.releaseStorage(storages);
    }
}
Also used : ArrayList(java.util.ArrayList) TupleStoreName(org.bboxdb.storage.entity.TupleStoreName) ReadOnlyTupleStore(org.bboxdb.storage.tuplestore.ReadOnlyTupleStore) Tuple(org.bboxdb.storage.entity.Tuple) StorageManagerException(org.bboxdb.storage.StorageManagerException) BBoxDBException(org.bboxdb.misc.BBoxDBException) TupleStoreManager(org.bboxdb.storage.tuplestore.manager.TupleStoreManager)

Aggregations

ReadOnlyTupleStore (org.bboxdb.storage.tuplestore.ReadOnlyTupleStore)11 ArrayList (java.util.ArrayList)9 StorageManagerException (org.bboxdb.storage.StorageManagerException)7 BBoxDBException (org.bboxdb.misc.BBoxDBException)6 Tuple (org.bboxdb.storage.entity.Tuple)6 TupleStoreName (org.bboxdb.storage.entity.TupleStoreName)5 TupleStoreManager (org.bboxdb.storage.tuplestore.manager.TupleStoreManager)5 RejectedException (org.bboxdb.commons.RejectedException)3 BoundingBox (org.bboxdb.commons.math.BoundingBox)3 TupleStoreManagerRegistry (org.bboxdb.storage.tuplestore.manager.TupleStoreManagerRegistry)3 File (java.io.File)2 IOException (java.io.IOException)2 List (java.util.List)2 ZookeeperException (org.bboxdb.distribution.zookeeper.ZookeeperException)2 ZookeeperNotFoundException (org.bboxdb.distribution.zookeeper.ZookeeperNotFoundException)2 DeletedTuple (org.bboxdb.storage.entity.DeletedTuple)2 SSTableWriter (org.bboxdb.storage.sstable.SSTableWriter)2 SSTableFacade (org.bboxdb.storage.sstable.reader.SSTableFacade)2 DiskStorage (org.bboxdb.storage.tuplestore.DiskStorage)2 Test (org.junit.Test)2