Examples with TupleStoreManager org.bboxdb.storage.tuplestore.manager.TupleStoreManager used on opensource projects

Example 26 with TupleStoreManager

use of org.bboxdb.storage.tuplestore.manager.TupleStoreManager in project bboxdb by jnidzwetzki.

the class TestTableCompactor method testCompactTestFileCreation.

@Test(timeout = 60000)
public void testCompactTestFileCreation() throws StorageManagerException {
    final List<Tuple> tupleList1 = new ArrayList<Tuple>();
    tupleList1.add(new Tuple("1", BoundingBox.FULL_SPACE, "abc".getBytes()));
    final SSTableKeyIndexReader reader1 = addTuplesToFileAndGetReader(tupleList1, 1);
    final List<Tuple> tupleList2 = new ArrayList<Tuple>();
    tupleList2.add(new Tuple("2", BoundingBox.FULL_SPACE, "def".getBytes()));
    final SSTableKeyIndexReader reader2 = addTuplesToFileAndGetReader(tupleList2, 2);
    storageRegistry.deleteTable(TEST_RELATION, true);
    storageRegistry.createTable(TEST_RELATION, new TupleStoreConfiguration());
    final TupleStoreManager storageManager = storageRegistry.getTupleStoreManager(TEST_RELATION);
    final SSTableCompactor compactor = new SSTableCompactor(storageManager, Arrays.asList(reader1, reader2));
    compactor.executeCompactation();
    final List<SSTableWriter> resultWriter = compactor.getResultList();
    Assert.assertEquals(1, resultWriter.size());
    Assert.assertEquals(2, compactor.getReadTuples());
    Assert.assertEquals(2, compactor.getWrittenTuples());
    for (final SSTableWriter writer : resultWriter) {
        Assert.assertTrue(writer.getSstableFile().exists());
        Assert.assertTrue(writer.getSstableIndexFile().exists());
        writer.close();
    }
}

Example 27 with TupleStoreManager

use of org.bboxdb.storage.tuplestore.manager.TupleStoreManager in project bboxdb by jnidzwetzki.

the class InsertTupleHandler method processInsertPackage.

/**
 * Insert the table into the local storage
 * @param tuple
 * @param requestTable
 * @param storageRegistry
 * @param routingHeader
 * @throws StorageManagerException
 * @throws RejectedException
 * @throws BBoxDBException
 */
protected void processInsertPackage(final Tuple tuple, final TupleStoreName requestTable, final TupleStoreManagerRegistry storageRegistry, final List<Long> distributionRegions) throws RejectedException {
    try {
        final String fullname = requestTable.getDistributionGroup();
        final SpacePartitioner spacePartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(fullname);
        final DistributionRegionIdMapper regionIdMapper = spacePartitioner.getDistributionRegionIdMapper();
        final Collection<TupleStoreName> localTables = regionIdMapper.convertRegionIdToTableNames(requestTable, distributionRegions);
        if (localTables.isEmpty()) {
            throw new BBoxDBException("Got no local tables for routed package");
        }
        // Are some tables unknown and needs to be created?
        final boolean unknownTables = localTables.stream().anyMatch((t) -> !storageRegistry.isStorageManagerKnown(t));
        // Expensive call (involves Zookeeper interaction)
        if (unknownTables) {
            createMissingTables(requestTable, storageRegistry, localTables);
        }
        // Insert tuples
        for (final TupleStoreName tupleStoreName : localTables) {
            final TupleStoreManager storageManager = storageRegistry.getTupleStoreManager(tupleStoreName);
            storageManager.put(tuple);
        }
    } catch (RejectedException e) {
        throw e;
    } catch (Throwable e) {
        throw new RejectedException(e);
    }
}

Example 28 with TupleStoreManager

use of org.bboxdb.storage.tuplestore.manager.TupleStoreManager in project bboxdb by jnidzwetzki.

the class SSTableCheckpointRunnable method isCheckpointNeeded.

/**
 * Decide if a new checkpoint is needed
 * @return
 */
protected boolean isCheckpointNeeded(final TupleStoreManager ssTableManager) {
    final List<ReadOnlyTupleStore> inMemoryStores = ssTableManager.getAllInMemoryStorages();
    if (inMemoryStores.isEmpty()) {
        return false;
    }
    final long currentTime = System.currentTimeMillis();
    // The checkpoint predicate
    final LongPredicate checkpointPredicate = m -> {
        final long checkpointThreshold = TimeUnit.MICROSECONDS.toMillis(m) + maxUncheckpointedMiliseconds;
        return checkpointThreshold < currentTime;
    };
    final boolean checkpointNeeded = inMemoryStores.stream().filter(Objects::nonNull).mapToLong(m -> m.getOldestTupleVersionTimestamp()).anyMatch(checkpointPredicate);
    logger.debug("Checkpoint for {} needed {}", ssTableManager.getTupleStoreName().getFullname(), checkpointNeeded);
    return checkpointNeeded;
}

Example 29 with TupleStoreManager

use of org.bboxdb.storage.tuplestore.manager.TupleStoreManager in project bboxdb by jnidzwetzki.

the class DistributedRecoveryService method runRecoveryForTable.

/**
 * Run the recovery for a given table
 * @param ssTableName
 * @param outdatedDistributionRegion
 * @param connection
 * @throws StorageManagerException
 * @throws InterruptedException
 * @throws ExecutionException
 * @throws RejectedException
 */
protected void runRecoveryForTable(final TupleStoreName ssTableName, final OutdatedDistributionRegion outdatedDistributionRegion, final BBoxDBClient connection) throws StorageManagerException, InterruptedException, ExecutionException, RejectedException {
    final String sstableName = ssTableName.getFullname();
    logger.info("Recovery: starting recovery for table {}", sstableName);
    final TupleStoreManager tableManager = storageRegistry.getTupleStoreManager(ssTableName);
    // Even with NTP, the clock of the nodes can have a delta.
    // We subtract this delta from the checkpoint timestamp to ensure
    // that all tuples for the recovery are requested
    final long requestTupleTimestamp = outdatedDistributionRegion.getLocalVersion() - Const.MAX_NODE_CLOCK_DELTA;
    final TupleListFuture result = connection.queryInsertedTime(sstableName, requestTupleTimestamp);
    result.waitForAll();
    if (result.isFailed()) {
        logger.warn("Recovery: Failed result for table {} - Some tuples could not be received!", sstableName);
        return;
    }
    long insertedTuples = 0;
    for (final Tuple tuple : result) {
        tableManager.put(tuple);
        insertedTuples++;
    }
    logger.info("Recovery: successfully inserted {} tuples into table {}", insertedTuples, sstableName);
}

Example 30 with TupleStoreManager

use of org.bboxdb.storage.tuplestore.manager.TupleStoreManager in project bboxdb by jnidzwetzki.

the class RegionMerger method mergeDataByLocalRead.

/**
 * Merge data by local data read
 *
 * @param region
 * @param tupleStoreName
 * @param tupleRedistributor
 * @param childRegion
 * @throws StorageManagerException
 * @throws TupleStoreManagerRegistry
 */
private void mergeDataByLocalRead(final DistributionRegion region, final TupleStoreName tupleStoreName, final TupleRedistributor tupleRedistributor, final DistributionRegion childRegion) throws StorageManagerException {
    final long regionId = region.getRegionId();
    final TupleStoreName childRegionName = tupleStoreName.cloneWithDifferntRegionId(regionId);
    final TupleStoreManager tupleStoreManager = registry.getTupleStoreManager(childRegionName);
    final List<ReadOnlyTupleStore> storages = new ArrayList<>();
    try {
        storages.addAll(tupleStoreManager.aquireStorage());
        for (final ReadOnlyTupleStore storage : storages) {
            for (final Tuple tuple : storage) {
                tupleRedistributor.redistributeTuple(tuple);
            }
        }
    } catch (Exception e) {
        throw e;
    } finally {
        tupleStoreManager.releaseStorage(storages);
    }
}