Examples with TupleRedistributor org.bboxdb.distribution.partitioner.regionsplit.tuplesink.TupleRedistributor used on opensource projects

Example 6 with TupleRedistributor

use of org.bboxdb.distribution.partitioner.regionsplit.tuplesink.TupleRedistributor in project bboxdb by jnidzwetzki.

the class RegionMerger method redistributeDataMerge.

/**
 * Redistribute the data in region merge
 * @param source
 * @param destination
 * @throws Exception
 */
private void redistributeDataMerge(final List<DistributionRegion> source, final DistributionRegion destination) throws Exception {
    logger.info("Redistributing all data for region (merge): " + destination.getIdentifier());
    final String distributionGroupName = destination.getDistributionGroupName();
    final List<TupleStoreName> localTables = TupleStoreUtil.getAllTablesForDistributionGroupAndRegionId(registry, distributionGroupName, destination.getRegionId());
    // Add the local mapping, new data is written to the region
    final SpacePartitioner spacePartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroupName);
    final DistributionRegionIdMapper mapper = spacePartitioner.getDistributionRegionIdMapper();
    // We have set the region to active, wait until we see this status change
    // from Zookeeper and the space partitioner add this region as active
    mapper.waitUntilMappingAppears(destination.getRegionId());
    // Redistribute data
    for (final TupleStoreName tupleStoreName : localTables) {
        logger.info("Merging data of tuple store {}", tupleStoreName);
        startFlushToDisk(tupleStoreName);
        final TupleRedistributor tupleRedistributor = new TupleRedistributor(registry, tupleStoreName);
        tupleRedistributor.registerRegion(destination);
        for (final DistributionRegion childRegion : source) {
            mergeDataFromChildRegion(destination, tupleStoreName, tupleRedistributor, childRegion);
        }
        logger.info("Final statistics for merge ({}): {}", tupleStoreName, tupleRedistributor.getStatistics());
    }
}

Example 7 with TupleRedistributor

use of org.bboxdb.distribution.partitioner.regionsplit.tuplesink.TupleRedistributor in project bboxdb by jnidzwetzki.

the class RegionSplitter method distributeData.

/**
 * Redistribute the given sstable
 * @param ssTableName
 * @param source
 * @param destination
 * @throws StorageManagerException
 */
private void distributeData(final TupleStoreName ssTableName, final DistributionRegion source, final List<DistributionRegion> destination) throws BBoxDBException, StorageManagerException {
    logger.info("Redistributing table {}", ssTableName.getFullname());
    final TupleStoreManager ssTableManager = registry.getTupleStoreManager(ssTableName);
    // Spread data
    final TupleRedistributor tupleRedistributor = getTupleRedistributor(source, destination, ssTableName);
    spreadTupleStores(ssTableManager, tupleRedistributor);
    logger.info("Redistributing table {} is DONE", ssTableName.getFullname());
}

Example 8 with TupleRedistributor

use of org.bboxdb.distribution.partitioner.regionsplit.tuplesink.TupleRedistributor in project bboxdb by jnidzwetzki.

the class TestTupleSink method testTupleRedistribution2.

/**
 * Test the tuple redistribution
 * @throws Exception
 */
@Test(timeout = 60000)
public void testTupleRedistribution2() throws Exception {
    final DistributionRegion distributionRegion1 = new DistributionRegion(TEST_GROUP, DistributionRegion.ROOT_NODE_ROOT_POINTER, new BoundingBox(0.0, 1.0, 0.0, 1.0, 0.0, 1.0), 1);
    final DistributionRegion distributionRegion2 = new DistributionRegion(TEST_GROUP, DistributionRegion.ROOT_NODE_ROOT_POINTER, new BoundingBox(5.0, 6.0, 5.0, 6.0, 5.0, 6.0), 1);
    final TupleRedistributor tupleRedistributor = createTupleRedistributor();
    final AbstractTupleSink tupleSink1 = Mockito.mock(AbstractTupleSink.class);
    tupleRedistributor.registerRegion(distributionRegion1, Arrays.asList(tupleSink1));
    final AbstractTupleSink tupleSink2 = Mockito.mock(AbstractTupleSink.class);
    tupleRedistributor.registerRegion(distributionRegion2, Arrays.asList(tupleSink2));
    final Tuple tuple1 = new Tuple("abc", new BoundingBox(0.0, 1.0, 0.0, 1.0, 0.0, 1.0), "".getBytes());
    tupleRedistributor.redistributeTuple(tuple1);
    (Mockito.verify(tupleSink1, Mockito.times(1))).sinkTuple(Mockito.any(Tuple.class));
    (Mockito.verify(tupleSink2, Mockito.never())).sinkTuple(Mockito.any(Tuple.class));
    final Tuple tuple2 = new Tuple("abc", new BoundingBox(5.0, 6.0, 5.0, 6.0, 5.0, 6.0), "".getBytes());
    tupleRedistributor.redistributeTuple(tuple2);
    (Mockito.verify(tupleSink1, Mockito.times(1))).sinkTuple(Mockito.any(Tuple.class));
    (Mockito.verify(tupleSink2, Mockito.times(1))).sinkTuple(Mockito.any(Tuple.class));
    final Tuple tuple3 = new Tuple("abc", new BoundingBox(0.0, 6.0, 0.0, 6.0, 0.0, 6.0), "".getBytes());
    tupleRedistributor.redistributeTuple(tuple3);
    (Mockito.verify(tupleSink1, Mockito.atLeast(2))).sinkTuple(Mockito.any(Tuple.class));
    (Mockito.verify(tupleSink2, Mockito.atLeast(2))).sinkTuple(Mockito.any(Tuple.class));
    final Tuple tuple4 = new DeletedTuple("abc");
    tupleRedistributor.redistributeTuple(tuple4);
    (Mockito.verify(tupleSink1, Mockito.atLeast(3))).sinkTuple(Mockito.any(Tuple.class));
    (Mockito.verify(tupleSink2, Mockito.atLeast(3))).sinkTuple(Mockito.any(Tuple.class));
    System.out.println(tupleRedistributor.getStatistics());
}