Examples with SpacePartitioner org.bboxdb.distribution.partitioner.SpacePartitioner used on opensource projects

Example 11 with SpacePartitioner

use of org.bboxdb.distribution.partitioner.SpacePartitioner in project bboxdb by jnidzwetzki.

the class RegionMerger method redistributeDataMerge.

/**
 * Redistribute the data in region merge
 * @param source
 * @param destination
 * @throws Exception
 */
private void redistributeDataMerge(final List<DistributionRegion> source, final DistributionRegion destination) throws Exception {
    logger.info("Redistributing all data for region (merge): " + destination.getIdentifier());
    final String distributionGroupName = destination.getDistributionGroupName();
    final List<TupleStoreName> localTables = TupleStoreUtil.getAllTablesForDistributionGroupAndRegionId(registry, distributionGroupName, destination.getRegionId());
    // Add the local mapping, new data is written to the region
    final SpacePartitioner spacePartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroupName);
    final DistributionRegionIdMapper mapper = spacePartitioner.getDistributionRegionIdMapper();
    // We have set the region to active, wait until we see this status change
    // from Zookeeper and the space partitioner add this region as active
    mapper.waitUntilMappingAppears(destination.getRegionId());
    // Redistribute data
    for (final TupleStoreName tupleStoreName : localTables) {
        logger.info("Merging data of tuple store {}", tupleStoreName);
        startFlushToDisk(tupleStoreName);
        final TupleRedistributor tupleRedistributor = new TupleRedistributor(registry, tupleStoreName);
        tupleRedistributor.registerRegion(destination);
        for (final DistributionRegion childRegion : source) {
            mergeDataFromChildRegion(destination, tupleStoreName, tupleRedistributor, childRegion);
        }
        logger.info("Final statistics for merge ({}): {}", tupleStoreName, tupleRedistributor.getStatistics());
    }
}

Example 12 with SpacePartitioner

use of org.bboxdb.distribution.partitioner.SpacePartitioner in project bboxdb by jnidzwetzki.

the class RegionSplitHelper method isSplittingSupported.

/**
 * Is the splitting of the region supported?
 * @param regionToSplit
 * @return
 */
public static boolean isSplittingSupported(final DistributionRegion region) {
    try {
        final String distributionGroupName = region.getDistributionGroupName();
        final SpacePartitioner spacePartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroupName);
        return spacePartitioner.isSplitable(region);
    } catch (BBoxDBException e) {
        logger.error("Got exception while testing for merge", e);
        return false;
    }
}

Example 13 with SpacePartitioner

use of org.bboxdb.distribution.partitioner.SpacePartitioner in project bboxdb by jnidzwetzki.

the class RegionSplitter method redistributeDataSplit.

/**
 * Redistribute data after region split
 * @param region
 */
private void redistributeDataSplit(final DistributionRegion source, final List<DistributionRegion> destination) {
    final long regionId = source.getRegionId();
    try {
        logger.info("Redistributing all data for region: {}", regionId);
        final String distributionGroupName = source.getDistributionGroupName();
        final List<TupleStoreName> localTables = TupleStoreUtil.getAllTablesForDistributionGroupAndRegionId(registry, distributionGroupName, regionId);
        // Remove the local mapping, no new data is written to the region
        final SpacePartitioner spacePartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroupName);
        final DistributionRegionIdMapper mapper = spacePartitioner.getDistributionRegionIdMapper();
        // We have set the region to splitting, wait until we see this status change
        // from Zookeeper and the space partitioner removed this region as active
        mapper.waitUntilMappingDisappears(regionId);
        // Redistribute data
        for (final TupleStoreName ssTableName : localTables) {
            // Reject new writes and flush to disk
            stopFlushToDisk(ssTableName);
            distributeData(ssTableName, source, destination);
        }
    } catch (InterruptedException e) {
        logger.warn("Thread was interrupted");
        Thread.currentThread().interrupt();
        return;
    } catch (Exception e) {
        logger.error("Got exception when redistribute local data", e);
        return;
    }
    logger.info("Redistributing data for region: {} DONE", regionId);
}

Example 14 with SpacePartitioner

use of org.bboxdb.distribution.partitioner.SpacePartitioner in project bboxdb by jnidzwetzki.

the class AbstractUtilizationPlacementStrategy method calculateSystemUsage.

/**
 * Calculate the usage of each system
 *
 * @return
 * @throws ZookeeperException
 * @throws ZookeeperNotFoundException
 * @throws BBoxDBException
 */
protected Multiset<BBoxDBInstance> calculateSystemUsage() throws ZookeeperException, ZookeeperNotFoundException, BBoxDBException {
    final DistributionGroupAdapter zookeeperAdapter = ZookeeperClientFactory.getZookeeperClient().getDistributionGroupAdapter();
    final List<String> distributionGroups = zookeeperAdapter.getDistributionGroups();
    // The overall usage
    final ImmutableMultiset.Builder<BBoxDBInstance> builder = ImmutableMultiset.builder();
    // Calculate usage for each distribution group
    for (final String groupName : distributionGroups) {
        final SpacePartitioner spacepartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(groupName);
        final DistributionRegion region = spacepartitioner.getRootNode();
        final Multiset<BBoxDBInstance> regionSystemUsage = DistributionRegionHelper.getSystemUtilization(region);
        // Merge results
        builder.addAll(regionSystemUsage);
    }
    return builder.build();
}

Example 15 with SpacePartitioner

use of org.bboxdb.distribution.partitioner.SpacePartitioner in project bboxdb by jnidzwetzki.

the class TupleStoreFlushZookeeperAdapter method accept.

@Override
public void accept(final TupleStoreName ssTableName, final Long flushTimestamp) {
    // Fetch the local instance
    final BBoxDBInstance localInstance = ZookeeperClientFactory.getLocalInstanceName();
    try {
        final String distributionGroup = ssTableName.getDistributionGroup();
        final SpacePartitioner spacepartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroup);
        final DistributionRegion distributionGroupRoot = spacepartitioner.getRootNode();
        if (distributionGroupRoot == null) {
            logger.error("Root for {} is null, not executing callback", ssTableName);
            return;
        }
        final long regionId = ssTableName.getRegionId().getAsLong();
        final DistributionRegion distributionRegion = DistributionRegionHelper.getDistributionRegionForNamePrefix(distributionGroupRoot, regionId);
        logger.debug("Updating checkpoint for: {} to {}", ssTableName.getFullname(), flushTimestamp);
        final DistributionRegionAdapter distributionRegionAdapter = ZookeeperClientFactory.getZookeeperClient().getDistributionRegionAdapter();
        if (distributionRegionAdapter != null && distributionRegion != null) {
            distributionRegionAdapter.setCheckpointForDistributionRegion(distributionRegion, localInstance, flushTimestamp);
        }
    } catch (ZookeeperException | BBoxDBException e) {
        if (Thread.currentThread().isInterrupted()) {
            return;
        }
        logger.warn("Unable to find distribution region: ", e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        return;
    }
}