Examples with DistributionRegionAdapter org.bboxdb.distribution.zookeeper.DistributionRegionAdapter used on opensource projects

Example 1 with DistributionRegionAdapter

use of org.bboxdb.distribution.zookeeper.DistributionRegionAdapter in project bboxdb by jnidzwetzki.

the class TestKDtreeSpacePartitioner method testDistributionRegionSplitAndMerge.

/**
 * Test the split of a distribution region
 * @throws Exception
 */
@Test(timeout = 60000)
public void testDistributionRegionSplitAndMerge() throws Exception {
    final DistributionRegionAdapter distributionRegionAdapter = ZookeeperClientFactory.getZookeeperClient().getDistributionRegionAdapter();
    // Split and update
    System.out.println("---> Get space partitioner");
    final KDtreeSpacePartitioner spacePartitioner = (KDtreeSpacePartitioner) getSpacePartitioner();
    System.out.println("---> Get space partitioner - DONE");
    final DistributionRegion rootNode = spacePartitioner.getRootNode();
    System.out.println("---> Get root node - DONE");
    Assert.assertEquals(0, rootNode.getRegionId());
    Assert.assertEquals(TEST_GROUP, rootNode.getDistributionGroupName());
    final DistributionRegionState stateForDistributionRegion1 = distributionRegionAdapter.getStateForDistributionRegion(rootNode);
    Assert.assertEquals(DistributionRegionState.ACTIVE, stateForDistributionRegion1);
    System.out.println("--> Set split for node");
    spacePartitioner.splitNode(rootNode, 10);
    spacePartitioner.waitForSplitCompleteZookeeperCallback(rootNode, 2);
    final DistributionRegion firstChild = rootNode.getDirectChildren().get(0);
    Assert.assertEquals(10.0, firstChild.getConveringBox().getCoordinateHigh(0), DELTA);
    final DistributionRegionState stateForDistributionRegion2 = distributionRegionAdapter.getStateForDistributionRegion(rootNode);
    Assert.assertEquals(DistributionRegionState.SPLITTING, stateForDistributionRegion2);
    // Reread group from zookeeper
    final DistributionRegion newDistributionGroup = spacePartitioner.getRootNode();
    final DistributionRegion firstChildNew = newDistributionGroup.getDirectChildren().get(0);
    Assert.assertEquals(10.0, firstChildNew.getConveringBox().getCoordinateHigh(0), DELTA);
    final DistributionRegionState stateForDistributionRegion3 = distributionRegionAdapter.getStateForDistributionRegion(newDistributionGroup);
    Assert.assertEquals(DistributionRegionState.SPLITTING, stateForDistributionRegion3);
    Assert.assertEquals(1, rootNode.getDirectChildren().get(0).getRegionId());
    Assert.assertEquals(2, rootNode.getDirectChildren().get(1).getRegionId());
    // Delete children
    System.out.println("---> Calling prepare merge");
    spacePartitioner.prepareMerge(spacePartitioner.getRootNode().getDirectChildren(), spacePartitioner.getRootNode());
    System.out.println("---> Calling merge complete");
    spacePartitioner.mergeComplete(spacePartitioner.getRootNode().getDirectChildren(), spacePartitioner.getRootNode());
    final DistributionRegion newDistributionGroup2 = spacePartitioner.getRootNode();
    final DistributionRegionState stateForDistributionRegion4 = distributionRegionAdapter.getStateForDistributionRegion(newDistributionGroup2);
    Assert.assertEquals(DistributionRegionState.ACTIVE, stateForDistributionRegion4);
}

Example 2 with DistributionRegionAdapter

use of org.bboxdb.distribution.zookeeper.DistributionRegionAdapter in project bboxdb by jnidzwetzki.

the class RegionMerger method mergeRegion.

/**
 * Merge the given region
 * @param region
 * @param distributionGroupZookeeperAdapter
 * @param spacePartitioner
 * @param diskStorage
 */
public void mergeRegion(final List<DistributionRegion> source, final SpacePartitioner spacePartitioner, final TupleStoreManagerRegistry tupleStoreManagerRegistry) {
    assert (!source.isEmpty());
    logger.info("Performing merge for: {}", source.get(0).getIdentifier());
    final DistributionRegionAdapter distributionGroupZookeeperAdapter = ZookeeperClientFactory.getZookeeperClient().getDistributionRegionAdapter();
    DistributionRegion destination = null;
    try {
        destination = spacePartitioner.getDestinationForMerge(source);
        if (destination == null) {
            logger.error("Got null when calling getDestinationForMerge from space partitoner {}", source);
            return;
        }
        final String identifier = destination.getIdentifier();
        // Try to set region state to merging. If this fails, another node is already
        // merges the region
        final boolean setToMergeResult = distributionGroupZookeeperAdapter.setToSplitMerging(destination);
        if (!setToMergeResult) {
            logger.info("Unable to set state to split merge for region: {}, stopping merge", identifier);
            logger.info("Old state was {}", distributionGroupZookeeperAdapter.getStateForDistributionRegion(destination));
            return;
        }
        spacePartitioner.prepareMerge(source, destination);
        redistributeDataMerge(source, destination);
        spacePartitioner.mergeComplete(source, destination);
    } catch (Throwable e) {
        logger.warn("Got uncought exception during merge: " + source.get(0).getIdentifier(), e);
        handleMergeFailed(source, destination, spacePartitioner);
    }
}

Example 3 with DistributionRegionAdapter

use of org.bboxdb.distribution.zookeeper.DistributionRegionAdapter in project bboxdb by jnidzwetzki.

the class TupleStoreFlushZookeeperAdapter method accept.

@Override
public void accept(final TupleStoreName ssTableName, final Long flushTimestamp) {
    // Fetch the local instance
    final BBoxDBInstance localInstance = ZookeeperClientFactory.getLocalInstanceName();
    try {
        final String distributionGroup = ssTableName.getDistributionGroup();
        final SpacePartitioner spacepartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroup);
        final DistributionRegion distributionGroupRoot = spacepartitioner.getRootNode();
        if (distributionGroupRoot == null) {
            logger.error("Root for {} is null, not executing callback", ssTableName);
            return;
        }
        final long regionId = ssTableName.getRegionId().getAsLong();
        final DistributionRegion distributionRegion = DistributionRegionHelper.getDistributionRegionForNamePrefix(distributionGroupRoot, regionId);
        logger.debug("Updating checkpoint for: {} to {}", ssTableName.getFullname(), flushTimestamp);
        final DistributionRegionAdapter distributionRegionAdapter = ZookeeperClientFactory.getZookeeperClient().getDistributionRegionAdapter();
        if (distributionRegionAdapter != null && distributionRegion != null) {
            distributionRegionAdapter.setCheckpointForDistributionRegion(distributionRegion, localInstance, flushTimestamp);
        }
    } catch (ZookeeperException | BBoxDBException e) {
        if (Thread.currentThread().isInterrupted()) {
            return;
        }
        logger.warn("Unable to find distribution region: ", e);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        return;
    }
}

Example 4 with DistributionRegionAdapter

use of org.bboxdb.distribution.zookeeper.DistributionRegionAdapter in project bboxdb by jnidzwetzki.

the class DistributionRegionComponent method addStatisticsToTooltip.

/**
 * Add the statistics to the tooltip
 * @param sb
 * @return
 * @throws ZookeeperException
 * @throws ZookeeperNotFoundException
 */
private Map<BBoxDBInstance, Map<String, Long>> addStatisticsToTooltip(final StringBuilder sb) throws ZookeeperException, ZookeeperNotFoundException {
    final DistributionRegionAdapter adapter = ZookeeperClientFactory.getZookeeperClient().getDistributionRegionAdapter();
    final Map<BBoxDBInstance, Map<String, Long>> statistics = adapter.getRegionStatistics(distributionRegion);
    for (final BBoxDBInstance instance : statistics.keySet()) {
        final Map<String, Long> statisticData = statistics.get(instance);
        sb.append("Node: ");
        sb.append(instance.toGUIString(guiModel.isScreenshotMode()));
        sb.append(" Tuples: ");
        sb.append(statisticData.get(ZookeeperNodeNames.NAME_STATISTICS_TOTAL_TUPLES));
        sb.append(", Size: ");
        sb.append(statisticData.get(ZookeeperNodeNames.NAME_STATISTICS_TOTAL_SIZE));
        sb.append(" MB <br>");
    }
    return statistics;
}

Example 5 with DistributionRegionAdapter

use of org.bboxdb.distribution.zookeeper.DistributionRegionAdapter in project bboxdb by jnidzwetzki.

the class CreateInitialPartitioning method run.

@Override
public void run() {
    final TupleFileReader tupleFile = new TupleFileReader(filename, format);
    final List<BoundingBox> samples = new ArrayList<>();
    tupleFile.addTupleListener(t -> {
        final BoundingBox polygonBoundingBox = t.getBoundingBox();
        samples.add(polygonBoundingBox);
    });
    try {
        tupleFile.processFile();
        final SpacePartitioner spacePartitioner = SpacePartitionerCache.getInstance().getSpacePartitionerForGroupName(distributionGroup);
        final DistributionRegionAdapter adapter = ZookeeperClientFactory.getZookeeperClient().getDistributionRegionAdapter();
        while (getActiveRegions(spacePartitioner).size() < partitions) {
            logger.info("We have now {} of {} active partitons, executing split", getActiveRegions(spacePartitioner).size() < partitions);
            final List<DistributionRegion> activeRegions = getActiveRegions(spacePartitioner);
            final DistributionRegion regionToSplit = ListHelper.getElementRandom(activeRegions);
            logger.info("Splitting region {}", regionToSplit.getRegionId());
            final List<DistributionRegion> destination = spacePartitioner.splitRegion(regionToSplit, samples);
            spacePartitioner.splitComplete(regionToSplit, destination);
        }
        // Prevent merging of nodes
        for (DistributionRegion region : spacePartitioner.getRootNode().getAllChildren()) {
            adapter.setMergingSupported(region, false);
        }
    } catch (Exception e) {
        logger.error("Got an exception", e);
        System.exit(-1);
    }
}