Examples with Distribution com.facebook.presto.raptor.metadata.Distribution used on opensource projects

Example 1 with Distribution

use of com.facebook.presto.raptor.metadata.Distribution in project presto by prestodb.

the class RaptorMetadata method getOrCreateDistribution.

private Optional<DistributionInfo> getOrCreateDistribution(Map<String, RaptorColumnHandle> columnHandleMap, Map<String, Object> properties) {
    OptionalInt bucketCount = getBucketCount(properties);
    List<RaptorColumnHandle> bucketColumnHandles = getBucketColumnHandles(getBucketColumns(properties), columnHandleMap);
    if (bucketCount.isPresent() && bucketColumnHandles.isEmpty()) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKETED_ON_PROPERTY, BUCKET_COUNT_PROPERTY));
    }
    if (!bucketCount.isPresent() && !bucketColumnHandles.isEmpty()) {
        throw new PrestoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKET_COUNT_PROPERTY, BUCKETED_ON_PROPERTY));
    }
    ImmutableList.Builder<Type> bucketColumnTypes = ImmutableList.builder();
    for (RaptorColumnHandle column : bucketColumnHandles) {
        validateBucketType(column.getColumnType());
        bucketColumnTypes.add(column.getColumnType());
    }
    long distributionId;
    String distributionName = getDistributionName(properties);
    if (distributionName != null) {
        if (bucketColumnHandles.isEmpty()) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, format("Must specify '%s' along with '%s'", BUCKETED_ON_PROPERTY, DISTRIBUTION_NAME_PROPERTY));
        }
        Distribution distribution = dao.getDistribution(distributionName);
        if (distribution == null) {
            if (!bucketCount.isPresent()) {
                throw new PrestoException(INVALID_TABLE_PROPERTY, "Distribution does not exist and bucket count is not specified");
            }
            distribution = getOrCreateDistribution(distributionName, bucketColumnTypes.build(), bucketCount.getAsInt());
        }
        distributionId = distribution.getId();
        if (bucketCount.isPresent() && (distribution.getBucketCount() != bucketCount.getAsInt())) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, "Bucket count must match distribution");
        }
        if (!distribution.getColumnTypes().equals(bucketColumnTypes.build())) {
            throw new PrestoException(INVALID_TABLE_PROPERTY, "Bucket column types must match distribution");
        }
    } else if (bucketCount.isPresent()) {
        String types = Distribution.serializeColumnTypes(bucketColumnTypes.build());
        distributionId = dao.insertDistribution(null, types, bucketCount.getAsInt());
    } else {
        return Optional.empty();
    }
    shardManager.createBuckets(distributionId, bucketCount.getAsInt());
    return Optional.of(new DistributionInfo(distributionId, bucketCount.getAsInt(), bucketColumnHandles));
}

Example 2 with Distribution

use of com.facebook.presto.raptor.metadata.Distribution in project presto by prestodb.

the class RaptorMetadata method getDistributionInfo.

private DistributionInfo getDistributionInfo(long distributionId, Map<String, RaptorColumnHandle> columnHandleMap, Map<String, Object> properties) {
    Distribution distribution = dao.getDistribution(distributionId);
    if (distribution == null) {
        throw new PrestoException(RAPTOR_ERROR, "Distribution ID does not exist: " + distributionId);
    }
    List<RaptorColumnHandle> bucketColumnHandles = getBucketColumnHandles(getBucketColumns(properties), columnHandleMap);
    return new DistributionInfo(distributionId, distribution.getBucketCount(), bucketColumnHandles);
}

Example 3 with Distribution

use of com.facebook.presto.raptor.metadata.Distribution in project presto by prestodb.

the class RaptorMetadata method getOrCreateDistribution.

private Distribution getOrCreateDistribution(String name, List<Type> columnTypes, int bucketCount) {
    String types = Distribution.serializeColumnTypes(columnTypes);
    runIgnoringConstraintViolation(() -> dao.insertDistribution(name, types, bucketCount));
    Distribution distribution = dao.getDistribution(name);
    if (distribution == null) {
        throw new PrestoException(RAPTOR_ERROR, "Distribution does not exist after insert");
    }
    return distribution;
}

Example 4 with Distribution

use of com.facebook.presto.raptor.metadata.Distribution in project presto by prestodb.

the class TestBucketBalancer method assertBalancing.

private static void assertBalancing(BucketBalancer balancer, int expectedMoves) {
    int actualMoves = balancer.balance();
    assertEquals(actualMoves, expectedMoves);
    // check that number of buckets per node is within bounds
    ClusterState clusterState = balancer.fetchClusterState();
    for (Distribution distribution : clusterState.getDistributionAssignments().keySet()) {
        Multiset<String> allocationCounts = HashMultiset.create();
        clusterState.getDistributionAssignments().get(distribution).stream().map(BucketAssignment::getNodeIdentifier).forEach(allocationCounts::add);
        double bucketsPerNode = (1.0 * allocationCounts.size()) / clusterState.getActiveNodes().size();
        for (String node : allocationCounts) {
            assertGreaterThanOrEqual(allocationCounts.count(node), (int) Math.floor(bucketsPerNode), node + " has fewer buckets than expected");
            assertLessThanOrEqual(allocationCounts.count(node), (int) Math.ceil(bucketsPerNode), node + " has more buckets than expected");
        }
    }
    // check stability
    assertEquals(balancer.balance(), 0);
}

Example 5 with Distribution

use of com.facebook.presto.raptor.metadata.Distribution in project presto by prestodb.

the class BucketBalancer method fetchClusterState.

@VisibleForTesting
ClusterState fetchClusterState() {
    Set<String> activeNodes = nodeSupplier.getWorkerNodes().stream().map(Node::getNodeIdentifier).collect(toSet());
    Map<String, Long> assignedNodeSize = new HashMap<>(activeNodes.stream().collect(toMap(node -> node, node -> 0L)));
    ImmutableMultimap.Builder<Distribution, BucketAssignment> distributionAssignments = ImmutableMultimap.builder();
    ImmutableMap.Builder<Distribution, Long> distributionBucketSize = ImmutableMap.builder();
    for (Distribution distribution : shardManager.getDistributions()) {
        long distributionSize = shardManager.getDistributionSizeInBytes(distribution.getId());
        long bucketSize = (long) (1.0 * distributionSize) / distribution.getBucketCount();
        distributionBucketSize.put(distribution, bucketSize);
        for (BucketNode bucketNode : shardManager.getBucketNodes(distribution.getId())) {
            String node = bucketNode.getNodeIdentifier();
            distributionAssignments.put(distribution, new BucketAssignment(distribution.getId(), bucketNode.getBucketNumber(), node));
            assignedNodeSize.merge(node, bucketSize, Math::addExact);
        }
    }
    return new ClusterState(activeNodes, assignedNodeSize, distributionAssignments.build(), distributionBucketSize.build());
}