Examples with FixedSplitSource com.facebook.presto.spi.FixedSplitSource used on opensource projects

Example 6 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class MemorySplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layoutHandle) {
    MemoryTableLayoutHandle layout = (MemoryTableLayoutHandle) layoutHandle;
    List<HostAddress> hosts = layout.getTable().getHosts();
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    for (HostAddress host : hosts) {
        for (int i = 0; i < splitsPerNode; i++) {
            splits.add(new MemorySplit(layout.getTable(), i, splitsPerNode, ImmutableList.of(host)));
        }
    }
    return new FixedSplitSource(splits.build());
}

Example 7 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class SystemSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    SystemTableLayoutHandle layoutHandle = (SystemTableLayoutHandle) layout;
    SystemTableHandle tableHandle = layoutHandle.getTable();
    TupleDomain<ColumnHandle> constraint = layoutHandle.getConstraint();
    SystemTable systemTable = tables.get(tableHandle.getSchemaTableName());
    Distribution tableDistributionMode = systemTable.getDistribution();
    if (tableDistributionMode == SINGLE_COORDINATOR) {
        HostAddress address = nodeManager.getCurrentNode().getHostAndPort();
        ConnectorSplit split = new SystemSplit(tableHandle.getConnectorId(), tableHandle, address, constraint);
        return new FixedSplitSource(ImmutableList.of(split));
    }
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    ImmutableSet.Builder<Node> nodes = ImmutableSet.builder();
    if (tableDistributionMode == ALL_COORDINATORS) {
        nodes.addAll(nodeManager.getCoordinators());
    } else if (tableDistributionMode == ALL_NODES) {
        nodes.addAll(nodeManager.getNodes(ACTIVE));
    }
    Set<Node> nodeSet = nodes.build();
    for (Node node : nodeSet) {
        splits.add(new SystemSplit(tableHandle.getConnectorId(), tableHandle, node.getHostAndPort(), constraint));
    }
    return new FixedSplitSource(splits.build());
}

Example 8 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class KafkaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    KafkaTableHandle kafkaTableHandle = convertLayout(layout).getTable();
    SimpleConsumer simpleConsumer = consumerManager.getConsumer(selectRandom(nodes));
    TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(ImmutableList.of(kafkaTableHandle.getTopicName()));
    TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) {
        for (PartitionMetadata part : metadata.partitionsMetadata()) {
            log.debug("Adding Partition %s/%s", metadata.topic(), part.partitionId());
            Broker leader = part.leader();
            if (leader == null) {
                // Leader election going on...
                log.warn("No leader for partition %s/%s found!", metadata.topic(), part.partitionId());
                continue;
            }
            HostAddress partitionLeader = HostAddress.fromParts(leader.host(), leader.port());
            SimpleConsumer leaderConsumer = consumerManager.getConsumer(partitionLeader);
            // Kafka contains a reverse list of "end - start" pairs for the splits
            long[] offsets = findAllOffsets(leaderConsumer, metadata.topic(), part.partitionId());
            for (int i = offsets.length - 1; i > 0; i--) {
                KafkaSplit split = new KafkaSplit(connectorId, metadata.topic(), kafkaTableHandle.getKeyDataFormat(), kafkaTableHandle.getMessageDataFormat(), part.partitionId(), offsets[i], offsets[i - 1], partitionLeader);
                splits.add(split);
            }
        }
    }
    return new FixedSplitSource(splits.build());
}

Example 9 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class JmxSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    JmxTableLayoutHandle jmxLayout = (JmxTableLayoutHandle) layout;
    JmxTableHandle tableHandle = jmxLayout.getTable();
    TupleDomain<ColumnHandle> predicate = jmxLayout.getConstraint();
    //TODO is there a better way to get the node column?
    Optional<JmxColumnHandle> nodeColumnHandle = tableHandle.getColumnHandles().stream().filter(jmxColumnHandle -> jmxColumnHandle.getColumnName().equals(NODE_COLUMN_NAME)).findFirst();
    checkState(nodeColumnHandle.isPresent(), "Failed to find %s column", NODE_COLUMN_NAME);
    List<ConnectorSplit> splits = nodeManager.getAllNodes().stream().filter(node -> {
        NullableValue value = NullableValue.of(createUnboundedVarcharType(), utf8Slice(node.getNodeIdentifier()));
        return predicate.overlaps(fromFixedValues(ImmutableMap.of(nodeColumnHandle.get(), value)));
    }).map(node -> new JmxSplit(tableHandle, ImmutableList.of(node.getHostAndPort()))).collect(toList());
    return new FixedSplitSource(splits);
}

Example 10 with FixedSplitSource

use of com.facebook.presto.spi.FixedSplitSource in project presto by prestodb.

the class TpchSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    TpchTableHandle tableHandle = ((TpchTableLayoutHandle) layout).getTable();
    Set<Node> nodes = nodeManager.getRequiredWorkerNodes();
    int totalParts = nodes.size() * splitsPerNode;
    int partNumber = 0;
    // Split the data using split and skew by the number of nodes available.
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    for (Node node : nodes) {
        for (int i = 0; i < splitsPerNode; i++) {
            splits.add(new TpchSplit(tableHandle, partNumber, totalParts, ImmutableList.of(node.getHostAndPort())));
            partNumber++;
        }
    }
    return new FixedSplitSource(splits.build());
}