Examples with HostAddress com.facebook.presto.spi.HostAddress used on opensource projects

Example 11 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class HostAddressFactory method toHostAddress.

public HostAddress toHostAddress(String hostAddressName) {
    HostAddress address = hostMap.get(hostAddressName);
    if (address == null) {
        address = HostAddress.fromString(hostAddressName);
        hostMap.put(hostAddressName, address);
    }
    return address;
}

Example 12 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class SystemSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    SystemTableLayoutHandle layoutHandle = (SystemTableLayoutHandle) layout;
    SystemTableHandle tableHandle = layoutHandle.getTable();
    TupleDomain<ColumnHandle> constraint = layoutHandle.getConstraint();
    SystemTable systemTable = tables.get(tableHandle.getSchemaTableName());
    Distribution tableDistributionMode = systemTable.getDistribution();
    if (tableDistributionMode == SINGLE_COORDINATOR) {
        HostAddress address = nodeManager.getCurrentNode().getHostAndPort();
        ConnectorSplit split = new SystemSplit(tableHandle.getConnectorId(), tableHandle, address, constraint);
        return new FixedSplitSource(ImmutableList.of(split));
    }
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    ImmutableSet.Builder<Node> nodes = ImmutableSet.builder();
    if (tableDistributionMode == ALL_COORDINATORS) {
        nodes.addAll(nodeManager.getCoordinators());
    } else if (tableDistributionMode == ALL_NODES) {
        nodes.addAll(nodeManager.getNodes(ACTIVE));
    }
    Set<Node> nodeSet = nodes.build();
    for (Node node : nodeSet) {
        splits.add(new SystemSplit(tableHandle.getConnectorId(), tableHandle, node.getHostAndPort(), constraint));
    }
    return new FixedSplitSource(splits.build());
}

Example 13 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class KafkaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    KafkaTableHandle kafkaTableHandle = convertLayout(layout).getTable();
    SimpleConsumer simpleConsumer = consumerManager.getConsumer(selectRandom(nodes));
    TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(ImmutableList.of(kafkaTableHandle.getTopicName()));
    TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) {
        for (PartitionMetadata part : metadata.partitionsMetadata()) {
            log.debug("Adding Partition %s/%s", metadata.topic(), part.partitionId());
            Broker leader = part.leader();
            if (leader == null) {
                // Leader election going on...
                log.warn("No leader for partition %s/%s found!", metadata.topic(), part.partitionId());
                continue;
            }
            HostAddress partitionLeader = HostAddress.fromParts(leader.host(), leader.port());
            SimpleConsumer leaderConsumer = consumerManager.getConsumer(partitionLeader);
            // Kafka contains a reverse list of "end - start" pairs for the splits
            long[] offsets = findAllOffsets(leaderConsumer, metadata.topic(), part.partitionId());
            for (int i = offsets.length - 1; i > 0; i--) {
                KafkaSplit split = new KafkaSplit(connectorId, metadata.topic(), kafkaTableHandle.getKeyDataFormat(), kafkaTableHandle.getMessageDataFormat(), part.partitionId(), offsets[i], offsets[i - 1], partitionLeader);
                splits.add(split);
            }
        }
    }
    return new FixedSplitSource(splits.build());
}

Example 14 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class TestJmxSplitManager method testNoPredicate.

@Test
public void testNoPredicate() throws Exception {
    ConnectorTableLayoutHandle layout = new JmxTableLayoutHandle(tableHandle, TupleDomain.all());
    ConnectorSplitSource splitSource = splitManager.getSplits(JmxTransactionHandle.INSTANCE, SESSION, layout);
    List<ConnectorSplit> allSplits = getAllSplits(splitSource);
    assertEquals(allSplits.size(), nodes.size());
    Set<String> actualNodes = nodes.stream().map(Node::getNodeIdentifier).collect(toSet());
    Set<String> expectedNodes = new HashSet<>();
    for (ConnectorSplit split : allSplits) {
        List<HostAddress> addresses = split.getAddresses();
        assertEquals(addresses.size(), 1);
        expectedNodes.add(addresses.get(0).getHostText());
    }
    assertEquals(actualNodes, expectedNodes);
}

Example 15 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class TestHiveSplit method testJsonRoundTrip.

@Test
public void testJsonRoundTrip() {
    Properties schema = new Properties();
    schema.setProperty("foo", "bar");
    schema.setProperty("bar", "baz");
    ImmutableList<HivePartitionKey> partitionKeys = ImmutableList.of(new HivePartitionKey("a", HIVE_STRING, "apple"), new HivePartitionKey("b", HiveType.HIVE_LONG, "42"));
    ImmutableList<HostAddress> addresses = ImmutableList.of(HostAddress.fromParts("127.0.0.1", 44), HostAddress.fromParts("127.0.0.1", 45));
    HiveSplit expected = new HiveSplit("clientId", "db", "table", "partitionId", "path", 42, 88, schema, partitionKeys, addresses, OptionalInt.empty(), true, TupleDomain.all(), ImmutableMap.of(1, HIVE_STRING));
    String json = codec.toJson(expected);
    HiveSplit actual = codec.fromJson(json);
    assertEquals(actual.getClientId(), expected.getClientId());
    assertEquals(actual.getDatabase(), expected.getDatabase());
    assertEquals(actual.getTable(), expected.getTable());
    assertEquals(actual.getPartitionName(), expected.getPartitionName());
    assertEquals(actual.getPath(), expected.getPath());
    assertEquals(actual.getStart(), expected.getStart());
    assertEquals(actual.getLength(), expected.getLength());
    assertEquals(actual.getSchema(), expected.getSchema());
    assertEquals(actual.getPartitionKeys(), expected.getPartitionKeys());
    assertEquals(actual.getAddresses(), expected.getAddresses());
    assertEquals(actual.getColumnCoercions(), expected.getColumnCoercions());
    assertEquals(actual.isForceLocalScheduling(), expected.isForceLocalScheduling());
}