Examples with HostAddress com.facebook.presto.spi.HostAddress used on opensource projects

Example 1 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class CassandraSplitManager method getSplitsForPartitions.

private List<ConnectorSplit> getSplitsForPartitions(CassandraTableHandle cassTableHandle, List<CassandraPartition> partitions, List<String> clusteringPredicates) {
    String schema = cassTableHandle.getSchemaName();
    HostAddressFactory hostAddressFactory = new HostAddressFactory();
    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    // For single partition key column table, we can merge multiple partitions into a single split
    // by using IN CLAUSE in a single select query if the partitions have the same host list.
    // For multiple partition key columns table, we can't merge them into a single select query, so
    // keep them in a separate split.
    boolean singlePartitionKeyColumn = true;
    String partitionKeyColumnName = null;
    if (!partitions.isEmpty()) {
        singlePartitionKeyColumn = partitions.get(0).getTupleDomain().getDomains().get().size() == 1;
        if (singlePartitionKeyColumn) {
            String partitionId = partitions.get(0).getPartitionId();
            partitionKeyColumnName = partitionId.substring(0, partitionId.lastIndexOf('=') - 1);
        }
    }
    Map<Set<String>, Set<String>> hostsToPartitionKeys = new HashMap<>();
    Map<Set<String>, List<HostAddress>> hostMap = new HashMap<>();
    for (CassandraPartition cassandraPartition : partitions) {
        Set<Host> hosts = cassandraSession.getReplicas(schema, cassandraPartition.getKeyAsByteBuffer());
        List<HostAddress> addresses = hostAddressFactory.toHostAddressList(hosts);
        if (singlePartitionKeyColumn) {
            // host ip addresses
            ImmutableSet.Builder<String> sb = ImmutableSet.builder();
            for (HostAddress address : addresses) {
                sb.add(address.getHostText());
            }
            Set<String> hostAddresses = sb.build();
            // partition key values
            Set<String> values = hostsToPartitionKeys.get(hostAddresses);
            if (values == null) {
                values = new HashSet<>();
            }
            String partitionId = cassandraPartition.getPartitionId();
            values.add(partitionId.substring(partitionId.lastIndexOf('=') + 2));
            hostsToPartitionKeys.put(hostAddresses, values);
            hostMap.put(hostAddresses, addresses);
        } else {
            builder.addAll(createSplitsForClusteringPredicates(cassTableHandle, cassandraPartition.getPartitionId(), addresses, clusteringPredicates));
        }
    }
    if (singlePartitionKeyColumn) {
        for (Map.Entry<Set<String>, Set<String>> entry : hostsToPartitionKeys.entrySet()) {
            StringBuilder sb = new StringBuilder(partitionSizeForBatchSelect);
            int size = 0;
            for (String value : entry.getValue()) {
                if (size > 0) {
                    sb.append(",");
                }
                sb.append(value);
                size++;
                if (size > partitionSizeForBatchSelect) {
                    String partitionId = String.format("%s in (%s)", partitionKeyColumnName, sb.toString());
                    builder.addAll(createSplitsForClusteringPredicates(cassTableHandle, partitionId, hostMap.get(entry.getKey()), clusteringPredicates));
                    size = 0;
                    sb.setLength(0);
                    sb.trimToSize();
                }
            }
            if (size > 0) {
                String partitionId = String.format("%s in (%s)", partitionKeyColumnName, sb.toString());
                builder.addAll(createSplitsForClusteringPredicates(cassTableHandle, partitionId, hostMap.get(entry.getKey()), clusteringPredicates));
            }
        }
    }
    return builder.build();
}

Example 2 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class CassandraSplitManager method getSplitsByTokenRange.

private List<ConnectorSplit> getSplitsByTokenRange(CassandraTable table, String partitionId) {
    String schema = table.getTableHandle().getSchemaName();
    String tableName = table.getTableHandle().getTableName();
    String tokenExpression = table.getTokenExpression();
    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    List<CassandraTokenSplitManager.TokenSplit> tokenSplits = tokenSplitMgr.getSplits(schema, tableName);
    for (CassandraTokenSplitManager.TokenSplit tokenSplit : tokenSplits) {
        String condition = buildTokenCondition(tokenExpression, tokenSplit.getStartToken(), tokenSplit.getEndToken());
        List<HostAddress> addresses = new HostAddressFactory().AddressNamesToHostAddressList(tokenSplit.getHosts());
        CassandraSplit split = new CassandraSplit(connectorId, schema, tableName, partitionId, condition, addresses);
        builder.add(split);
    }
    return builder.build();
}

Example 3 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class InformationSchemaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout) {
    InformationSchemaTableLayoutHandle handle = (InformationSchemaTableLayoutHandle) layout;
    Map<ColumnHandle, NullableValue> bindings = extractFixedValues(handle.getConstraint()).orElse(ImmutableMap.of());
    List<HostAddress> localAddress = ImmutableList.of(nodeManager.getCurrentNode().getHostAndPort());
    Map<String, NullableValue> filters = bindings.entrySet().stream().collect(toMap(entry -> ((InformationSchemaColumnHandle) entry.getKey()).getColumnName(), Entry::getValue));
    ConnectorSplit split = new InformationSchemaSplit(handle.getTable(), filters, localAddress);
    return new FixedSplitSource(ImmutableList.of(split));
}

Example 4 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class NodeScheduler method createNodeSelector.

public NodeSelector createNodeSelector(ConnectorId connectorId) {
    // this supplier is thread-safe. TODO: this logic should probably move to the scheduler since the choice of which node to run in should be
    // done as close to when the the split is about to be scheduled
    Supplier<NodeMap> nodeMap = Suppliers.memoizeWithExpiration(() -> {
        ImmutableSetMultimap.Builder<HostAddress, Node> byHostAndPort = ImmutableSetMultimap.builder();
        ImmutableSetMultimap.Builder<InetAddress, Node> byHost = ImmutableSetMultimap.builder();
        ImmutableSetMultimap.Builder<NetworkLocation, Node> workersByNetworkPath = ImmutableSetMultimap.builder();
        Set<Node> nodes;
        if (connectorId != null) {
            nodes = nodeManager.getActiveConnectorNodes(connectorId);
        } else {
            nodes = nodeManager.getNodes(ACTIVE);
        }
        Set<String> coordinatorNodeIds = nodeManager.getCoordinators().stream().map(Node::getNodeIdentifier).collect(toImmutableSet());
        for (Node node : nodes) {
            if (useNetworkTopology && (includeCoordinator || !coordinatorNodeIds.contains(node.getNodeIdentifier()))) {
                NetworkLocation location = networkLocationCache.get(node.getHostAndPort());
                for (int i = 0; i <= location.getSegments().size(); i++) {
                    workersByNetworkPath.put(location.subLocation(0, i), node);
                }
            }
            try {
                byHostAndPort.put(node.getHostAndPort(), node);
                InetAddress host = InetAddress.getByName(node.getHttpUri().getHost());
                byHost.put(host, node);
            } catch (UnknownHostException e) {
            // ignore
            }
        }
        return new NodeMap(byHostAndPort.build(), byHost.build(), workersByNetworkPath.build(), coordinatorNodeIds);
    }, 5, TimeUnit.SECONDS);
    if (useNetworkTopology) {
        return new TopologyAwareNodeSelector(nodeManager, nodeTaskMap, includeCoordinator, nodeMap, minCandidates, maxSplitsPerNode, maxPendingSplitsPerTask, topologicalSplitCounters, networkLocationSegmentNames, networkLocationCache);
    } else {
        return new SimpleNodeSelector(nodeManager, nodeTaskMap, includeCoordinator, nodeMap, minCandidates, maxSplitsPerNode, maxPendingSplitsPerTask);
    }
}

Example 5 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class TestHostAddressFactory method testToHostAddressList.

@Test
public void testToHostAddressList() throws Exception {
    Set<Host> hosts = ImmutableSet.of(new TestHost(new InetSocketAddress(InetAddress.getByAddress(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }), 3000)), new TestHost(new InetSocketAddress(InetAddress.getByAddress(new byte[] { 1, 2, 3, 4 }), 3000)));
    HostAddressFactory hostAddressFactory = new HostAddressFactory();
    List<HostAddress> list = hostAddressFactory.toHostAddressList(hosts);
    assertEquals(list.toString(), "[[102:304:506:708:90a:b0c:d0e:f10], 1.2.3.4]");
}