Examples with HostAddress com.facebook.presto.spi.HostAddress used on opensource projects

Example 21 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class MemoryMetadata method updateRowsOnHosts.

private void updateRowsOnHosts(MemoryTableHandle table, Collection<Slice> fragments) {
    checkState(tableDataFragments.containsKey(table.getTableId()), "Uninitialized table [%s.%s]", table.getSchemaName(), table.getTableName());
    Map<HostAddress, MemoryDataFragment> dataFragments = tableDataFragments.get(table.getTableId());
    for (Slice fragment : fragments) {
        MemoryDataFragment memoryDataFragment = MemoryDataFragment.fromSlice(fragment);
        dataFragments.merge(memoryDataFragment.getHostAddress(), memoryDataFragment, MemoryDataFragment::merge);
    }
}

Example 22 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class CassandraSplitManager method getSplitsByTokenRange.

private List<ConnectorSplit> getSplitsByTokenRange(CassandraTable table, String partitionId, Optional<Long> sessionSplitsPerNode) {
    String schema = table.getTableHandle().getSchemaName();
    String tableName = table.getTableHandle().getTableName();
    String tokenExpression = table.getTokenExpression();
    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    List<CassandraTokenSplitManager.TokenSplit> tokenSplits = tokenSplitMgr.getSplits(schema, tableName, sessionSplitsPerNode);
    for (CassandraTokenSplitManager.TokenSplit tokenSplit : tokenSplits) {
        String condition = buildTokenCondition(tokenExpression, tokenSplit.getStartToken(), tokenSplit.getEndToken());
        List<HostAddress> addresses = new HostAddressFactory().AddressNamesToHostAddressList(tokenSplit.getHosts());
        CassandraSplit split = new CassandraSplit(connectorId, schema, tableName, partitionId, condition, addresses);
        builder.add(split);
    }
    return builder.build();
}

Example 23 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class HostAddressFactory method toHostAddress.

public HostAddress toHostAddress(String hostAddressName) {
    HostAddress address = hostMap.get(hostAddressName);
    if (address == null) {
        address = HostAddress.fromString(hostAddressName);
        hostMap.put(hostAddressName, address);
    }
    return address;
}

Example 24 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class TestHiveSplit method testJsonRoundTrip.

@Test
public void testJsonRoundTrip() throws Exception {
    ImmutableList<HivePartitionKey> partitionKeys = ImmutableList.of(new HivePartitionKey("a", Optional.of("apple")), new HivePartitionKey("b", Optional.of("42")));
    ImmutableList<HostAddress> addresses = ImmutableList.of(HostAddress.fromParts("127.0.0.1", 44), HostAddress.fromParts("127.0.0.1", 45));
    Map<String, String> customSplitInfo = ImmutableMap.of("key", "value");
    Set<ColumnHandle> redundantColumnDomains = ImmutableSet.of(new HiveColumnHandle("test_column", HIVE_LONG, HIVE_LONG.getTypeSignature(), 5, REGULAR, Optional.empty(), ImmutableList.of(), Optional.empty()));
    HiveSplit expected = new HiveSplit("db", "table", "partitionId", "path", 42, 87, 88, Instant.now().toEpochMilli(), new Storage(StorageFormat.create("serde", "input", "output"), "location", Optional.empty(), false, ImmutableMap.of(), ImmutableMap.of()), partitionKeys, addresses, OptionalInt.empty(), OptionalInt.empty(), NO_PREFERENCE, 10, TableToPartitionMapping.mapColumnsByIndex(ImmutableMap.of(1, new Column("name", HIVE_STRING, Optional.empty(), Optional.empty()))), Optional.of(new HiveSplit.BucketConversion(32, 16, ImmutableList.of(new HiveColumnHandle("col", HIVE_LONG, BIGINT.getTypeSignature(), 5, REGULAR, Optional.of("comment"), Optional.empty())))), false, Optional.empty(), NO_CACHE_REQUIREMENT, Optional.of(EncryptionInformation.fromEncryptionMetadata(DwrfEncryptionMetadata.forPerField(ImmutableMap.of("field1", "test1".getBytes()), ImmutableMap.of(), "test_algo", "test_provider"))), customSplitInfo, redundantColumnDomains, // some non-standard value
    SplitWeight.fromProportion(2.0));
    JsonCodec<HiveSplit> codec = getJsonCodec();
    String json = codec.toJson(expected);
    HiveSplit actual = codec.fromJson(json);
    assertEquals(actual.getDatabase(), expected.getDatabase());
    assertEquals(actual.getTable(), expected.getTable());
    assertEquals(actual.getPartitionName(), expected.getPartitionName());
    assertEquals(actual.getPath(), expected.getPath());
    assertEquals(actual.getStart(), expected.getStart());
    assertEquals(actual.getLength(), expected.getLength());
    assertEquals(actual.getFileSize(), expected.getFileSize());
    assertEquals(actual.getStorage(), expected.getStorage());
    assertEquals(actual.getPartitionKeys(), expected.getPartitionKeys());
    assertEquals(actual.getAddresses(), expected.getAddresses());
    assertEquals(actual.getPartitionDataColumnCount(), expected.getPartitionDataColumnCount());
    assertEquals(actual.getTableToPartitionMapping().getPartitionSchemaDifference(), expected.getTableToPartitionMapping().getPartitionSchemaDifference());
    assertEquals(actual.getTableToPartitionMapping().getTableToPartitionColumns(), expected.getTableToPartitionMapping().getTableToPartitionColumns());
    assertEquals(actual.getBucketConversion(), expected.getBucketConversion());
    assertEquals(actual.getNodeSelectionStrategy(), expected.getNodeSelectionStrategy());
    assertEquals(actual.isS3SelectPushdownEnabled(), expected.isS3SelectPushdownEnabled());
    assertEquals(actual.getCacheQuotaRequirement(), expected.getCacheQuotaRequirement());
    assertEquals(actual.getEncryptionInformation(), expected.getEncryptionInformation());
    assertEquals(actual.getCustomSplitInfo(), expected.getCustomSplitInfo());
    assertEquals(actual.getSplitWeight(), expected.getSplitWeight());
}

Example 25 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class TopologyAwareNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks) {
    NodeMap nodeMap = this.nodeMap.get().get();
    Multimap<InternalNode, Split> assignment = HashMultimap.create();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    int[] topologicCounters = new int[topologicalSplitCounters.size()];
    Set<NetworkLocation> filledLocations = new HashSet<>();
    Set<InternalNode> blockedExactNodes = new HashSet<>();
    boolean splitWaitingForAnyNode = false;
    NodeProvider nodeProvider = nodeMap.getActiveNodeProvider(nodeSelectionHashStrategy);
    for (Split split : splits) {
        SplitWeight splitWeight = split.getSplitWeight();
        if (split.getNodeSelectionStrategy() == HARD_AFFINITY) {
            List<InternalNode> candidateNodes = selectExactNodes(nodeMap, split.getPreferredNodes(nodeProvider), includeCoordinator);
            if (candidateNodes.isEmpty()) {
                log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getActiveNodes());
                throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
            }
            InternalNode chosenNode = bestNodeSplitCount(splitWeight, candidateNodes.iterator(), minCandidates, maxPendingSplitsWeightPerTask, assignmentStats);
            if (chosenNode != null) {
                assignment.put(chosenNode, split);
                assignmentStats.addAssignedSplit(chosenNode, splitWeight);
            } else // Exact node set won't matter, if a split is waiting for any node
            if (!splitWaitingForAnyNode) {
                blockedExactNodes.addAll(candidateNodes);
            }
            continue;
        }
        InternalNode chosenNode = null;
        int depth = networkLocationSegmentNames.size();
        int chosenDepth = 0;
        Set<NetworkLocation> locations = new HashSet<>();
        for (HostAddress host : split.getPreferredNodes(nodeProvider)) {
            locations.add(networkLocationCache.get(host));
        }
        if (locations.isEmpty()) {
            // Add the root location
            locations.add(ROOT_LOCATION);
            depth = 0;
        }
        // Try each address at progressively shallower network locations
        for (int i = depth; i >= 0 && chosenNode == null; i--) {
            for (NetworkLocation location : locations) {
                // For example, locations which couldn't be located will be at the "root" location
                if (location.getSegments().size() < i) {
                    continue;
                }
                location = location.subLocation(0, i);
                if (filledLocations.contains(location)) {
                    continue;
                }
                Set<InternalNode> nodes = nodeMap.getActiveWorkersByNetworkPath().get(location);
                chosenNode = bestNodeSplitCount(splitWeight, new ResettableRandomizedIterator<>(nodes), minCandidates, calculateMaxPendingSplitsWeightPerTask(i, depth), assignmentStats);
                if (chosenNode != null) {
                    chosenDepth = i;
                    break;
                }
                filledLocations.add(location);
            }
        }
        if (chosenNode != null) {
            assignment.put(chosenNode, split);
            assignmentStats.addAssignedSplit(chosenNode, splitWeight);
            topologicCounters[chosenDepth]++;
        } else {
            splitWaitingForAnyNode = true;
        }
    }
    for (int i = 0; i < topologicCounters.length; i++) {
        if (topologicCounters[i] > 0) {
            topologicalSplitCounters.get(i).update(topologicCounters[i]);
        }
    }
    ListenableFuture<?> blocked;
    long maxPendingForWildcardNetworkAffinity = calculateMaxPendingSplitsWeightPerTask(0, networkLocationSegmentNames.size());
    if (splitWaitingForAnyNode) {
        blocked = toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingForWildcardNetworkAffinity));
    } else {
        blocked = toWhenHasSplitQueueSpaceFuture(blockedExactNodes, existingTasks, calculateLowWatermark(maxPendingForWildcardNetworkAffinity));
    }
    return new SplitPlacementResult(blocked, assignment);
}