Examples with HostAddress io.trino.spi.HostAddress used on opensource projects

Example 16 with HostAddress

use of io.trino.spi.HostAddress in project trino by trinodb.

the class InformationSchemaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableHandle table, SplitSchedulingStrategy splitSchedulingStrategy, DynamicFilter dynamicFilter) {
    List<HostAddress> localAddress = ImmutableList.of(nodeManager.getCurrentNode().getHostAndPort());
    ConnectorSplit split = new InformationSchemaSplit(localAddress);
    return new FixedSplitSource(ImmutableList.of(split));
}

Example 17 with HostAddress

use of io.trino.spi.HostAddress in project trino by trinodb.

the class TestStageTaskSourceFactory method testSourceDistributionTaskSource.

@Test
public void testSourceDistributionTaskSource() {
    TaskSource taskSource = createSourceDistributionTaskSource(ImmutableList.of(), ImmutableListMultimap.of(), 2, 0, 3 * STANDARD_WEIGHT, 1000);
    assertFalse(taskSource.isFinished());
    assertEquals(taskSource.getMoreTasks(), ImmutableList.of());
    assertTrue(taskSource.isFinished());
    Split split1 = createSplit(1);
    Split split2 = createSplit(2);
    Split split3 = createSplit(3);
    taskSource = createSourceDistributionTaskSource(ImmutableList.of(split1), ImmutableListMultimap.of(), 2, 0, 2 * STANDARD_WEIGHT, 1000);
    assertEquals(taskSource.getMoreTasks(), ImmutableList.of(new TaskDescriptor(0, ImmutableListMultimap.of(PLAN_NODE_1, split1), ImmutableListMultimap.of(), new NodeRequirements(Optional.of(CATALOG), ImmutableSet.of(), DataSize.of(4, GIGABYTE)))));
    assertTrue(taskSource.isFinished());
    taskSource = createSourceDistributionTaskSource(ImmutableList.of(split1, split2, split3), ImmutableListMultimap.of(), 3, 0, 2 * STANDARD_WEIGHT, 1000);
    List<TaskDescriptor> tasks = readAllTasks(taskSource);
    assertThat(tasks).hasSize(2);
    assertThat(tasks.get(0).getSplits().values()).hasSize(2);
    assertThat(tasks.get(1).getSplits().values()).hasSize(1);
    assertThat(tasks).allMatch(taskDescriptor -> taskDescriptor.getNodeRequirements().equals(new NodeRequirements(Optional.of(CATALOG), ImmutableSet.of(), DataSize.of(4, GIGABYTE))));
    assertThat(tasks).allMatch(taskDescriptor -> taskDescriptor.getExchangeSourceHandles().isEmpty());
    assertThat(flattenSplits(tasks)).hasSameEntriesAs(ImmutableMultimap.of(PLAN_NODE_1, split1, PLAN_NODE_1, split2, PLAN_NODE_1, split3));
    assertTrue(taskSource.isFinished());
    ImmutableListMultimap<PlanNodeId, ExchangeSourceHandle> replicatedSources = ImmutableListMultimap.of(PLAN_NODE_2, new TestingExchangeSourceHandle(0, 1));
    taskSource = createSourceDistributionTaskSource(ImmutableList.of(split1, split2, split3), replicatedSources, 2, 0, 2 * STANDARD_WEIGHT, 1000);
    tasks = readAllTasks(taskSource);
    assertThat(tasks).hasSize(2);
    assertThat(tasks.get(0).getSplits().values()).hasSize(2);
    assertThat(tasks.get(1).getSplits().values()).hasSize(1);
    assertThat(tasks).allMatch(taskDescriptor -> taskDescriptor.getNodeRequirements().equals(new NodeRequirements(Optional.of(CATALOG), ImmutableSet.of(), DataSize.of(4, GIGABYTE))));
    assertThat(tasks).allMatch(taskDescriptor -> taskDescriptor.getExchangeSourceHandles().equals(replicatedSources));
    assertThat(flattenSplits(tasks)).hasSameEntriesAs(ImmutableMultimap.of(PLAN_NODE_1, split1, PLAN_NODE_1, split2, PLAN_NODE_1, split3));
    assertTrue(taskSource.isFinished());
    // non remotely accessible splits
    ImmutableList<Split> splits = ImmutableList.of(createSplit(1, "host1:8080", "host2:8080"), createSplit(2, "host2:8080"), createSplit(3, "host1:8080", "host3:8080"), createSplit(4, "host3:8080", "host1:8080"), createSplit(5, "host1:8080", "host2:8080"), createSplit(6, "host2:8080", "host3:8080"), createSplit(7, "host3:8080", "host4:8080"));
    taskSource = createSourceDistributionTaskSource(splits, ImmutableListMultimap.of(), 3, 0, 2 * STANDARD_WEIGHT, 1000);
    tasks = readAllTasks(taskSource);
    assertThat(tasks).hasSize(4);
    assertThat(tasks.stream()).allMatch(taskDescriptor -> taskDescriptor.getExchangeSourceHandles().isEmpty());
    assertThat(flattenSplits(tasks)).hasSameEntriesAs(Multimaps.index(splits, split -> PLAN_NODE_1));
    assertThat(tasks).allMatch(task -> task.getSplits().values().stream().allMatch(split -> {
        HostAddress requiredAddress = getOnlyElement(task.getNodeRequirements().getAddresses());
        return split.getAddresses().contains(requiredAddress);
    }));
    assertTrue(taskSource.isFinished());
}

Example 18 with HostAddress

use of io.trino.spi.HostAddress in project trino by trinodb.

the class CassandraSplitManager method getSplitsByTokenRange.

private List<ConnectorSplit> getSplitsByTokenRange(CassandraTable table, String partitionId, Optional<Long> sessionSplitsPerNode) {
    String schema = table.getTableHandle().getSchemaName();
    String tableName = table.getTableHandle().getTableName();
    String tokenExpression = table.getTokenExpression();
    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    List<CassandraTokenSplitManager.TokenSplit> tokenSplits = tokenSplitMgr.getSplits(schema, tableName, sessionSplitsPerNode);
    for (CassandraTokenSplitManager.TokenSplit tokenSplit : tokenSplits) {
        String condition = buildTokenCondition(tokenExpression, tokenSplit.getStartToken(), tokenSplit.getEndToken());
        List<HostAddress> addresses = new HostAddressFactory().hostAddressNamesToHostAddressList(tokenSplit.getHosts());
        CassandraSplit split = new CassandraSplit(partitionId, condition, addresses);
        builder.add(split);
    }
    return builder.build();
}

Example 19 with HostAddress

use of io.trino.spi.HostAddress in project trino by trinodb.

the class TestHiveSplit method testJsonRoundTrip.

@Test
public void testJsonRoundTrip() {
    ObjectMapperProvider objectMapperProvider = new ObjectMapperProvider();
    objectMapperProvider.setJsonDeserializers(ImmutableMap.of(Type.class, new TypeDeserializer(new TestingTypeManager())));
    JsonCodec<HiveSplit> codec = new JsonCodecFactory(objectMapperProvider).jsonCodec(HiveSplit.class);
    Properties schema = new Properties();
    schema.setProperty("foo", "bar");
    schema.setProperty("bar", "baz");
    ImmutableList<HivePartitionKey> partitionKeys = ImmutableList.of(new HivePartitionKey("a", "apple"), new HivePartitionKey("b", "42"));
    ImmutableList<HostAddress> addresses = ImmutableList.of(HostAddress.fromParts("127.0.0.1", 44), HostAddress.fromParts("127.0.0.1", 45));
    AcidInfo.Builder acidInfoBuilder = AcidInfo.builder(new Path("file:///data/fullacid"));
    acidInfoBuilder.addDeleteDelta(new Path("file:///data/fullacid/delete_delta_0000004_0000004_0000"));
    acidInfoBuilder.addDeleteDelta(new Path("file:///data/fullacid/delete_delta_0000007_0000007_0000"));
    AcidInfo acidInfo = acidInfoBuilder.build().get();
    HiveSplit expected = new HiveSplit("db", "table", "partitionId", "path", 42, 87, 88, Instant.now().toEpochMilli(), schema, partitionKeys, addresses, OptionalInt.empty(), 0, true, TableToPartitionMapping.mapColumnsByIndex(ImmutableMap.of(1, new HiveTypeName("string"))), Optional.of(new HiveSplit.BucketConversion(BUCKETING_V1, 32, 16, ImmutableList.of(createBaseColumn("col", 5, HIVE_LONG, BIGINT, ColumnType.REGULAR, Optional.of("comment"))))), Optional.empty(), false, Optional.of(acidInfo), 555534, // some non-standard value
    SplitWeight.fromProportion(2.0));
    String json = codec.toJson(expected);
    HiveSplit actual = codec.fromJson(json);
    assertEquals(actual.getDatabase(), expected.getDatabase());
    assertEquals(actual.getTable(), expected.getTable());
    assertEquals(actual.getPartitionName(), expected.getPartitionName());
    assertEquals(actual.getPath(), expected.getPath());
    assertEquals(actual.getStart(), expected.getStart());
    assertEquals(actual.getLength(), expected.getLength());
    assertEquals(actual.getEstimatedFileSize(), expected.getEstimatedFileSize());
    assertEquals(actual.getSchema(), expected.getSchema());
    assertEquals(actual.getPartitionKeys(), expected.getPartitionKeys());
    assertEquals(actual.getAddresses(), expected.getAddresses());
    assertEquals(actual.getTableToPartitionMapping().getPartitionColumnCoercions(), expected.getTableToPartitionMapping().getPartitionColumnCoercions());
    assertEquals(actual.getTableToPartitionMapping().getTableToPartitionColumns(), expected.getTableToPartitionMapping().getTableToPartitionColumns());
    assertEquals(actual.getBucketConversion(), expected.getBucketConversion());
    assertEquals(actual.isForceLocalScheduling(), expected.isForceLocalScheduling());
    assertEquals(actual.isS3SelectPushdownEnabled(), expected.isS3SelectPushdownEnabled());
    assertEquals(actual.getAcidInfo().get(), expected.getAcidInfo().get());
    assertEquals(actual.getSplitNumber(), expected.getSplitNumber());
    assertEquals(actual.getSplitWeight(), expected.getSplitWeight());
}

Example 20 with HostAddress

use of io.trino.spi.HostAddress in project trino by trinodb.

the class NodeScheduler method selectExactNodes.

public static List<InternalNode> selectExactNodes(NodeMap nodeMap, List<HostAddress> hosts, boolean includeCoordinator) {
    Set<InternalNode> chosen = new LinkedHashSet<>();
    Set<String> coordinatorIds = nodeMap.getCoordinatorNodeIds();
    for (HostAddress host : hosts) {
        nodeMap.getNodesByHostAndPort().get(host).stream().filter(node -> includeCoordinator || !coordinatorIds.contains(node.getNodeIdentifier())).forEach(chosen::add);
        InetAddress address;
        try {
            address = host.toInetAddress();
        } catch (UnknownHostException e) {
            // skip hosts that don't resolve
            continue;
        }
        // consider a split with a host without a port as being accessible by all nodes in that host
        if (!host.hasPort()) {
            nodeMap.getNodesByHost().get(address).stream().filter(node -> includeCoordinator || !coordinatorIds.contains(node.getNodeIdentifier())).forEach(chosen::add);
        }
    }
    // if the chosen set is empty and the host is the coordinator, force pick the coordinator
    if (chosen.isEmpty() && !includeCoordinator) {
        for (HostAddress host : hosts) {
            // In the code below, before calling `chosen::add`, it could have been checked that
            // `coordinatorIds.contains(node.getNodeIdentifier())`. But checking the condition isn't necessary
            // because every node satisfies it. Otherwise, `chosen` wouldn't have been empty.
            chosen.addAll(nodeMap.getNodesByHostAndPort().get(host));
            InetAddress address;
            try {
                address = host.toInetAddress();
            } catch (UnknownHostException e) {
                // skip hosts that don't resolve
                continue;
            }
            // consider a split with a host without a port as being accessible by all nodes in that host
            if (!host.hasPort()) {
                chosen.addAll(nodeMap.getNodesByHost().get(address));
            }
        }
    }
    return ImmutableList.copyOf(chosen);
}