Examples with HostAddress com.facebook.presto.spi.HostAddress used on opensource projects

Example 26 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class TestConsistentHashingNodeProvider method testDistribution.

@Test
public void testDistribution() {
    List<InternalNode> nodes = IntStream.range(0, 10).mapToObj(i -> new InternalNode(format("other%d", i), URI.create(format("http://127.0.0.%d:100", i)), NodeVersion.UNKNOWN, false)).collect(toImmutableList());
    ConsistentHashingNodeProvider nodeProvider = ConsistentHashingNodeProvider.create(nodes, 100);
    Random random = new Random();
    Map<HostAddress, Integer> result = new HashMap<>();
    for (int i = 0; i < 1_000_000; i++) {
        List<HostAddress> candidates = nodeProvider.get(format("split%d", random.nextInt()), 2);
        assertNotEquals(candidates.get(1), candidates.get(0));
        HostAddress hostAddress = candidates.get(0);
        int count = result.getOrDefault(hostAddress, 0);
        result.put(hostAddress, count + 1);
    }
    assertTrue(result.values().stream().allMatch(count -> count >= 80000 && count <= 120000));
}

Example 27 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class InternalHiveSplit method getEstimatedSizeInBytes.

/**
 * Estimate the size of this InternalHiveSplit. Note that
 * PartitionInfo is a shared object, so its memory usage is
 * tracked separately in HiveSplitSource.
 */
public int getEstimatedSizeInBytes() {
    int result = INSTANCE_SIZE;
    result += sizeOf(relativeUri);
    result += sizeOf(blockEndOffsets);
    if (!blockAddresses.isEmpty()) {
        result += sizeOfObjectArray(blockAddresses.size());
        for (List<HostAddress> addresses : blockAddresses) {
            result += sizeOfObjectArray(addresses.size());
            for (HostAddress address : addresses) {
                result += HOST_ADDRESS_INSTANCE_SIZE + address.getHostText().length() * Character.BYTES;
            }
        }
    }
    if (extraFileInfo.isPresent()) {
        result += sizeOf(extraFileInfo.get());
    }
    return result;
}

Example 28 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class Failures method toFailure.

private static ExecutionFailureInfo toFailure(Throwable throwable, Set<Throwable> seenFailures) {
    if (throwable == null) {
        return null;
    }
    String type;
    HostAddress remoteHost = null;
    if (throwable instanceof Failure) {
        type = ((Failure) throwable).getType();
    } else {
        Class<?> clazz = throwable.getClass();
        type = firstNonNull(clazz.getCanonicalName(), clazz.getName());
    }
    if (throwable instanceof PrestoTransportException) {
        remoteHost = ((PrestoTransportException) throwable).getRemoteHost();
    }
    if (seenFailures.contains(throwable)) {
        return new ExecutionFailureInfo(type, "[cyclic] " + throwable.getMessage(), null, ImmutableList.of(), ImmutableList.of(), null, GENERIC_INTERNAL_ERROR.toErrorCode(), remoteHost);
    }
    seenFailures.add(throwable);
    ExecutionFailureInfo cause = toFailure(throwable.getCause(), seenFailures);
    ErrorCode errorCode = toErrorCode(throwable);
    if (errorCode == null) {
        if (cause == null) {
            errorCode = GENERIC_INTERNAL_ERROR.toErrorCode();
        } else {
            errorCode = cause.getErrorCode();
        }
    }
    return new ExecutionFailureInfo(type, throwable.getMessage(), cause, Arrays.stream(throwable.getSuppressed()).map(failure -> toFailure(failure, seenFailures)).collect(toImmutableList()), Lists.transform(asList(throwable.getStackTrace()), toStringFunction()), getErrorLocation(throwable), errorCode, remoteHost);
}

Example 29 with HostAddress

use of com.facebook.presto.spi.HostAddress in project presto by prestodb.

the class RedisSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableLayoutHandle layout, SplitSchedulingContext splitSchedulingContext) {
    RedisTableHandle redisTableHandle = convertLayout(layout).getTable();
    List<HostAddress> nodes = new ArrayList<>(redisConnectorConfig.getNodes());
    Collections.shuffle(nodes);
    checkState(!nodes.isEmpty(), "No Redis nodes available");
    ImmutableList.Builder<ConnectorSplit> builder = ImmutableList.builder();
    long numberOfKeys = 1;
    // splits by splitting zset in chunks
    if (redisTableHandle.getKeyDataFormat().equals("zset")) {
        try (Jedis jedis = jedisManager.getJedisPool(nodes.get(0)).getResource()) {
            numberOfKeys = jedis.zcount(redisTableHandle.getKeyName(), "-inf", "+inf");
        }
    }
    long stride = REDIS_STRIDE_SPLITS;
    if (numberOfKeys / stride > REDIS_MAX_SPLITS) {
        stride = numberOfKeys / REDIS_MAX_SPLITS;
    }
    for (long startIndex = 0; startIndex < numberOfKeys; startIndex += stride) {
        long endIndex = startIndex + stride - 1;
        if (endIndex >= numberOfKeys) {
            endIndex = -1;
        }
        RedisSplit split = new RedisSplit(connectorId, redisTableHandle.getSchemaName(), redisTableHandle.getTableName(), redisTableHandle.getKeyDataFormat(), redisTableHandle.getValueDataFormat(), redisTableHandle.getKeyName(), startIndex, endIndex, nodes);
        builder.add(split);
    }
    return new FixedSplitSource(builder.build());
}