Examples with Duration io.airlift.units.Duration used on opensource projects

Example 1 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class PrestoS3FileSystem method initialize.

@Override
public void initialize(URI uri, Configuration conf) throws IOException {
    requireNonNull(uri, "uri is null");
    requireNonNull(conf, "conf is null");
    super.initialize(uri, conf);
    setConf(conf);
    this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
    this.workingDirectory = new Path(PATH_SEPARATOR).makeQualified(this.uri, new Path(PATH_SEPARATOR));
    HiveS3Config defaults = new HiveS3Config();
    this.stagingDirectory = new File(conf.get(S3_STAGING_DIRECTORY, defaults.getS3StagingDirectory().toString()));
    this.maxAttempts = conf.getInt(S3_MAX_CLIENT_RETRIES, defaults.getS3MaxClientRetries()) + 1;
    this.maxBackoffTime = Duration.valueOf(conf.get(S3_MAX_BACKOFF_TIME, defaults.getS3MaxBackoffTime().toString()));
    this.maxRetryTime = Duration.valueOf(conf.get(S3_MAX_RETRY_TIME, defaults.getS3MaxRetryTime().toString()));
    int maxErrorRetries = conf.getInt(S3_MAX_ERROR_RETRIES, defaults.getS3MaxErrorRetries());
    boolean sslEnabled = conf.getBoolean(S3_SSL_ENABLED, defaults.isS3SslEnabled());
    Duration connectTimeout = Duration.valueOf(conf.get(S3_CONNECT_TIMEOUT, defaults.getS3ConnectTimeout().toString()));
    Duration socketTimeout = Duration.valueOf(conf.get(S3_SOCKET_TIMEOUT, defaults.getS3SocketTimeout().toString()));
    int maxConnections = conf.getInt(S3_MAX_CONNECTIONS, defaults.getS3MaxConnections());
    long minFileSize = conf.getLong(S3_MULTIPART_MIN_FILE_SIZE, defaults.getS3MultipartMinFileSize().toBytes());
    long minPartSize = conf.getLong(S3_MULTIPART_MIN_PART_SIZE, defaults.getS3MultipartMinPartSize().toBytes());
    this.useInstanceCredentials = conf.getBoolean(S3_USE_INSTANCE_CREDENTIALS, defaults.isS3UseInstanceCredentials());
    this.pinS3ClientToCurrentRegion = conf.getBoolean(S3_PIN_CLIENT_TO_CURRENT_REGION, defaults.isPinS3ClientToCurrentRegion());
    this.sseEnabled = conf.getBoolean(S3_SSE_ENABLED, defaults.isS3SseEnabled());
    this.sseType = PrestoS3SseType.valueOf(conf.get(S3_SSE_TYPE, defaults.getS3SseType().name()));
    this.sseKmsKeyId = conf.get(S3_SSE_KMS_KEY_ID, defaults.getS3SseKmsKeyId());
    String userAgentPrefix = conf.get(S3_USER_AGENT_PREFIX, defaults.getS3UserAgentPrefix());
    ClientConfiguration configuration = new ClientConfiguration().withMaxErrorRetry(maxErrorRetries).withProtocol(sslEnabled ? Protocol.HTTPS : Protocol.HTTP).withConnectionTimeout(toIntExact(connectTimeout.toMillis())).withSocketTimeout(toIntExact(socketTimeout.toMillis())).withMaxConnections(maxConnections).withUserAgentPrefix(userAgentPrefix).withUserAgentSuffix(S3_USER_AGENT_SUFFIX);
    this.s3 = createAmazonS3Client(uri, conf, configuration);
    transferConfig.setMultipartUploadThreshold(minFileSize);
    transferConfig.setMinimumUploadPartSize(minPartSize);
}

Example 2 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class PrestoS3FileSystemMetricCollector method collectMetrics.

@Override
public void collectMetrics(Request<?> request, Response<?> response) {
    AWSRequestMetrics metrics = request.getAWSRequestMetrics();
    TimingInfo timingInfo = metrics.getTimingInfo();
    Number requestCounts = timingInfo.getCounter(RequestCount.name());
    Number retryCounts = timingInfo.getCounter(HttpClientRetryCount.name());
    Number throttleExceptions = timingInfo.getCounter(ThrottleException.name());
    TimingInfo requestTime = timingInfo.getSubMeasurement(HttpRequestTime.name());
    TimingInfo clientExecuteTime = timingInfo.getSubMeasurement(ClientExecuteTime.name());
    if (requestCounts != null) {
        stats.updateAwsRequestCount(requestCounts.longValue());
    }
    if (retryCounts != null) {
        stats.updateAwsRetryCount(retryCounts.longValue());
    }
    if (throttleExceptions != null) {
        stats.updateAwsThrottleExceptionsCount(throttleExceptions.longValue());
    }
    if (requestTime != null && requestTime.getTimeTakenMillisIfKnown() != null) {
        stats.addAwsRequestTime(new Duration(requestTime.getTimeTakenMillisIfKnown(), MILLISECONDS));
    }
    if (clientExecuteTime != null && clientExecuteTime.getTimeTakenMillisIfKnown() != null) {
        stats.addAwsClientExecuteTime(new Duration(clientExecuteTime.getTimeTakenMillisIfKnown(), MILLISECONDS));
    }
}

Example 3 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class PrestoConnection method startQuery.

StatementClient startQuery(String sql) {
    String source = firstNonNull(clientInfo.get("ApplicationName"), "presto-jdbc");
    ClientSession session = new ClientSession(httpUri, user, source, clientInfo.get("ClientInfo"), catalog.get(), schema.get(), timeZoneId.get(), locale.get(), ImmutableMap.copyOf(sessionProperties), transactionId.get(), false, new Duration(2, MINUTES));
    return queryExecutor.startQuery(session, sql);
}

Example 4 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class QueryRunner method getHttpClientConfig.

private static HttpClientConfig getHttpClientConfig(Optional<HostAndPort> socksProxy, Optional<String> keystorePath, Optional<String> keystorePassword, Optional<String> truststorePath, Optional<String> truststorePassword, Optional<String> kerberosPrincipal, Optional<String> kerberosRemoteServiceName, boolean authenticationEnabled) {
    HttpClientConfig httpClientConfig = new HttpClientConfig().setConnectTimeout(new Duration(5, TimeUnit.SECONDS)).setRequestTimeout(new Duration(5, TimeUnit.SECONDS));
    socksProxy.ifPresent(httpClientConfig::setSocksProxy);
    httpClientConfig.setAuthenticationEnabled(authenticationEnabled);
    keystorePath.ifPresent(httpClientConfig::setKeyStorePath);
    keystorePassword.ifPresent(httpClientConfig::setKeyStorePassword);
    truststorePath.ifPresent(httpClientConfig::setTrustStorePath);
    truststorePassword.ifPresent(httpClientConfig::setTrustStorePassword);
    kerberosPrincipal.ifPresent(httpClientConfig::setKerberosPrincipal);
    kerberosRemoteServiceName.ifPresent(httpClientConfig::setKerberosRemoteServiceName);
    return httpClientConfig;
}

Example 5 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class StatusPrinter method printQueryInfo.

private void printQueryInfo(QueryResults results) {
    StatementStats stats = results.getStats();
    Duration wallTime = nanosSince(start);
    // cap progress at 99%, otherwise it looks weird when the query is still running and it says 100%
    int progressPercentage = (int) min(99, percentage(stats.getCompletedSplits(), stats.getTotalSplits()));
    if (console.isRealTerminal()) {
        // blank line
        reprintLine("");
        int terminalWidth = console.getWidth();
        if (terminalWidth < 75) {
            reprintLine("WARNING: Terminal");
            reprintLine("must be at least");
            reprintLine("80 characters wide");
            reprintLine("");
            reprintLine(stats.getState());
            reprintLine(String.format("%s %d%%", formatTime(wallTime), progressPercentage));
            return;
        }
        int nodes = stats.getNodes();
        // Query 10, RUNNING, 1 node, 778 splits
        String querySummary = String.format("Query %s, %s, %,d %s, %,d splits", results.getId(), stats.getState(), nodes, pluralize("node", nodes), stats.getTotalSplits());
        reprintLine(querySummary);
        String url = results.getInfoUri().toString();
        if (debug && (url.length() < terminalWidth)) {
            reprintLine(url);
        }
        if ((nodes == 0) || (stats.getTotalSplits() == 0)) {
            return;
        }
        if (debug) {
            // Splits:   620 queued, 34 running, 124 done
            String splitsSummary = String.format("Splits:   %,d queued, %,d running, %,d done", stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits());
            reprintLine(splitsSummary);
            // CPU Time: 56.5s total, 36.4K rows/s, 4.44MB/s, 60% active
            Duration cpuTime = millis(stats.getCpuTimeMillis());
            String cpuTimeSummary = String.format("CPU Time: %.1fs total, %5s rows/s, %8s, %d%% active", cpuTime.getValue(SECONDS), formatCountRate(stats.getProcessedRows(), cpuTime, false), formatDataRate(bytes(stats.getProcessedBytes()), cpuTime, true), (int) percentage(stats.getCpuTimeMillis(), stats.getWallTimeMillis()));
            reprintLine(cpuTimeSummary);
            double parallelism = cpuTime.getValue(MILLISECONDS) / wallTime.getValue(MILLISECONDS);
            // Per Node: 3.5 parallelism, 83.3K rows/s, 0.7 MB/s
            String perNodeSummary = String.format("Per Node: %.1f parallelism, %5s rows/s, %8s", parallelism / nodes, formatCountRate((double) stats.getProcessedRows() / nodes, wallTime, false), formatDataRate(bytes(stats.getProcessedBytes() / nodes), wallTime, true));
            reprintLine(perNodeSummary);
            reprintLine(String.format("Parallelism: %.1f", parallelism));
        }
        // otherwise handled above
        verify(terminalWidth >= 75);
        // progress bar is 17-42 characters wide
        int progressWidth = (min(terminalWidth, 100) - 75) + 17;
        if (stats.isScheduled()) {
            String progressBar = formatProgressBar(progressWidth, stats.getCompletedSplits(), max(0, stats.getRunningSplits()), stats.getTotalSplits());
            // 0:17 [ 103MB,  802K rows] [5.74MB/s, 44.9K rows/s] [=====>>                                   ] 10%
            String progressLine = String.format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s] %d%%", formatTime(wallTime), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), true), formatCountRate(stats.getProcessedRows(), wallTime, false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true), progressBar, progressPercentage);
            reprintLine(progressLine);
        } else {
            String progressBar = formatProgressBar(progressWidth, Ints.saturatedCast(nanosSince(start).roundTo(SECONDS)));
            // 0:17 [ 103MB,  802K rows] [5.74MB/s, 44.9K rows/s] [    <=>                                  ]
            String progressLine = String.format("%s [%5s rows, %6s] [%5s rows/s, %8s] [%s]", formatTime(wallTime), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), true), formatCountRate(stats.getProcessedRows(), wallTime, false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, true), progressBar);
            reprintLine(progressLine);
        }
        // todo Mem: 1949M shared, 7594M private
        // blank line
        reprintLine("");
        // STAGE  S    ROWS    RPS  BYTES    BPS   QUEUED    RUN   DONE
        String stagesHeader = String.format("%10s%1s  %5s  %6s  %5s  %7s  %6s  %5s  %5s", "STAGE", "S", "ROWS", "ROWS/s", "BYTES", "BYTES/s", "QUEUED", "RUN", "DONE");
        reprintLine(stagesHeader);
        printStageTree(stats.getRootStage(), "", new AtomicInteger());
    } else {
        // Query 31 [S] i[2.7M 67.3MB 62.7MBps] o[35 6.1KB 1KBps] splits[252/16/380]
        String querySummary = String.format("Query %s [%s] i[%s %s %s] o[%s %s %s] splits[%,d/%,d/%,d]", results.getId(), stats.getState(), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false), formatCount(stats.getProcessedRows()), formatDataSize(bytes(stats.getProcessedBytes()), false), formatDataRate(bytes(stats.getProcessedBytes()), wallTime, false), stats.getQueuedSplits(), stats.getRunningSplits(), stats.getCompletedSplits());
        reprintLine(querySummary);
    }
}