Examples with S3MetaRequest software.amazon.awssdk.crt.s3.S3MetaRequest used on opensource projects

Example 1 with S3MetaRequest

use of software.amazon.awssdk.crt.s3.S3MetaRequest in project aws-crt-java by awslabs.

the class S3NativeClient method getObject.

public CompletableFuture<GetObjectOutput> getObject(GetObjectRequest request, final ResponseDataConsumer<GetObjectOutput> dataHandler) {
    final CompletableFuture<GetObjectOutput> resultFuture = new CompletableFuture<>();
    final GetObjectOutput.Builder resultBuilder = GetObjectOutput.builder();
    final S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {

        private GetObjectOutput getObjectOutput;

        @Override
        public void onResponseHeaders(final int statusCode, final HttpHeader[] headers) {
            for (int headerIndex = 0; headerIndex < headers.length; ++headerIndex) {
                try {
                    populateGetObjectOutputHeader(resultBuilder, headers[headerIndex]);
                } catch (Exception e) {
                    resultFuture.completeExceptionally(new RuntimeException(String.format("Could not process response header {%s}: " + headers[headerIndex].getName()), e));
                }
            }
            populateGetObjectOutputUserMetadata(resultBuilder, headers);
            dataHandler.onResponseHeaders(statusCode, headers);
            getObjectOutput = resultBuilder.build();
            dataHandler.onResponse(getObjectOutput);
        }

        @Override
        public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) {
            dataHandler.onResponseData(bodyBytesIn);
            return 0;
        }

        @Override
        public void onFinished(int errorCode, int responseStatus, byte[] errorPayload) {
            CrtS3RuntimeException ex = null;
            try {
                if (errorCode != CRT.AWS_CRT_SUCCESS) {
                    ex = new CrtS3RuntimeException(errorCode, responseStatus, errorPayload);
                    dataHandler.onException(ex);
                } else {
                    dataHandler.onFinished();
                }
            } catch (Exception e) {
            /* ignore user callback exception */
            } finally {
                if (ex != null) {
                    resultFuture.completeExceptionally(ex);
                } else {
                    resultFuture.complete(getObjectOutput);
                }
            }
        }
    };
    List<HttpHeader> headers = new LinkedList<>();
    // TODO: additional logic needed for *special* partitions
    headers.add(new HttpHeader("Host", request.bucket() + ".s3." + signingRegion + ".amazonaws.com"));
    populateGetObjectRequestHeaders(header -> headers.add(header), request);
    addCustomHeaders(headers, request.customHeaders());
    String getObjectRequestQueryParameters = getGetObjectRequestQueryParameters(request);
    String encodedPath = getEncodedPath(request.key(), getObjectRequestQueryParameters);
    HttpRequest httpRequest = new HttpRequest("GET", encodedPath, headers.toArray(new HttpHeader[0]), null);
    S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions().withMetaRequestType(S3MetaRequestOptions.MetaRequestType.GET_OBJECT).withHttpRequest(httpRequest).withResponseHandler(responseHandler);
    try (final S3MetaRequest metaRequest = s3Client.makeMetaRequest(metaRequestOptions)) {
        addCancelCheckToFuture(resultFuture, metaRequest);
        return resultFuture;
    }
}

Example 2 with S3MetaRequest

use of software.amazon.awssdk.crt.s3.S3MetaRequest in project aws-sdk-java-v2 by aws.

the class S3CrtAsyncHttpClient method execute.

@Override
public CompletableFuture<Void> execute(AsyncExecuteRequest request) {
    CompletableFuture<Void> executeFuture = new CompletableFuture<>();
    HttpRequest httpRequest = toCrtRequest(request);
    S3CrtResponseHandlerAdapter responseHandler = new S3CrtResponseHandlerAdapter(executeFuture, request.responseHandler());
    S3MetaRequestOptions.MetaRequestType requestType = requestType(request);
    S3MetaRequestOptions requestOptions = new S3MetaRequestOptions().withHttpRequest(httpRequest).withMetaRequestType(requestType).withResponseHandler(responseHandler);
    try (S3MetaRequest s3MetaRequest = crtS3Client.makeMetaRequest(requestOptions)) {
        closeResourcesWhenComplete(executeFuture, s3MetaRequest);
    }
    return executeFuture;
}

Example 3 with S3MetaRequest

use of software.amazon.awssdk.crt.s3.S3MetaRequest in project aws-sdk-java-v2 by aws.

the class S3CrtAsyncHttpClient method execute.

@Override
public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
    CompletableFuture<Void> executeFuture = new CompletableFuture<>();
    URI uri = asyncRequest.request().getUri();
    HttpRequest httpRequest = toCrtRequest(uri, asyncRequest);
    S3CrtResponseHandlerAdapter responseHandler = new S3CrtResponseHandlerAdapter(executeFuture, asyncRequest.responseHandler());
    S3MetaRequestOptions.MetaRequestType requestType = requestType(asyncRequest);
    S3MetaRequestOptions requestOptions = new S3MetaRequestOptions().withHttpRequest(httpRequest).withMetaRequestType(requestType).withResponseHandler(responseHandler).withEndpoint(s3NativeClientConfiguration.endpointOverride());
    try (S3MetaRequest s3MetaRequest = crtS3Client.makeMetaRequest(requestOptions)) {
        closeResourcesWhenComplete(executeFuture, s3MetaRequest, responseHandler);
    }
    return executeFuture;
}

Example 4 with S3MetaRequest

use of software.amazon.awssdk.crt.s3.S3MetaRequest in project aws-crt-java by awslabs.

the class S3NativeClient method putObject.

public CompletableFuture<PutObjectOutput> putObject(PutObjectRequest request, final RequestDataSupplier requestDataSupplier) {
    final CompletableFuture<PutObjectOutput> resultFuture = new CompletableFuture<>();
    final PutObjectOutput.Builder resultBuilder = PutObjectOutput.builder();
    HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() {

        @Override
        public boolean sendRequestBody(final ByteBuffer outBuffer) {
            try {
                return requestDataSupplier.getRequestBytes(outBuffer);
            } catch (Exception e) {
                resultFuture.completeExceptionally(e);
                return true;
            }
        }

        @Override
        public boolean resetPosition() {
            try {
                return requestDataSupplier.resetPosition();
            } catch (Exception e) {
                return false;
            }
        }

        @Override
        public long getLength() {
            return request.contentLength();
        }
    };
    final List<HttpHeader> headers = new LinkedList<>();
    // TODO: additional logic needed for *special* partitions
    headers.add(new HttpHeader("Host", request.bucket() + ".s3." + signingRegion + ".amazonaws.com"));
    populatePutObjectRequestHeaders(header -> headers.add(header), request);
    addCustomHeaders(headers, request.customHeaders());
    String encodedPath = getEncodedPath(request.key(), request.customQueryParameters());
    HttpRequest httpRequest = new HttpRequest("PUT", encodedPath, headers.toArray(new HttpHeader[0]), payloadStream);
    final S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {

        @Override
        public void onResponseHeaders(final int statusCode, final HttpHeader[] headers) {
            for (int headerIndex = 0; headerIndex < headers.length; ++headerIndex) {
                try {
                    populatePutObjectOutputHeader(resultBuilder, headers[headerIndex]);
                } catch (Exception e) {
                    resultFuture.completeExceptionally(new RuntimeException(String.format("Could not process response header {%s}: " + headers[headerIndex].getName()), e));
                }
            }
            requestDataSupplier.onResponseHeaders(statusCode, headers);
        }

        @Override
        public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) {
            return 0;
        }

        @Override
        public void onFinished(int errorCode, int responseStatus, byte[] errorPayload) {
            CrtS3RuntimeException ex = null;
            try {
                if (errorCode != CRT.AWS_CRT_SUCCESS) {
                    ex = new CrtS3RuntimeException(errorCode, responseStatus, errorPayload);
                    requestDataSupplier.onException(ex);
                } else {
                    requestDataSupplier.onFinished();
                }
            } catch (Exception e) {
            /* ignore user callback exception */
            } finally {
                if (ex != null) {
                    resultFuture.completeExceptionally(ex);
                } else {
                    resultFuture.complete(resultBuilder.build());
                }
            }
        }
    };
    S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions().withMetaRequestType(S3MetaRequestOptions.MetaRequestType.PUT_OBJECT).withHttpRequest(httpRequest).withResponseHandler(responseHandler);
    try (final S3MetaRequest metaRequest = s3Client.makeMetaRequest(metaRequestOptions)) {
        addCancelCheckToFuture(resultFuture, metaRequest);
        return resultFuture;
    }
}

Example 5 with S3MetaRequest

use of software.amazon.awssdk.crt.s3.S3MetaRequest in project aws-crt-java by awslabs.

the class S3ClientTest method benchmarkS3Get.

@Test
public void benchmarkS3Get() {
    skipIfNetworkUnavailable();
    Assume.assumeTrue(hasAwsCredentials());
    Assume.assumeNotNull(System.getProperty("aws.crt.s3.benchmark"));
    // Log.initLoggingToStdout(LogLevel.Trace);
    // Override defaults with values from system properties, via -D on mvn
    // commandline
    final int threadCount = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.threads", "0"));
    final String region = System.getProperty("aws.crt.s3.benchmark.region", "us-west-2");
    final String bucket = System.getProperty("aws.crt.s3.benchmark.bucket", (region == "us-west-2") ? "aws-crt-canary-bucket" : String.format("aws-crt-canary-bucket-%s", region));
    final String endpoint = System.getProperty("aws.crt.s3.benchmark.endpoint", String.format("%s.s3.%s.amazonaws.com", bucket, region));
    final String objectName = System.getProperty("aws.crt.s3.benchmark.object", "crt-canary-obj-single-part-9223372036854775807");
    final boolean useTls = Boolean.parseBoolean(System.getProperty("aws.crt.s3.benchmark.tls", "false"));
    final double expectedGbps = Double.parseDouble(System.getProperty("aws.crt.s3.benchmark.gbps", "10"));
    final int numTransfers = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.transfers", "16"));
    final int concurrentTransfers = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.concurrent", "16"));
    /* should be 1.6 * expectedGbps */
    // avg of .3Gbps per connection, 32 connections per vip, 5 seconds per vip
    // resolution
    final int vipsNeeded = (int) Math.ceil(expectedGbps / 0.5 / 10);
    final int sampleDelay = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.warmup", new Integer((int) Math.ceil(vipsNeeded / 5)).toString()));
    System.out.println(String.format("REGION=%s, WARMUP=%s", region, sampleDelay));
    // Ignore stats during warm up time, they skew results
    TransferStats.global.withSampleDelay(Duration.ofSeconds(sampleDelay));
    try (TlsContext tlsCtx = createTlsContextOptions(getContext().trustStore)) {
        S3ClientOptions clientOptions = new S3ClientOptions().withRegion(region).withEndpoint(endpoint).withThroughputTargetGbps(expectedGbps).withTlsContext(useTls ? tlsCtx : null);
        try (S3Client client = createS3Client(clientOptions, threadCount)) {
            HttpHeader[] headers = { new HttpHeader("Host", endpoint) };
            HttpRequest httpRequest = new HttpRequest("GET", String.format("/%s", objectName), headers, null);
            List<CompletableFuture<TransferStats>> requestFutures = new LinkedList<>();
            // Each meta request will acquire a slot, and release it when it's done
            Semaphore concurrentSlots = new Semaphore(concurrentTransfers);
            for (int transferIdx = 0; transferIdx < numTransfers; ++transferIdx) {
                try {
                    concurrentSlots.acquire();
                } catch (InterruptedException ex) {
                    Assert.fail(ex.toString());
                }
                final int myIdx = transferIdx;
                CompletableFuture<TransferStats> onFinishedFuture = new CompletableFuture<>();
                requestFutures.add(onFinishedFuture);
                S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {

                    TransferStats stats = new TransferStats();

                    @Override
                    public int onResponseBody(ByteBuffer bodyBytesIn, long objectRangeStart, long objectRangeEnd) {
                        stats.recordRead(bodyBytesIn.remaining());
                        return 0;
                    }

                    @Override
                    public void onFinished(int errorCode, int responseStatus, byte[] errorPayload) {
                        // release the slot first
                        concurrentSlots.release();
                        if (errorCode != 0) {
                            onFinishedFuture.completeExceptionally(new CrtS3RuntimeException(errorCode, responseStatus, errorPayload));
                            return;
                        }
                        synchronized (System.out) {
                            System.out.println(String.format("Transfer %d:  Avg: %.3f Gbps Peak: %.3f Gbps First Byte: %dms", myIdx + 1, stats.avgGbps(), stats.peakGbps(), stats.latency()));
                        }
                        onFinishedFuture.complete(stats);
                    }
                };
                S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions().withMetaRequestType(MetaRequestType.GET_OBJECT).withHttpRequest(httpRequest).withResponseHandler(responseHandler);
                try (S3MetaRequest metaRequest = client.makeMetaRequest(metaRequestOptions)) {
                }
            }
            // Finish each future, and deduct failures from completedTransfers
            int completedTransfers = numTransfers;
            double totalGbps = 0;
            for (CompletableFuture<TransferStats> request : requestFutures) {
                try {
                    request.join();
                    totalGbps += request.get().avgGbps();
                } catch (CompletionException | InterruptedException | ExecutionException ex) {
                    System.out.println(ex.toString());
                    Throwable cause = ex.getCause();
                    if (cause != ex && cause != null) {
                        System.out.println(cause.toString());
                    }
                    cause = cause.getCause();
                    if (cause != null && cause != ex) {
                        System.out.println(cause.toString());
                    }
                    --completedTransfers;
                }
            }
            // Dump overall stats
            TransferStats overall = TransferStats.global;
            System.out.println(String.format("%d/%d successful transfers", completedTransfers, numTransfers));
            System.out.println(String.format("Avg: %.3f Gbps", overall.avgGbps()));
            System.out.println(String.format("Peak: %.3f Gbps", overall.peakGbps()));
            System.out.println(String.format("P90: %.3f Gbps (stddev: %.3f)", overall.p90Gbps(), overall.stddev()));
            System.out.println(String.format("Avg Latency: %dms", overall.latency()));
            System.out.flush();
            try {
                File csvFile = new File("samples.csv");
                try (PrintWriter writer = new PrintWriter(csvFile)) {
                    writer.println("seconds,gbps");
                    AtomicInteger idx = new AtomicInteger(0);
                    overall.allSamples().mapToObj((gbps) -> {
                        return String.format("%d,%.3f", idx.getAndIncrement(), gbps);
                    }).forEach(writer::println);
                }
            } catch (FileNotFoundException ex) {
                System.err.println(ex.getMessage());
            }
        }
    }
}