Examples with HttpRequestBodyStream software.amazon.awssdk.crt.http.HttpRequestBodyStream used on opensource projects

Example 1 with HttpRequestBodyStream

use of software.amazon.awssdk.crt.http.HttpRequestBodyStream in project aws-crt-java by awslabs.

the class SigningTest method makeBodyStreamFromString.

private HttpRequestBodyStream makeBodyStreamFromString(String body) {
    final ByteBuffer bodyBytesIn = ByteBuffer.wrap(body.getBytes(StandardCharsets.UTF_8));
    HttpRequestBodyStream bodyStream = new HttpRequestBodyStream() {

        @Override
        public boolean sendRequestBody(ByteBuffer bodyBytesOut) {
            transferData(bodyBytesIn, bodyBytesOut);
            return bodyBytesIn.remaining() == 0;
        }

        @Override
        public boolean resetPosition() {
            bodyBytesIn.position(0);
            return true;
        }
    };
    return bodyStream;
}

Example 2 with HttpRequestBodyStream

use of software.amazon.awssdk.crt.http.HttpRequestBodyStream in project aws-crt-java by awslabs.

the class SigningTest method testTrailingHeadersSigv4Signing.

@Test
public void testTrailingHeadersSigv4Signing() throws Exception {
    HttpRequest request = createChunkedTestRequest();
    CompletableFuture<HttpRequest> result = AwsSigner.signRequest(request, createChunkedRequestSigningConfig());
    HttpRequest signedRequest = result.get();
    assertNotNull(signedRequest);
    assertTrue(hasHeaderWithValue(signedRequest, "Authorization", EXPECTED_CHUNK_REQUEST_AUTHORIZATION_HEADER));
    /*
        * If the authorization header is equal then certainly we can assume the
        * signature value
        */
    byte[] signature = EXPECTED_REQUEST_SIGNATURE;
    HttpRequestBodyStream chunk1 = createChunk1Stream();
    CompletableFuture<AwsSigningResult> chunk1Result = AwsSigner.sign(chunk1, signature, createChunkSigningConfig());
    signature = chunk1Result.get().getSignature();
    assertTrue(Arrays.equals(signature, EXPECTED_FIRST_CHUNK_SIGNATURE));
    HttpRequestBodyStream chunk2 = createChunk2Stream();
    CompletableFuture<AwsSigningResult> chunk2Result = AwsSigner.sign(chunk2, signature, createChunkSigningConfig());
    signature = chunk2Result.get().getSignature();
    assertTrue(Arrays.equals(signature, EXPECTED_SECOND_CHUNK_SIGNATURE));
    CompletableFuture<AwsSigningResult> finalChunkResult = AwsSigner.sign((HttpRequestBodyStream) null, signature, createChunkSigningConfig());
    signature = finalChunkResult.get().getSignature();
    assertTrue(Arrays.equals(signature, EXPECTED_FINAL_CHUNK_SIGNATURE));
    List<HttpHeader> trailingHeaders = createTrailingHeaders();
    AwsSigningConfig trailingHeadersSigningConfig = createTrailingHeadersSigningConfig();
    CompletableFuture<AwsSigningResult> trailingHeadersResult = AwsSigner.sign(trailingHeaders, signature, trailingHeadersSigningConfig);
    signature = trailingHeadersResult.get().getSignature();
    assertTrue(Arrays.equals(signature, EXPECTED_TRAILING_HEADERS_SIGNATURE));
}

Example 3 with HttpRequestBodyStream

use of software.amazon.awssdk.crt.http.HttpRequestBodyStream in project aws-crt-java by awslabs.

the class SigningTest method testChunkedSigv4Signing.

@Test
public void testChunkedSigv4Signing() throws Exception {
    HttpRequest request = createChunkedTestRequest();
    CompletableFuture<HttpRequest> result = AwsSigner.signRequest(request, createChunkedRequestSigningConfig());
    HttpRequest signedRequest = result.get();
    assertNotNull(signedRequest);
    assertTrue(hasHeaderWithValue(signedRequest, "Authorization", EXPECTED_CHUNK_REQUEST_AUTHORIZATION_HEADER));
    /* If the authorization header is equal then certainly we can assume the signature value */
    byte[] signature = EXPECTED_REQUEST_SIGNATURE;
    HttpRequestBodyStream chunk1 = createChunk1Stream();
    CompletableFuture<byte[]> chunk1Result = AwsSigner.signChunk(chunk1, signature, createChunkSigningConfig());
    signature = chunk1Result.get();
    assertTrue(Arrays.equals(signature, EXPECTED_FIRST_CHUNK_SIGNATURE));
    HttpRequestBodyStream chunk2 = createChunk2Stream();
    CompletableFuture<byte[]> chunk2Result = AwsSigner.signChunk(chunk2, signature, createChunkSigningConfig());
    signature = chunk2Result.get();
    assertTrue(Arrays.equals(signature, EXPECTED_SECOND_CHUNK_SIGNATURE));
    CompletableFuture<byte[]> finalChunkResult = AwsSigner.signChunk(null, signature, createChunkSigningConfig());
    signature = finalChunkResult.get();
    assertTrue(Arrays.equals(signature, EXPECTED_FINAL_CHUNK_SIGNATURE));
}

Example 4 with HttpRequestBodyStream

use of software.amazon.awssdk.crt.http.HttpRequestBodyStream in project aws-crt-java by awslabs.

the class SigningTest method testTrailingHeadersSigv4aSigning.

@Test
public void testTrailingHeadersSigv4aSigning() throws Exception {
    HttpRequest request = createChunkedTrailerTestRequest();
    AwsSigningConfig chunkedRequestSigningConfig = createChunkedRequestSigningConfig();
    chunkedRequestSigningConfig.setAlgorithm(AwsSigningConfig.AwsSigningAlgorithm.SIGV4_ASYMMETRIC);
    chunkedRequestSigningConfig.setSignedBodyValue(AwsSigningConfig.AwsSignedBodyValue.STREAMING_AWS4_ECDSA_P256_SHA256_PAYLOAD_TRAILER);
    CompletableFuture<AwsSigningResult> result = AwsSigner.sign(request, chunkedRequestSigningConfig);
    HttpRequest signedRequest = result.get().getSignedRequest();
    assertNotNull(signedRequest);
    byte[] requestSignature = result.get().getSignature();
    assertTrue(AwsSigningUtils.verifySigv4aEcdsaSignature(request, CHUNKED_TRAILER_SIGV4A_CANONICAL_REQUEST, chunkedRequestSigningConfig, requestSignature, CHUNKED_SIGV4A_TEST_ECC_PUB_X, CHUNKED_SIGV4A_TEST_ECC_PUB_Y));
    HttpRequestBodyStream chunk1 = createChunk1Stream();
    AwsSigningConfig chunkSigningConfig = createChunkSigningConfig();
    chunkSigningConfig.setAlgorithm(AwsSigningConfig.AwsSigningAlgorithm.SIGV4_ASYMMETRIC);
    CompletableFuture<AwsSigningResult> chunk1Result = AwsSigner.sign(chunk1, requestSignature, chunkSigningConfig);
    byte[] chunk1StringToSign = buildChunkStringToSign(requestSignature, CHUNK1_STS_POST_SIGNATURE);
    assertTrue(AwsSigningUtils.verifyRawSha256EcdsaSignature(chunk1StringToSign, chunk1Result.get().getSignature(), CHUNKED_SIGV4A_TEST_ECC_PUB_X, CHUNKED_SIGV4A_TEST_ECC_PUB_Y));
    HttpRequestBodyStream chunk2 = createChunk2Stream();
    CompletableFuture<AwsSigningResult> chunk2Result = AwsSigner.sign(chunk2, chunk1Result.get().getSignature(), chunkSigningConfig);
    byte[] chunk2StringToSign = buildChunkStringToSign(chunk1Result.get().getSignature(), CHUNK2_STS_POST_SIGNATURE);
    assertTrue(AwsSigningUtils.verifyRawSha256EcdsaSignature(chunk2StringToSign, chunk2Result.get().getSignature(), CHUNKED_SIGV4A_TEST_ECC_PUB_X, CHUNKED_SIGV4A_TEST_ECC_PUB_Y));
    CompletableFuture<AwsSigningResult> chunk3Result = AwsSigner.sign((HttpRequestBodyStream) null, chunk2Result.get().getSignature(), chunkSigningConfig);
    byte[] chunk3StringToSign = buildChunkStringToSign(chunk2Result.get().getSignature(), CHUNK3_STS_POST_SIGNATURE);
    assertTrue(AwsSigningUtils.verifyRawSha256EcdsaSignature(chunk3StringToSign, chunk3Result.get().getSignature(), CHUNKED_SIGV4A_TEST_ECC_PUB_X, CHUNKED_SIGV4A_TEST_ECC_PUB_Y));
    List<HttpHeader> trailingHeaders = createTrailingHeaders();
    AwsSigningConfig trailingHeadersSigningConfig = createTrailingHeadersSigningConfig();
    trailingHeadersSigningConfig.setAlgorithm(AwsSigningConfig.AwsSigningAlgorithm.SIGV4_ASYMMETRIC);
    CompletableFuture<AwsSigningResult> trailingHeadersResult = AwsSigner.sign(trailingHeaders, chunk3Result.get().getSignature(), trailingHeadersSigningConfig);
    byte[] trailingHeadersStringToSign = buildTrailingHeadersStringToSign(chunk3Result.get().getSignature(), TRAILING_HEADERS_STS_POST_SIGNATURE);
    assertTrue(AwsSigningUtils.verifyRawSha256EcdsaSignature(trailingHeadersStringToSign, trailingHeadersResult.get().getSignature(), CHUNKED_SIGV4A_TEST_ECC_PUB_X, CHUNKED_SIGV4A_TEST_ECC_PUB_Y));
}

Example 5 with HttpRequestBodyStream

use of software.amazon.awssdk.crt.http.HttpRequestBodyStream in project aws-crt-java by awslabs.

the class S3ClientTest method benchmarkS3Put.

@Test
public void benchmarkS3Put() {
    skipIfNetworkUnavailable();
    Assume.assumeTrue(hasAwsCredentials());
    Assume.assumeNotNull(System.getProperty("aws.crt.s3.benchmark"));
    // Override defaults with values from system properties, via -D on mvn
    // commandline
    final int threadCount = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.threads", "0"));
    final String region = System.getProperty("aws.crt.s3.benchmark.region", "us-west-2");
    final String bucket = System.getProperty("aws.crt.s3.benchmark.bucket", (region == "us-west-2") ? "aws-crt-canary-bucket" : String.format("aws-crt-canary-bucket-%s", region));
    final String endpoint = System.getProperty("aws.crt.s3.benchmark.endpoint", String.format("%s.s3.%s.amazonaws.com", bucket, region));
    final boolean useTls = Boolean.parseBoolean(System.getProperty("aws.crt.s3.benchmark.tls", "false"));
    final double expectedGbps = Double.parseDouble(System.getProperty("aws.crt.s3.benchmark.gbps", "10"));
    final int numTransfers = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.transfers", "16"));
    final int concurrentTransfers = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.concurrent", "16"));
    /* should be 1.6 * expectedGbps */
    // avg of .3Gbps per connection, 32 connections per vip, 5 seconds per vip
    // resolution
    final int vipsNeeded = (int) Math.ceil(expectedGbps / 0.5 / 10);
    final int sampleDelay = Integer.parseInt(System.getProperty("aws.crt.s3.benchmark.warmup", new Integer((int) Math.ceil(vipsNeeded / 5)).toString()));
    System.out.println(String.format("REGION=%s, WARMUP=%s", region, sampleDelay));
    // Ignore stats during warm up time, they skew results
    TransferStats.global.withSampleDelay(Duration.ofSeconds(sampleDelay));
    try (TlsContext tlsCtx = createTlsContextOptions(getContext().trustStore)) {
        S3ClientOptions clientOptions = new S3ClientOptions().withRegion(region).withEndpoint(endpoint).withThroughputTargetGbps(expectedGbps).withTlsContext(useTls ? tlsCtx : null);
        try (S3Client client = createS3Client(clientOptions, threadCount)) {
            List<CompletableFuture<TransferStats>> requestFutures = new LinkedList<>();
            // Each meta request will acquire a slot, and release it when it's done
            Semaphore concurrentSlots = new Semaphore(concurrentTransfers);
            for (int transferIdx = 0; transferIdx < numTransfers; ++transferIdx) {
                try {
                    concurrentSlots.acquire();
                } catch (InterruptedException ex) {
                    Assert.fail(ex.toString());
                }
                final int myIdx = transferIdx;
                CompletableFuture<TransferStats> onFinishedFuture = new CompletableFuture<>();
                requestFutures.add(onFinishedFuture);
                S3MetaRequestResponseHandler responseHandler = new S3MetaRequestResponseHandler() {

                    TransferStats stats = new TransferStats();

                    @Override
                    public void onFinished(int errorCode, int responseStatus, byte[] errorPayload) {
                        // release the slot first
                        concurrentSlots.release();
                        if (errorCode != 0) {
                            onFinishedFuture.completeExceptionally(new CrtS3RuntimeException(errorCode, responseStatus, errorPayload));
                            return;
                        }
                        synchronized (System.out) {
                            System.out.println(String.format("Transfer %d finished.", myIdx + 1));
                        }
                        onFinishedFuture.complete(stats);
                    }
                };
                final long payloadSize = 5L * 1024L * 1024L * 1024L;
                final String payloadString = "This is an S3 Test.  This is an S3 Test.  This is an S3 Test.  This is an S3 Test.";
                HttpRequestBodyStream payloadStream = new HttpRequestBodyStream() {

                    private long remainingBody = payloadSize;

                    @Override
                    public boolean sendRequestBody(ByteBuffer outBuffer) {
                        byte[] payloadBytes = null;
                        try {
                            payloadBytes = payloadString.getBytes("ASCII");
                        } catch (Exception ex) {
                            System.out.println("Encountered error trying to get payload bytes.");
                            return true;
                        }
                        while (remainingBody > 0 && outBuffer.remaining() > 0) {
                            long amtToTransfer = Math.min(remainingBody, (long) outBuffer.remaining());
                            amtToTransfer = Math.min(amtToTransfer, (long) payloadBytes.length);
                            // Transfer the data
                            outBuffer.put(payloadBytes, 0, (int) amtToTransfer);
                            remainingBody -= amtToTransfer;
                        }
                        return remainingBody == 0;
                    }

                    @Override
                    public boolean resetPosition() {
                        return true;
                    }

                    @Override
                    public long getLength() {
                        return payloadSize;
                    }
                };
                HttpHeader[] headers = { new HttpHeader("Host", endpoint), new HttpHeader("Content-Length", Long.valueOf(payloadSize).toString()) };
                HttpRequest httpRequest = new HttpRequest("PUT", String.format("/put_object_test_5GB_%d.txt", myIdx + 1), headers, payloadStream);
                S3MetaRequestOptions metaRequestOptions = new S3MetaRequestOptions().withMetaRequestType(MetaRequestType.PUT_OBJECT).withHttpRequest(httpRequest).withResponseHandler(responseHandler);
                try (S3MetaRequest metaRequest = client.makeMetaRequest(metaRequestOptions)) {
                }
            }
            // Finish each future, and deduct failures from completedTransfers
            int completedTransfers = numTransfers;
            for (CompletableFuture<TransferStats> request : requestFutures) {
                try {
                    request.join();
                } catch (CompletionException ex) {
                    System.out.println(ex.toString());
                    Throwable cause = ex.getCause();
                    if (cause != ex && cause != null) {
                        System.out.println(cause.toString());
                    }
                    cause = cause.getCause();
                    if (cause != null && cause != ex) {
                        System.out.println(cause.toString());
                    }
                    --completedTransfers;
                }
            }
        }
    }
}