Examples with HttpPostRequestEncoder org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpPostRequestEncoder used on opensource projects

Example 16 with HttpPostRequestEncoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpPostRequestEncoder in project ambry by linkedin.

the class FrontendIntegrationTestBase method createEncoder.

/**
 * Creates a {@link HttpPostRequestEncoder} that encodes the given {@code request} and {@code blobContent}.
 * @param request the {@link HttpRequest} containing headers and other metadata about the request.
 * @param blobContent the {@link ByteBuffer} that represents the content of the blob.
 * @param usermetadata the {@link ByteBuffer} that represents user metadata
 * @return a {@link HttpPostRequestEncoder} that can encode the {@code request} and {@code blobContent}.
 * @throws HttpPostRequestEncoder.ErrorDataEncoderException
 * @throws IOException
 */
HttpPostRequestEncoder createEncoder(HttpRequest request, ByteBuffer blobContent, ByteBuffer usermetadata) throws HttpPostRequestEncoder.ErrorDataEncoderException, IOException {
    HttpDataFactory httpDataFactory = new DefaultHttpDataFactory(false);
    HttpPostRequestEncoder encoder = new HttpPostRequestEncoder(httpDataFactory, request, true);
    FileUpload fileUpload = new MemoryFileUpload(RestUtils.MultipartPost.BLOB_PART, RestUtils.MultipartPost.BLOB_PART, "application/octet-stream", "", StandardCharsets.UTF_8, blobContent.remaining());
    fileUpload.setContent(Unpooled.wrappedBuffer(blobContent));
    encoder.addBodyHttpData(fileUpload);
    fileUpload = new MemoryFileUpload(RestUtils.MultipartPost.USER_METADATA_PART, RestUtils.MultipartPost.USER_METADATA_PART, "application/octet-stream", "", StandardCharsets.UTF_8, usermetadata.remaining());
    fileUpload.setContent(Unpooled.wrappedBuffer(usermetadata));
    encoder.addBodyHttpData(fileUpload);
    return encoder;
}

Example 17 with HttpPostRequestEncoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpPostRequestEncoder in project ambry by linkedin.

the class NettyMultipartRequestTest method sizeLimitationTest.

/**
 * Tests to make sure the max allowed size for multipart requests is enforced.
 * @throws Exception
 */
// Disabling test because the encoded size is different at different times and it is hard to predict. Need a better
// test
// @Test
public void sizeLimitationTest() throws Exception {
    int blobPartSize = 1024;
    byte[] bytes = TestUtils.getRandomBytes(blobPartSize);
    int encodedSize = getEncodedSize(bytes);
    long[] maxSizesAllowed = { encodedSize + 1, encodedSize, encodedSize - 1, 0 };
    for (long maxSizeAllowed : maxSizesAllowed) {
        InMemoryFile[] files = { new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(bytes)) };
        HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
        HttpPostRequestEncoder encoder = createEncoder(httpRequest, files);
        NettyMultipartRequest request = new NettyMultipartRequest(encoder.finalizeRequest(), new MockChannel(), NETTY_METRICS, Collections.emptySet(), maxSizeAllowed);
        assertTrue("Request channel is not open", request.isOpen());
        long currentSizeAdded = 0;
        boolean failedToAdd = false;
        while (!encoder.isEndOfInput()) {
            HttpContent httpContent = encoder.readChunk(PooledByteBufAllocator.DEFAULT);
            int readableBytes = httpContent.content().readableBytes();
            if (currentSizeAdded + readableBytes <= maxSizeAllowed) {
                request.addContent(httpContent);
            } else {
                assertTrue("Max size [" + maxSizeAllowed + "] must be lesser than content size: " + encodedSize, maxSizeAllowed < encodedSize);
                try {
                    request.addContent(httpContent);
                    fail("Should have failed to add content of size [" + encodedSize + "] because it is over the max size allowed: " + maxSizeAllowed);
                } catch (RestServiceException e) {
                    failedToAdd = true;
                    assertEquals("Unexpected RestServiceErrorCode", RestServiceErrorCode.RequestTooLarge, e.getErrorCode());
                    break;
                }
            }
            currentSizeAdded += readableBytes;
        }
        assertEquals("Success state not as expected. maxSizeAllowed=[" + maxSizeAllowed + "], encodedSize expected=[" + encodedSize + "], actual size added=[" + currentSizeAdded + "]", maxSizeAllowed < encodedSize, failedToAdd);
    }
}

Example 18 with HttpPostRequestEncoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpPostRequestEncoder in project ambry by linkedin.

the class NettyMultipartRequestTest method getEncodedSize.

// sizeLimitationTest() helpers
/**
 * Gets the encoded size for a set of bytes
 * @param bytes the bytes to encode.
 * @return the encoded size
 * @throws Exception
 */
private int getEncodedSize(byte[] bytes) throws Exception {
    int encodedSize = 0;
    InMemoryFile[] files = { new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(bytes)) };
    HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
    HttpPostRequestEncoder encoder = createEncoder(httpRequest, files);
    encoder.finalizeRequest();
    while (!encoder.isEndOfInput()) {
        HttpContent httpContent = encoder.readChunk(PooledByteBufAllocator.DEFAULT);
        encodedSize += httpContent.content().readableBytes();
    }
    return encodedSize;
}

Example 19 with HttpPostRequestEncoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpPostRequestEncoder in project ambry by linkedin.

the class NettyMultipartRequestTest method createEncoder.

/**
 * Creates a {@link HttpPostRequestEncoder} that encodes the given {@code request} and {@code parts}.
 * @param request the {@link HttpRequest} containing headers and other metadata about the request.
 * @param parts the {@link InMemoryFile}s that will form the parts of the request.
 * @return a {@link HttpPostRequestEncoder} that can encode the {@code request} and {@code parts}.
 * @throws HttpPostRequestEncoder.ErrorDataEncoderException
 * @throws IOException
 */
private HttpPostRequestEncoder createEncoder(HttpRequest request, InMemoryFile[] parts) throws HttpPostRequestEncoder.ErrorDataEncoderException, IOException {
    HttpDataFactory httpDataFactory = new DefaultHttpDataFactory(false);
    HttpPostRequestEncoder encoder = new HttpPostRequestEncoder(httpDataFactory, request, true);
    if (parts != null) {
        for (InMemoryFile part : parts) {
            FileUpload fileUpload = new MemoryFileUpload(part.name, part.name, "application/octet-stream", "", Charset.forName("UTF-8"), part.content.remaining());
            fileUpload.setContent(Unpooled.wrappedBuffer(part.content));
            encoder.addBodyHttpData(fileUpload);
        }
    }
    return encoder;
}

Example 20 with HttpPostRequestEncoder

use of org.apache.flink.shaded.netty4.io.netty.handler.codec.http.multipart.HttpPostRequestEncoder in project ambry by linkedin.

the class NettyMultipartRequestTest method prepareTest.

/**
 * Tests different scenarios with {@link NettyMultipartRequest#prepare()}.
 * Currently tests:
 * 1. Idempotency of {@link NettyMultipartRequest#prepare()}.
 * 2. Exception scenarios of {@link NettyMultipartRequest#prepare()}.
 * @throws Exception
 */
@Test
public void prepareTest() throws Exception {
    // prepare half baked data
    HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
    HttpPostRequestEncoder encoder = createEncoder(httpRequest, null);
    NettyMultipartRequest request = new NettyMultipartRequest(encoder.finalizeRequest(), new MockChannel(), NETTY_METRICS, Collections.emptySet(), Long.MAX_VALUE);
    assertTrue("Request channel is not open", request.isOpen());
    // insert random data
    HttpContent httpContent = new DefaultHttpContent(Unpooled.wrappedBuffer(TestUtils.getRandomBytes(10)));
    request.addContent(httpContent);
    // prepare should fail
    try {
        request.prepare();
        fail("Preparing request should have failed");
    } catch (HttpPostRequestDecoder.NotEnoughDataDecoderException e) {
        assertEquals("Reference count is not as expected", 1, httpContent.refCnt());
    } finally {
        closeRequestAndValidate(request);
    }
    // more than one blob part
    HttpHeaders httpHeaders = new DefaultHttpHeaders();
    httpHeaders.set(RestUtils.Headers.BLOB_SIZE, 256);
    InMemoryFile[] files = new InMemoryFile[2];
    files[0] = new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(TestUtils.getRandomBytes(256)));
    files[1] = new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(TestUtils.getRandomBytes(256)));
    request = createRequest(httpHeaders, files);
    assertEquals("Request size does not match", 256, request.getSize());
    try {
        request.prepare();
        fail("Prepare should have failed because there was more than one " + RestUtils.MultipartPost.BLOB_PART);
    } catch (RestServiceException e) {
        assertEquals("Unexpected RestServiceErrorCode", RestServiceErrorCode.BadRequest, e.getErrorCode());
    } finally {
        closeRequestAndValidate(request);
    }
    // more than one part named "part-1"
    files = new InMemoryFile[2];
    files[0] = new InMemoryFile("Part-1", ByteBuffer.wrap(TestUtils.getRandomBytes(256)));
    files[1] = new InMemoryFile("Part-1", ByteBuffer.wrap(TestUtils.getRandomBytes(256)));
    request = createRequest(null, files);
    try {
        request.prepare();
        fail("Prepare should have failed because there was more than one part named Part-1");
    } catch (RestServiceException e) {
        assertEquals("Unexpected RestServiceErrorCode", RestServiceErrorCode.BadRequest, e.getErrorCode());
    } finally {
        closeRequestAndValidate(request);
    }
    // size of blob does not match the advertized size
    httpHeaders = new DefaultHttpHeaders();
    httpHeaders.set(RestUtils.Headers.BLOB_SIZE, 256);
    files = new InMemoryFile[1];
    files[0] = new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(TestUtils.getRandomBytes(128)));
    request = createRequest(httpHeaders, files);
    try {
        request.prepare();
        fail("Prepare should have failed because the size advertised does not match the actual size");
    } catch (RestServiceException e) {
        assertEquals("Unexpected RestServiceErrorCode", RestServiceErrorCode.BadRequest, e.getErrorCode());
    } finally {
        closeRequestAndValidate(request);
    }
    // non fileupload (file attribute present)
    httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
    httpRequest.headers().set(RestUtils.Headers.BLOB_SIZE, 256);
    files = new InMemoryFile[1];
    files[0] = new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(TestUtils.getRandomBytes(256)));
    encoder = createEncoder(httpRequest, files);
    encoder.addBodyAttribute("dummyKey", "dummyValue");
    request = new NettyMultipartRequest(encoder.finalizeRequest(), new MockChannel(), NETTY_METRICS, Collections.emptySet(), Long.MAX_VALUE);
    assertTrue("Request channel is not open", request.isOpen());
    while (!encoder.isEndOfInput()) {
        // Sending null for ctx because the encoder is OK with that.
        request.addContent(encoder.readChunk(PooledByteBufAllocator.DEFAULT));
    }
    try {
        request.prepare();
        fail("Prepare should have failed because there was non fileupload");
    } catch (RestServiceException e) {
        assertEquals("Unexpected RestServiceErrorCode", RestServiceErrorCode.BadRequest, e.getErrorCode());
    } finally {
        closeRequestAndValidate(request);
    }
    // size of blob is not set. Prepare should succeed.
    httpHeaders = new DefaultHttpHeaders();
    files = new InMemoryFile[1];
    files[0] = new InMemoryFile(RestUtils.MultipartPost.BLOB_PART, ByteBuffer.wrap(TestUtils.getRandomBytes(128)));
    request = createRequest(httpHeaders, files);
    try {
        request.prepare();
    } finally {
        closeRequestAndValidate(request);
    }
}