Examples with ByteBufferInputStream com.github.ambry.utils.ByteBufferInputStream used on opensource projects

Example 6 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MockRouterCallback method testBlobNotFoundCase.

/**
 * Test the case where every server returns Blob_Not_Found. All servers must have been contacted,
 * due to cross-colo proxying.
 * @throws Exception
 */
@Test
public void testBlobNotFoundCase() throws Exception {
    NonBlockingRouter.currentOperationsCount.incrementAndGet();
    GetBlobInfoOperation op = new GetBlobInfoOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, null, routerCallback, kms, cryptoService, cryptoJobHandler, time);
    ArrayList<RequestInfo> requestListToFill = new ArrayList<>();
    requestRegistrationCallback.requestListToFill = requestListToFill;
    for (MockServer server : mockServerLayout.getMockServers()) {
        server.setServerErrorForAllRequests(ServerErrorCode.Blob_Not_Found);
    }
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        List<ResponseInfo> responses = sendAndWaitForResponses(requestListToFill);
        for (ResponseInfo responseInfo : responses) {
            GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new DataInputStream(new ByteBufferInputStream(responseInfo.getResponse())), mockClusterMap) : null;
            op.handleResponse(responseInfo, getResponse);
            if (op.isOperationComplete()) {
                break;
            }
        }
    }
    Assert.assertEquals("Must have attempted sending requests to all replicas", replicasCount, correlationIdToGetOperation.size());
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
    RouterException routerException = (RouterException) op.getOperationException();
    Assert.assertEquals(RouterErrorCode.BlobDoesNotExist, routerException.getErrorCode());
}

Example 7 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MockRouterCallback method testVariousErrors.

private void testVariousErrors(String dcWherePutHappened) throws Exception {
    NonBlockingRouter.currentOperationsCount.incrementAndGet();
    GetBlobInfoOperation op = new GetBlobInfoOperation(routerConfig, routerMetrics, mockClusterMap, responseHandler, blobId, options, null, routerCallback, kms, cryptoService, cryptoJobHandler, time);
    ArrayList<RequestInfo> requestListToFill = new ArrayList<>();
    requestRegistrationCallback.requestListToFill = requestListToFill;
    ArrayList<MockServer> mockServers = new ArrayList<>(mockServerLayout.getMockServers());
    // set the status to various server level or partition level errors (not Blob_Deleted or Blob_Expired).
    mockServers.get(0).setServerErrorForAllRequests(ServerErrorCode.Unknown_Error);
    mockServers.get(1).setServerErrorForAllRequests(ServerErrorCode.Unknown_Error);
    mockServers.get(2).setServerErrorForAllRequests(ServerErrorCode.IO_Error);
    mockServers.get(3).setServerErrorForAllRequests(ServerErrorCode.Blob_Not_Found);
    mockServers.get(4).setServerErrorForAllRequests(ServerErrorCode.Data_Corrupt);
    mockServers.get(5).setServerErrorForAllRequests(ServerErrorCode.Blob_Not_Found);
    mockServers.get(6).setServerErrorForAllRequests(ServerErrorCode.Blob_Not_Found);
    mockServers.get(7).setServerErrorForAllRequests(ServerErrorCode.Disk_Unavailable);
    mockServers.get(8).setServerErrorForAllRequests(ServerErrorCode.Unknown_Error);
    // clear the hard error in one of the servers in the datacenter where the put happened.
    for (int i = 0; i < mockServers.size(); i++) {
        MockServer mockServer = mockServers.get(i);
        if (mockServer.getDataCenter().equals(dcWherePutHappened)) {
            mockServer.setServerErrorForAllRequests(ServerErrorCode.No_Error);
            break;
        }
    }
    while (!op.isOperationComplete()) {
        op.poll(requestRegistrationCallback);
        List<ResponseInfo> responses = sendAndWaitForResponses(requestListToFill);
        for (ResponseInfo responseInfo : responses) {
            GetResponse getResponse = responseInfo.getError() == null ? GetResponse.readFrom(new DataInputStream(new ByteBufferInputStream(responseInfo.getResponse())), mockClusterMap) : null;
            op.handleResponse(responseInfo, getResponse);
            if (op.isOperationComplete()) {
                break;
            }
        }
    }
    Assert.assertTrue("Operation should be complete at this time", op.isOperationComplete());
    assertSuccess(op);
}

Example 8 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class HardDeleteRecoveryMetadata method getBlobPropertiesRecord.

private BlobProperties getBlobPropertiesRecord(MessageReadSet readSet, int readSetIndex, long relativeOffset, long blobPropertiesSize) throws MessageFormatException, IOException {
    /* Read the field from the channel */
    ByteBuffer blobProperties = ByteBuffer.allocate((int) blobPropertiesSize);
    readSet.writeTo(readSetIndex, Channels.newChannel(new ByteBufferOutputStream(blobProperties)), relativeOffset, blobPropertiesSize);
    blobProperties.flip();
    return deserializeBlobProperties(new ByteBufferInputStream(blobProperties));
}

Example 9 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class MessageSievingInputStreamTest method testDeletedBlobs.

private void testDeletedBlobs(short blobVersion, BlobType blobType) throws IOException, MessageFormatException {
    // MessageSievingInputStream contains put records for 2 valid blobs and 1 deleted blob
    // id1(put record for valid blob), id2(delete record) and id3(put record for valid blob)
    ArrayList<Short> versions = new ArrayList<>();
    versions.add(Message_Header_Version_V1);
    if (blobVersion != Blob_Version_V1) {
        versions.add(Message_Header_Version_V2);
    }
    try {
        for (short version : versions) {
            headerVersionToUse = version;
            // create message stream for blob 1
            StoreKey key1 = new MockId("id1");
            short accountId = Utils.getRandomShort(TestUtils.RANDOM);
            short containerId = Utils.getRandomShort(TestUtils.RANDOM);
            BlobProperties prop1 = new BlobProperties(10, "servid1", accountId, containerId, false);
            byte[] encryptionKey1 = new byte[100];
            TestUtils.RANDOM.nextBytes(encryptionKey1);
            byte[] usermetadata1 = new byte[1000];
            TestUtils.RANDOM.nextBytes(usermetadata1);
            int blobContentSize = 2000;
            byte[] data1 = new byte[blobContentSize];
            TestUtils.RANDOM.nextBytes(data1);
            if (blobVersion == Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
                ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
                data1 = byteBufferBlob.array();
                blobContentSize = data1.length;
            }
            ByteBufferInputStream stream1 = new ByteBufferInputStream(ByteBuffer.wrap(data1));
            MessageFormatInputStream messageFormatStream1 = (blobVersion == Blob_Version_V2) ? new PutMessageFormatInputStream(key1, ByteBuffer.wrap(encryptionKey1), prop1, ByteBuffer.wrap(usermetadata1), stream1, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key1, prop1, ByteBuffer.wrap(usermetadata1), stream1, blobContentSize, blobType);
            MessageInfo msgInfo1 = new MessageInfo(key1, messageFormatStream1.getSize(), accountId, containerId, prop1.getCreationTimeInMs());
            // create message stream for blob 2 and mark it as deleted
            StoreKey key2 = new MockId("id2");
            accountId = Utils.getRandomShort(TestUtils.RANDOM);
            containerId = Utils.getRandomShort(TestUtils.RANDOM);
            long deletionTimeMs = SystemTime.getInstance().milliseconds() + TestUtils.RANDOM.nextInt();
            MessageFormatInputStream messageFormatStream2 = new DeleteMessageFormatInputStream(key2, accountId, containerId, deletionTimeMs);
            MessageInfo msgInfo2 = new MessageInfo(key2, messageFormatStream2.getSize(), accountId, containerId, deletionTimeMs);
            // create message stream for blob 3
            StoreKey key3 = new MockId("id3");
            accountId = Utils.getRandomShort(TestUtils.RANDOM);
            containerId = Utils.getRandomShort(TestUtils.RANDOM);
            BlobProperties prop3 = new BlobProperties(10, "servid3", accountId, containerId, false);
            byte[] encryptionKey3 = new byte[100];
            TestUtils.RANDOM.nextBytes(encryptionKey3);
            byte[] usermetadata3 = new byte[1000];
            TestUtils.RANDOM.nextBytes(usermetadata3);
            blobContentSize = 2000;
            byte[] data3 = new byte[blobContentSize];
            TestUtils.RANDOM.nextBytes(data3);
            if (blobVersion == Blob_Version_V2 && blobType == BlobType.MetadataBlob) {
                ByteBuffer byteBufferBlob = MessageFormatTestUtils.getBlobContentForMetadataBlob(blobContentSize);
                data3 = byteBufferBlob.array();
                blobContentSize = data3.length;
            }
            ByteBufferInputStream stream3 = new ByteBufferInputStream(ByteBuffer.wrap(data3));
            MessageFormatInputStream messageFormatStream3 = (blobVersion == Blob_Version_V2) ? new PutMessageFormatInputStream(key3, ByteBuffer.wrap(encryptionKey3), prop3, ByteBuffer.wrap(usermetadata3), stream3, blobContentSize, blobType) : new PutMessageFormatBlobV1InputStream(key3, prop3, ByteBuffer.wrap(usermetadata3), stream3, blobContentSize, blobType);
            MessageInfo msgInfo3 = new MessageInfo(key3, messageFormatStream3.getSize(), accountId, containerId, prop3.getCreationTimeInMs());
            // create input stream for all blob messages together
            byte[] totalMessageContent = new byte[(int) messageFormatStream1.getSize() + (int) messageFormatStream2.getSize() + (int) messageFormatStream3.getSize()];
            messageFormatStream1.read(totalMessageContent, 0, (int) messageFormatStream1.getSize());
            messageFormatStream2.read(totalMessageContent, (int) messageFormatStream1.getSize(), (int) messageFormatStream2.getSize());
            messageFormatStream3.read(totalMessageContent, (int) messageFormatStream1.getSize() + (int) messageFormatStream2.getSize(), (int) messageFormatStream3.getSize());
            InputStream inputStream = new ByteBufferInputStream(ByteBuffer.wrap(totalMessageContent));
            List<MessageInfo> msgInfoList = new ArrayList<MessageInfo>();
            msgInfoList.add(msgInfo1);
            msgInfoList.add(msgInfo2);
            msgInfoList.add(msgInfo3);
            MessageSievingInputStream validMessageDetectionInputStream = new MessageSievingInputStream(inputStream, msgInfoList, new MockIdFactory(), new MetricRegistry());
            Assert.fail("IllegalStateException should have been thrown due to delete record ");
        }
    } catch (IllegalStateException e) {
        Assert.assertTrue("IllegalStateException thrown as expected ", true);
    }
    headerVersionToUse = Message_Header_Version_V1;
}

Example 10 with ByteBufferInputStream

use of com.github.ambry.utils.ByteBufferInputStream in project ambry by linkedin.

the class BoundedByteBufferReceiveTest method testBoundedByteBufferReceive.

@Test
public void testBoundedByteBufferReceive() {
    try {
        ByteBuffer buffer = ByteBuffer.allocate(2000);
        buffer.putLong(2000);
        byte[] buf = new byte[1992];
        new Random().nextBytes(buf);
        buffer.put(buf);
        buffer.flip();
        BoundedByteBufferReceive set = new BoundedByteBufferReceive();
        set.readFrom(Channels.newChannel(new ByteBufferInputStream(buffer)));
        buffer.clear();
        ByteBuffer payload = set.getPayload();
        for (int i = 8; i < 2000; i++) {
            Assert.assertEquals(buffer.array()[i], payload.get());
        }
    } catch (Exception e) {
        Assert.assertEquals(true, false);
    }
}