Examples with BlobProperties com.github.ambry.messageformat.BlobProperties used on opensource projects

Example 71 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class ServerTestUtil method checkTtlUpdateStatus.

/**
 * Checks the TTL update status of the given {@code blobId} based on the args provided
 * @param channel the {@link ConnectedChannel} to make the {@link GetRequest} on.
 * @param clusterMap the {@link ClusterMap} to use
 * @param storeKeyFactory the {@link StoreKeyFactory} to use
 * @param blobId the ID of the blob to check
 * @param expectedBlobData the expected blob data
 * @param ttlUpdated {@code true} if the blob has been ttl updated
 * @param expectedExpiryTimeMs the expected expiry time (in ms)
 * @throws IOException
 * @throws MessageFormatException
 */
static void checkTtlUpdateStatus(ConnectedChannel channel, ClusterMap clusterMap, StoreKeyFactory storeKeyFactory, BlobId blobId, byte[] expectedBlobData, boolean ttlUpdated, long expectedExpiryTimeMs) throws IOException, MessageFormatException {
    PartitionRequestInfo requestInfo = new PartitionRequestInfo(blobId.getPartition(), Collections.singletonList(blobId));
    List<PartitionRequestInfo> requestInfos = Collections.singletonList(requestInfo);
    // blob properties
    GetRequest request = new GetRequest(1, "checkTtlUpdateStatus", MessageFormatFlags.BlobProperties, requestInfos, GetOption.None);
    DataInputStream stream = channel.sendAndReceive(request).getInputStream();
    GetResponse response = GetResponse.readFrom(stream, clusterMap);
    BlobProperties blobProperties = MessageFormatRecord.deserializeBlobProperties(response.getInputStream());
    if (!ttlUpdated) {
        assertEquals("TTL does not match", expectedExpiryTimeMs, getExpiryTimeMs(blobProperties));
    }
    MessageInfo messageInfo = response.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
    assertEquals("Blob ID not as expected", blobId, messageInfo.getStoreKey());
    assertEquals("TTL update state not as expected", ttlUpdated, messageInfo.isTtlUpdated());
    assertEquals("Expiry time is not as expected", expectedExpiryTimeMs, messageInfo.getExpirationTimeInMs());
    releaseNettyBufUnderneathStream(stream);
    // blob all
    request = new GetRequest(1, "checkTtlUpdateStatus", MessageFormatFlags.All, requestInfos, GetOption.None);
    stream = channel.sendAndReceive(request).getInputStream();
    response = GetResponse.readFrom(stream, clusterMap);
    InputStream responseStream = response.getInputStream();
    BlobAll blobAll = MessageFormatRecord.deserializeBlobAll(responseStream, storeKeyFactory);
    byte[] actualBlobData = getBlobDataAndRelease(blobAll.getBlobData());
    assertArrayEquals("Content mismatch.", expectedBlobData, actualBlobData);
    messageInfo = response.getPartitionResponseInfoList().get(0).getMessageInfoList().get(0);
    assertEquals("Blob ID not as expected", blobId, messageInfo.getStoreKey());
    assertEquals("TTL update state not as expected", ttlUpdated, messageInfo.isTtlUpdated());
    assertEquals("Expiry time is not as expected", expectedExpiryTimeMs, messageInfo.getExpirationTimeInMs());
    releaseNettyBufUnderneathStream(stream);
}

Example 72 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class VcrBackupTest method sendBlobToDataNode.

/**
 * Send blobs to given dataNode.
 * @param dataNode the target node.
 * @param blobCount number of blobs to send.
 * @return list of blobs successfully sent.
 */
private List<BlobId> sendBlobToDataNode(DataNodeId dataNode, int blobCount) throws Exception {
    int userMetaDataSize = 10;
    // Send blobs to DataNode
    byte[] userMetadata = new byte[userMetaDataSize];
    byte[] data = new byte[blobSize];
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties properties = new BlobProperties(blobSize, "serviceid1", null, null, false, -1, accountId, containerId, false, null, null, null);
    TestUtils.RANDOM.nextBytes(userMetadata);
    TestUtils.RANDOM.nextBytes(data);
    Port port = new Port(dataNode.getPort(), PortType.PLAINTEXT);
    ConnectedChannel channel = ServerTestUtil.getBlockingChannelBasedOnPortType(port, "localhost", null, null);
    channel.connect();
    CountDownLatch latch = new CountDownLatch(1);
    DirectSender runnable = new DirectSender(mockCluster, channel, blobCount, data, userMetadata, properties, null, latch);
    Thread threadToRun = new Thread(runnable);
    threadToRun.start();
    assertTrue("Did not put all blobs in 2 minutes", latch.await(2, TimeUnit.MINUTES));
    List<BlobId> blobIds = runnable.getBlobIds();
    for (BlobId blobId : blobIds) {
        notificationSystem.awaitBlobCreations(blobId.getID());
    }
    return blobIds;
}

Example 73 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class PutOperationTest method testStitchErrorDataChunkHandling.

/**
 * Ensure that errors while stitching blobs do not result in data chunk deletions.
 * @throws Exception
 */
@Test
public void testStitchErrorDataChunkHandling() throws Exception {
    BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
    byte[] userMetadata = new byte[10];
    FutureResult<String> future = new FutureResult<>();
    MockNetworkClient mockNetworkClient = new MockNetworkClient();
    List<ChunkInfo> chunksToStitch = RouterTestHelpers.buildChunkList(mockClusterMap, BlobId.BlobDataType.DATACHUNK, Utils.Infinite_Time, LongStream.of(10, 10, 11));
    PutOperation op = PutOperation.forStitching(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), userMetadata, chunksToStitch, future, null, new RouterCallback(mockNetworkClient, new ArrayList<>()), null, null, null, time, blobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    // Trigger an exception by making the last chunk size too large.
    op.startOperation();
    Assert.assertTrue("Operation should be completed", op.isOperationComplete());
    Assert.assertEquals("Wrong RouterException error code", RouterErrorCode.InvalidPutArgument, ((RouterException) op.getOperationException()).getErrorCode());
    // Ensure that the operation does not provide the background deleter with any data chunks to delete.
    Assert.assertEquals("List of chunks to delete should be empty", 0, op.getSuccessfullyPutChunkIdsIfCompositeDirectUpload().size());
}

Example 74 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class PutOperationTest method testSlippedPutsWithServerErrors.

/**
 * Test PUT operation that handles ServerErrorCode = Temporarily_Disabled and Replica_Unavailable
 * @throws Exception
 */
@Test
public void testSlippedPutsWithServerErrors() throws Exception {
    Properties properties = new Properties();
    properties.setProperty("router.hostname", "localhost");
    properties.setProperty("router.datacenter.name", "DC1");
    properties.setProperty("router.max.put.chunk.size.bytes", Integer.toString(chunkSize));
    properties.setProperty("router.put.request.parallelism", Integer.toString(requestParallelism));
    // Expect at least two successes so that you can create slipped puts.
    properties.setProperty("router.put.success.target", Integer.toString(2));
    VerifiableProperties vProps = new VerifiableProperties(properties);
    RouterConfig routerConfig = new RouterConfig(vProps);
    int numChunks = 1;
    BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
    byte[] userMetadata = new byte[10];
    byte[] content = new byte[chunkSize * numChunks];
    random.nextBytes(content);
    ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
    MockNetworkClient mockNetworkClient = new MockNetworkClient();
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), userMetadata, channel, PutBlobOptions.DEFAULT, new FutureResult<>(), null, new RouterCallback(mockNetworkClient, new ArrayList<>()), null, null, null, null, time, blobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    List<RequestInfo> requestInfos = new ArrayList<>();
    requestRegistrationCallback.setRequestsToSend(requestInfos);
    // fill chunks would end up filling the maximum number of PutChunks.
    op.fillChunks();
    Assert.assertTrue("ReadyForPollCallback should have been invoked as chunks were fully filled", mockNetworkClient.getAndClearWokenUpStatus());
    // poll to populate request
    op.poll(requestRegistrationCallback);
    // Set up server errors such that put fails on 2 out 3 nodes, hence creating a slipped put on the succeeding node.
    // Second attempts on all node succeed.
    List<ServerErrorCode> serverErrorList = new ArrayList<>();
    // Success on the first host, slipped put
    serverErrorList.add(ServerErrorCode.No_Error);
    // Fail on the second host
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    // Fail on the third host
    serverErrorList.add(ServerErrorCode.Unknown_Error);
    // Success on the second attempts on all hosts
    serverErrorList.add(ServerErrorCode.No_Error);
    serverErrorList.add(ServerErrorCode.No_Error);
    serverErrorList.add(ServerErrorCode.No_Error);
    mockServer.setServerErrors(serverErrorList);
    // Send all requests.
    for (int i = 0; i < requestInfos.size(); i++) {
        ResponseInfo responseInfo = getResponseInfo(requestInfos.get(i));
        PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
        op.handleResponse(responseInfo, putResponse);
        requestInfos.get(i).getRequest().release();
        responseInfo.release();
    }
    Assert.assertEquals("Number of slipped puts should be 1", 1, op.getSlippedPutBlobIds().size());
    // fill chunks again.
    op.fillChunks();
    requestInfos.clear();
    // poll to populate request
    op.poll(requestRegistrationCallback);
    // Send all requests again.
    for (int i = 0; i < requestInfos.size(); i++) {
        ResponseInfo responseInfo = getResponseInfo(requestInfos.get(i));
        PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
        op.handleResponse(responseInfo, putResponse);
        requestInfos.get(i).getRequest().release();
        responseInfo.release();
    }
    Assert.assertEquals("Number of slipped puts should be 1", 1, op.getSlippedPutBlobIds().size());
    PutOperation.PutChunk putChunk = op.getPutChunks().get(0);
    // Make sure the chunk blob id which has been put successfully is not part of the slipped puts.
    Assert.assertFalse(op.getSlippedPutBlobIds().contains(putChunk.chunkBlobId));
}

Example 75 with BlobProperties

use of com.github.ambry.messageformat.BlobProperties in project ambry by linkedin.

the class PutOperationTest method testHandleResponseWithServerErrors.

/**
 * Test PUT operation that handles ServerErrorCode = Temporarily_Disabled and Replica_Unavailable
 * @throws Exception
 */
@Test
public void testHandleResponseWithServerErrors() throws Exception {
    int numChunks = routerConfig.routerMaxInMemPutChunks + 1;
    BlobProperties blobProperties = new BlobProperties(-1, "serviceId", "memberId", "contentType", false, Utils.Infinite_Time, Utils.getRandomShort(TestUtils.RANDOM), Utils.getRandomShort(TestUtils.RANDOM), false, null, null, null);
    byte[] userMetadata = new byte[10];
    byte[] content = new byte[chunkSize * numChunks];
    random.nextBytes(content);
    ReadableStreamChannel channel = new ByteBufferReadableStreamChannel(ByteBuffer.wrap(content));
    PutOperation op = PutOperation.forUpload(routerConfig, routerMetrics, mockClusterMap, new LoggingNotificationSystem(), new InMemAccountService(true, false), userMetadata, channel, PutBlobOptions.DEFAULT, new FutureResult<>(), null, new RouterCallback(new MockNetworkClient(), new ArrayList<>()), null, null, null, null, time, blobProperties, MockClusterMap.DEFAULT_PARTITION_CLASS, quotaChargeCallback);
    op.startOperation();
    List<RequestInfo> requestInfos = new ArrayList<>();
    requestRegistrationCallback.setRequestsToSend(requestInfos);
    // fill chunks would end up filling the maximum number of PutChunks.
    op.fillChunks();
    // poll to populate request
    op.poll(requestRegistrationCallback);
    // make 1st request of first chunk encounter Temporarily_Disabled
    mockServer.setServerErrorForAllRequests(ServerErrorCode.Temporarily_Disabled);
    ResponseInfo responseInfo = getResponseInfo(requestInfos.get(0));
    PutResponse putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
    op.handleResponse(responseInfo, putResponse);
    responseInfo.release();
    PutOperation.PutChunk putChunk = op.getPutChunks().get(0);
    SimpleOperationTracker operationTracker = (SimpleOperationTracker) putChunk.getOperationTrackerInUse();
    Assert.assertEquals("Disabled count should be 1", 1, operationTracker.getDisabledCount());
    Assert.assertEquals("Disabled count should be 0", 0, operationTracker.getFailedCount());
    // make 2nd request of first chunk encounter Replica_Unavailable
    mockServer.setServerErrorForAllRequests(ServerErrorCode.Replica_Unavailable);
    responseInfo = getResponseInfo(requestInfos.get(1));
    putResponse = responseInfo.getError() == null ? PutResponse.readFrom(new NettyByteBufDataInputStream(responseInfo.content())) : null;
    op.handleResponse(responseInfo, putResponse);
    responseInfo.release();
    putChunk = op.getPutChunks().get(0);
    Assert.assertEquals("Failure count should be 1", 1, ((SimpleOperationTracker) putChunk.getOperationTrackerInUse()).getFailedCount());
    mockServer.resetServerErrors();
    // Release all the other requests
    requestInfos.forEach(info -> info.getRequest().release());
}