Examples with StoreInfo com.github.ambry.store.StoreInfo used on opensource projects

Example 6 with StoreInfo

use of com.github.ambry.store.StoreInfo in project ambry by linkedin.

the class InMemoryStore method get.

@Override
public StoreInfo get(List<? extends StoreKey> ids, EnumSet<StoreGetOptions> getOptions) throws StoreException {
    // unused function
    List<MessageInfo> infos = new ArrayList<>();
    List<ByteBuffer> buffers = new ArrayList<>();
    List<StoreKey> keys = new ArrayList<>();
    for (StoreKey id : ids) {
        for (int i = 0; i < messageInfos.size(); i++) {
            MessageInfo info = messageInfos.get(i);
            if (info.getStoreKey().equals(id)) {
                infos.add(info);
                buffers.add(log.getData(i));
                keys.add(info.getStoreKey());
            }
        }
    }
    return new StoreInfo(new MockMessageReadSet(buffers, keys), infos);
}

Example 7 with StoreInfo

use of com.github.ambry.store.StoreInfo in project ambry by linkedin.

the class CloudBlobStore method get.

@Override
public StoreInfo get(List<? extends StoreKey> ids, EnumSet<StoreGetOptions> storeGetOptions) throws StoreException {
    checkStarted();
    checkStoreKeyDuplicates(ids);
    List<CloudMessageReadSet.BlobReadInfo> blobReadInfos = new ArrayList<>(ids.size());
    List<MessageInfo> messageInfos = new ArrayList<>(ids.size());
    try {
        List<BlobId> blobIdList = ids.stream().map(key -> (BlobId) key).collect(Collectors.toList());
        Map<String, CloudBlobMetadata> cloudBlobMetadataListMap = requestAgent.doWithRetries(() -> cloudDestination.getBlobMetadata(blobIdList), "GetBlobMetadata", partitionId.toPathString());
        // Throw StoreException with ID_Not_Found if cloudBlobMetadataListMap size is less than expected.
        if (cloudBlobMetadataListMap.size() < blobIdList.size()) {
            Set<BlobId> missingBlobs = blobIdList.stream().filter(blobId -> !cloudBlobMetadataListMap.containsKey(blobId)).collect(Collectors.toSet());
            throw new StoreException("Some of the keys were missing in the cloud metadata store: " + missingBlobs, StoreErrorCodes.ID_Not_Found);
        }
        long currentTimeStamp = System.currentTimeMillis();
        // Validate cloud meta data, may throw StoreException with ID_Deleted, TTL_Expired and Authorization_Failure
        validateCloudMetadata(cloudBlobMetadataListMap, storeGetOptions, currentTimeStamp, ids);
        for (BlobId blobId : blobIdList) {
            CloudBlobMetadata blobMetadata = cloudBlobMetadataListMap.get(blobId.getID());
            // TODO: need to add ttlUpdated to CloudBlobMetadata so we can use it here
            // For now, set ttlUpdated = true for all permanent blobs, so the correct ttl
            // is applied by GetOperation.
            boolean ttlUpdated = blobMetadata.getExpirationTime() == Utils.Infinite_Time;
            boolean deleted = blobMetadata.getDeletionTime() != Utils.Infinite_Time;
            MessageInfo messageInfo = new MessageInfo(blobId, blobMetadata.getSize(), deleted, ttlUpdated, blobMetadata.isUndeleted(), blobMetadata.getExpirationTime(), null, (short) blobMetadata.getAccountId(), (short) blobMetadata.getContainerId(), getOperationTime(blobMetadata), blobMetadata.getLifeVersion());
            messageInfos.add(messageInfo);
            blobReadInfos.add(new CloudMessageReadSet.BlobReadInfo(blobMetadata, blobId));
        }
    } catch (CloudStorageException e) {
        if (e.getCause() instanceof StoreException) {
            throw (StoreException) e.getCause();
        } else {
            throw new StoreException(e, StoreErrorCodes.IOError);
        }
    }
    CloudMessageReadSet messageReadSet = new CloudMessageReadSet(blobReadInfos, this);
    return new StoreInfo(messageReadSet, messageInfos);
}

Example 8 with StoreInfo

use of com.github.ambry.store.StoreInfo in project ambry by linkedin.

the class CloudBlobStoreTest method testGetForDeletedBlobWithIncludeDeleteOption.

/**
 * Test cloud store get with a list of blobs such that atleast one of them is deleted, and {@code StoreGetOptions.Store_Include_Deleted} is set in getoptions
 * @param blobIds list of blobids to get
 * @param deletedBlobId blobId which is marked as deleted
 * @throws Exception
 */
private void testGetForDeletedBlobWithIncludeDeleteOption(List<BlobId> blobIds, BlobId deletedBlobId) throws Exception {
    StoreInfo storeInfo = store.get(blobIds, EnumSet.of(StoreGetOptions.Store_Include_Deleted));
    for (MessageInfo msgInfo : storeInfo.getMessageReadSetInfo()) {
        if (msgInfo.getStoreKey().equals(deletedBlobId)) {
            return;
        }
    }
    fail("get should be successful for a deleted blob with Store_Include_Deleted set in get options");
}

Example 9 with StoreInfo

use of com.github.ambry.store.StoreInfo in project ambry by linkedin.

the class CloudBlobStoreIntegrationTest method testDeleteFromVcr.

/**
 * Test {@link CloudBlobStore#delete} method for vcr.
 */
public void testDeleteFromVcr() throws StoreException {
    // First upload a blob with a life version 2
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    addBlobToMessageSet(messageWriteSet, Utils.Infinite_Time, accountId, containerId, partitionId, operationTime, (short) 2);
    cloudBlobStore.put(messageWriteSet);
    // verify that the blob was uploaded with expected metadata.
    StoreInfo storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", messageWriteSet.getMessageSetInfo().get(0).getLifeVersion(), storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertEquals("Unexpected delete status", messageWriteSet.getMessageSetInfo().get(0).isDeleted(), storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Now delete with a smaller life version should fail silently without updating the life version.
    MessageInfo messageInfo = messageWriteSet.getMessageSetInfo().get(0);
    MessageInfo deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), (short) 1);
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", messageWriteSet.getMessageSetInfo().get(0).getLifeVersion(), storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertEquals("Unexpected delete status", messageWriteSet.getMessageSetInfo().get(0).isDeleted(), storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Delete with same life version should pass without changing life version.
    deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), messageInfo.getLifeVersion());
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", messageWriteSet.getMessageSetInfo().get(0).getLifeVersion(), storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Deleting a deleted blob with higher life version should update life version.
    deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), (short) 3);
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", 3, storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Deleting again with smaller life version should fail with exception.
    deleteMessageInfo = new MessageInfo(messageInfo.getStoreKey(), messageInfo.getSize(), messageInfo.isDeleted(), messageInfo.isTtlUpdated(), messageInfo.isUndeleted(), messageInfo.getExpirationTimeInMs(), messageInfo.getCrc(), messageInfo.getAccountId(), messageInfo.getContainerId(), messageInfo.getOperationTimeMs(), (short) 1);
    try {
        cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
        fail("Delete should fail with ID_Deleted StoreException");
    } catch (StoreException ex) {
        assertEquals("Unexpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", 3, storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
    // Restart cloud blob store to clear cache. Deleting again with smaller life version should fail silently without updating anything.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    cloudBlobStore.delete(Collections.singletonList(deleteMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertEquals("Unexpected live version", 3, storeInfo.getMessageReadSetInfo().get(0).getLifeVersion());
    assertTrue("Unexpected delete status", storeInfo.getMessageReadSetInfo().get(0).isDeleted());
}

Example 10 with StoreInfo

use of com.github.ambry.store.StoreInfo in project ambry by linkedin.

the class CloudBlobStoreIntegrationTest method testUpdateTtl.

/**
 * Test {@link CloudBlobStore#updateTtl} method.
 */
@Test
public void testUpdateTtl() throws StoreException {
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    StoreConfig storeConfig = new StoreConfig(verifiableProperties);
    CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
    long now = System.currentTimeMillis();
    long expirationTimeMs = now;
    if (isVcr) {
        // vcr doesn't upload a blob that is within CloudConfig#vcrMinTtlDays of expiry.
        expirationTimeMs += Math.max(TimeUnit.DAYS.toMillis(cloudConfig.vcrMinTtlDays), TimeUnit.SECONDS.toMillis(storeConfig.storeTtlUpdateBufferTimeSeconds));
    } else {
        expirationTimeMs += TimeUnit.SECONDS.toMillis(storeConfig.storeTtlUpdateBufferTimeSeconds);
    }
    expirationTimeMs += 100000;
    addBlobToMessageSet(messageWriteSet, expirationTimeMs, accountId, containerId, partitionId, operationTime, (short) -1);
    cloudBlobStore.put(messageWriteSet);
    // verify that the blob was uploaded with expected metadata.
    StoreInfo storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertFalse("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", expirationTimeMs, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Do a ttl update without setting ttl update flag.
    MessageInfo ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Do a ttl update on a updated blob. It should fail silently.
    ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Clear cache by restarting blob store. Do a ttl update on a updated blob. It should fail silently.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Delete the blob.
    cloudBlobStore.delete(Collections.singletonList(ttlUpdateMessageInfo));
    // ttlupdate of a deleted blob should throw ID_Delete Store Exception for frontend and fail silently for vcr.
    try {
        cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
        if (!isVcr) {
            fail("Update ttl of a deleted blob should fail for frontend.");
        }
    } catch (StoreException ex) {
        assertEquals("Unexcpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
    // Clear cache by restarting blob store. ttlupdate of a deleted blob should throw ID_Delete Store Exception.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    try {
        cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
        if (!isVcr) {
            fail("Update ttl of a deleted blob should fail.");
        }
    } catch (StoreException ex) {
        assertEquals("Unexpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
}