Examples with CloudConfig com.github.ambry.config.CloudConfig used on opensource projects

Example 16 with CloudConfig

use of com.github.ambry.config.CloudConfig in project ambry by linkedin.

the class CloudBlobStoreIntegrationTest method testUpdateTtl.

/**
 * Test {@link CloudBlobStore#updateTtl} method.
 */
@Test
public void testUpdateTtl() throws StoreException {
    MockMessageWriteSet messageWriteSet = new MockMessageWriteSet();
    StoreConfig storeConfig = new StoreConfig(verifiableProperties);
    CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
    long now = System.currentTimeMillis();
    long expirationTimeMs = now;
    if (isVcr) {
        // vcr doesn't upload a blob that is within CloudConfig#vcrMinTtlDays of expiry.
        expirationTimeMs += Math.max(TimeUnit.DAYS.toMillis(cloudConfig.vcrMinTtlDays), TimeUnit.SECONDS.toMillis(storeConfig.storeTtlUpdateBufferTimeSeconds));
    } else {
        expirationTimeMs += TimeUnit.SECONDS.toMillis(storeConfig.storeTtlUpdateBufferTimeSeconds);
    }
    expirationTimeMs += 100000;
    addBlobToMessageSet(messageWriteSet, expirationTimeMs, accountId, containerId, partitionId, operationTime, (short) -1);
    cloudBlobStore.put(messageWriteSet);
    // verify that the blob was uploaded with expected metadata.
    StoreInfo storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertFalse("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", expirationTimeMs, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Do a ttl update without setting ttl update flag.
    MessageInfo ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Do a ttl update on a updated blob. It should fail silently.
    ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Clear cache by restarting blob store. Do a ttl update on a updated blob. It should fail silently.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    ttlUpdateMessageInfo = new MessageInfo(messageWriteSet.getMessageSetInfo().get(0).getStoreKey(), 100, false, true, -1, accountId, containerId, now);
    cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
    storeInfo = cloudBlobStore.get(messageWriteSet.getMessageSetInfo().stream().map(MessageInfo::getStoreKey).collect(Collectors.toList()), EnumSet.allOf(StoreGetOptions.class));
    assertTrue("Unexpected ttl update status", storeInfo.getMessageReadSetInfo().get(0).isTtlUpdated());
    assertEquals("Unexpected expiration time", -1, storeInfo.getMessageReadSetInfo().get(0).getExpirationTimeInMs());
    // Delete the blob.
    cloudBlobStore.delete(Collections.singletonList(ttlUpdateMessageInfo));
    // ttlupdate of a deleted blob should throw ID_Delete Store Exception for frontend and fail silently for vcr.
    try {
        cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
        if (!isVcr) {
            fail("Update ttl of a deleted blob should fail for frontend.");
        }
    } catch (StoreException ex) {
        assertEquals("Unexcpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
    // Clear cache by restarting blob store. ttlupdate of a deleted blob should throw ID_Delete Store Exception.
    cloudBlobStore.shutdown();
    cloudBlobStore.start();
    try {
        cloudBlobStore.updateTtl(Collections.singletonList(ttlUpdateMessageInfo));
        if (!isVcr) {
            fail("Update ttl of a deleted blob should fail.");
        }
    } catch (StoreException ex) {
        assertEquals("Unexpected error code", ex.getErrorCode(), StoreErrorCodes.ID_Deleted);
    }
}

Example 17 with CloudConfig

use of com.github.ambry.config.CloudConfig in project ambry by linkedin.

the class CloudTokenPersistorTest method basicTest.

@Test
public void basicTest() throws Exception {
    Properties props = VcrTestUtil.createVcrProperties("DC1", "vcrClusterName", "zkConnectString", 12310, 12410, 12510, null);
    props.setProperty("replication.cloud.token.factory", replicationCloudTokenFactory);
    CloudConfig cloudConfig = new CloudConfig(new VerifiableProperties(props));
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(new VerifiableProperties(props));
    ClusterMap clusterMap = new MockClusterMap();
    DataNodeId dataNodeId = new CloudDataNode(cloudConfig, clusterMapConfig);
    Map<String, Set<PartitionInfo>> mountPathToPartitionInfoList = new HashMap<>();
    BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
    StoreFindTokenFactory factory = new StoreFindTokenFactory(blobIdFactory);
    PartitionId partitionId = clusterMap.getAllPartitionIds(null).get(0);
    ReplicaId cloudReplicaId = new CloudReplica(partitionId, dataNodeId);
    List<? extends ReplicaId> peerReplicas = cloudReplicaId.getPeerReplicaIds();
    List<RemoteReplicaInfo> remoteReplicas = new ArrayList<RemoteReplicaInfo>();
    List<RemoteReplicaInfo.ReplicaTokenInfo> replicaTokenInfos = new ArrayList<>();
    for (ReplicaId remoteReplica : peerReplicas) {
        RemoteReplicaInfo remoteReplicaInfo = new RemoteReplicaInfo(remoteReplica, cloudReplicaId, null, factory.getNewFindToken(), 10, SystemTime.getInstance(), remoteReplica.getDataNodeId().getPortToConnectTo());
        remoteReplicas.add(remoteReplicaInfo);
        replicaTokenInfos.add(new RemoteReplicaInfo.ReplicaTokenInfo(remoteReplicaInfo));
    }
    PartitionInfo partitionInfo = new PartitionInfo(remoteReplicas, partitionId, null, cloudReplicaId);
    mountPathToPartitionInfoList.computeIfAbsent(cloudReplicaId.getMountPath(), key -> ConcurrentHashMap.newKeySet()).add(partitionInfo);
    LatchBasedInMemoryCloudDestination cloudDestination = new LatchBasedInMemoryCloudDestination(Collections.emptyList(), AzureCloudDestinationFactory.getReplicationFeedType(new VerifiableProperties(props)), clusterMap);
    ReplicationConfig replicationConfig = new ReplicationConfig(new VerifiableProperties(props));
    CloudTokenPersistor cloudTokenPersistor = new CloudTokenPersistor("replicaTokens", mountPathToPartitionInfoList, new ReplicationMetrics(new MetricRegistry(), Collections.emptyList()), clusterMap, new FindTokenHelper(blobIdFactory, replicationConfig), cloudDestination);
    cloudTokenPersistor.persist(cloudReplicaId.getMountPath(), replicaTokenInfos);
    List<RemoteReplicaInfo.ReplicaTokenInfo> retrievedReplicaTokenInfos = cloudTokenPersistor.retrieve(cloudReplicaId.getMountPath());
    Assert.assertEquals("Number of tokens doesn't match.", replicaTokenInfos.size(), retrievedReplicaTokenInfos.size());
    for (int i = 0; i < replicaTokenInfos.size(); i++) {
        Assert.assertArrayEquals("Token is not correct.", replicaTokenInfos.get(i).getReplicaToken().toBytes(), retrievedReplicaTokenInfos.get(i).getReplicaToken().toBytes());
    }
}

Example 18 with CloudConfig

use of com.github.ambry.config.CloudConfig in project ambry by linkedin.

the class CloudRouterFactory method getRouter.

/**
 * Construct and return a {@link NonBlockingRouter} that works with cloud storage.
 * @return a {@link NonBlockingRouter}
 */
@Override
public Router getRouter() throws InstantiationException {
    try {
        MetricRegistry registry = clusterMap.getMetricRegistry();
        CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
        CloudDestinationFactory cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, verifiableProperties, registry, clusterMap);
        CloudDestination cloudDestination = cloudDestinationFactory.getCloudDestination();
        RequestHandlerPool requestHandlerPool = getRequestHandlerPool(verifiableProperties, clusterMap, cloudDestination, cloudConfig);
        CompositeNetworkClientFactory networkClientFactory = getCompositeNetworkClientFactory(requestHandlerPool);
        NonBlockingRouter router = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, notificationSystem, clusterMap, kms, cryptoService, cryptoJobHandler, accountService, time, defaultPartitionClass);
        // Make sure requestHandlerPool is shut down properly
        router.addResourceToClose(requestHandlerPool);
        router.addResourceToClose(cloudDestination);
        logger.info("Instantiated NonBlockingRouter");
        return router;
    } catch (Exception e) {
        logger.error("Error instantiating NonBlocking Router", e);
        throw new InstantiationException("Error instantiating NonBlocking Router: " + e.toString());
    }
}

Example 19 with CloudConfig

use of com.github.ambry.config.CloudConfig in project ambry by linkedin.

the class CloudRouterTest method setRouter.

/**
 * Initialize and set the router with the given {@link Properties} and {@link MockServerLayout}
 * @param props the {@link Properties}
 * @param notificationSystem the {@link NotificationSystem} to use.
 */
@Override
protected void setRouter(Properties props, MockServerLayout mockServerLayout, NotificationSystem notificationSystem) throws Exception {
    VerifiableProperties verifiableProperties = new VerifiableProperties((props));
    RouterConfig routerConfig = new RouterConfig(verifiableProperties);
    routerMetrics = new NonBlockingRouterMetrics(mockClusterMap, routerConfig);
    CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
    CloudDestinationFactory cloudDestinationFactory = Utils.getObj(cloudConfig.cloudDestinationFactoryClass, verifiableProperties, mockClusterMap.getMetricRegistry(), mockClusterMap);
    CloudDestination cloudDestination = cloudDestinationFactory.getCloudDestination();
    AccountService accountService = new InMemAccountService(false, true);
    CloudRouterFactory cloudRouterFactory = new CloudRouterFactory(verifiableProperties, mockClusterMap, new LoggingNotificationSystem(), null, accountService);
    RequestHandlerPool requestHandlerPool = cloudRouterFactory.getRequestHandlerPool(verifiableProperties, mockClusterMap, cloudDestination, cloudConfig);
    Map<ReplicaType, NetworkClientFactory> childFactories = new EnumMap<>(ReplicaType.class);
    childFactories.put(ReplicaType.CLOUD_BACKED, new LocalNetworkClientFactory((LocalRequestResponseChannel) requestHandlerPool.getChannel(), new NetworkConfig(verifiableProperties), new NetworkMetrics(routerMetrics.getMetricRegistry()), mockTime));
    childFactories.put(ReplicaType.DISK_BACKED, new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime));
    NetworkClientFactory networkClientFactory = new CompositeNetworkClientFactory(childFactories);
    router = new NonBlockingRouter(routerConfig, routerMetrics, networkClientFactory, notificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
    router.addResourceToClose(requestHandlerPool);
}

Example 20 with CloudConfig

use of com.github.ambry.config.CloudConfig in project ambry by linkedin.

the class AzureCompactionTool method main.

public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    ArgumentAcceptingOptionSpec<String> propsFileOpt = parser.accepts(PROPS_FILE, "Properties file path").withRequiredArg().describedAs(PROPS_FILE).ofType(String.class);
    String commandName = AzureCompactionTool.class.getSimpleName();
    parser.accepts(PURGE_OPTION, "Flag to purge dead blobs from the partition");
    parser.nonOptions("The partitions to compact").ofType(String.class);
    OptionSet optionSet = parser.parse(args);
    String propsFilePath = optionSet.valueOf(propsFileOpt);
    if (propsFilePath == null) {
        printHelpAndExit(parser);
    }
    Properties properties = Utils.loadProps(propsFilePath);
    ToolUtils.addClusterMapProperties(properties);
    VerifiableProperties verifiableProperties = new VerifiableProperties(properties);
    // User needs to specify this option to actually delete blobs
    boolean testMode = !optionSet.has(PURGE_OPTION);
    List<String> partitions = (List<String>) optionSet.nonOptionArguments();
    if (!testMode && partitions.isEmpty()) {
        printHelpAndExit(parser);
    }
    Set<PartitionId> partitionIdSet = partitions.stream().map(path -> new PartitionPathId(path)).collect(Collectors.toSet());
    AzureCloudDestination azureDest = null;
    try {
        azureDest = (AzureCloudDestination) new AzureCloudDestinationFactory(verifiableProperties, new MetricRegistry(), null).getCloudDestination();
        CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
        CloudStorageCompactor compactor = new CloudStorageCompactor(azureDest, cloudConfig, partitionIdSet, new VcrMetrics(new MetricRegistry()));
        // Attempt clean shutdown if someone Ctrl-C's us.
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            logger.info("Received shutdown signal. Shutting down compactor.");
            compactor.shutdown();
        }));
        if (testMode) {
            DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneId.systemDefault());
            List<Pair<String, Long>> progressList = azureDest.getAzureStorageCompactor().getAllCompactionProgress();
            progressList.forEach(pair -> {
                String progress = dateTimeFormatter.format(Instant.ofEpochMilli(pair.getSecond()));
                // TODO: write to user specified output file
                System.out.println(pair.getFirst() + "\t" + progress);
            });
        } else {
            compactor.compactPartitions();
        }
        System.exit(0);
    } catch (Exception ex) {
        logger.error("Command {} failed", commandName, ex);
        System.exit(1);
    } finally {
        if (azureDest != null) {
            azureDest.close();
        }
    }
}