Examples with LatchBasedInMemoryCloudDestination com.github.ambry.cloud.LatchBasedInMemoryCloudDestination used on opensource projects

Example 1 with LatchBasedInMemoryCloudDestination

use of com.github.ambry.cloud.LatchBasedInMemoryCloudDestination in project ambry by linkedin.

the class ServerTestUtil method endToEndCloudBackupTest.

/**
 * Tests blobs put to dataNode can be backed up by {@link com.github.ambry.cloud.VcrReplicationManager}.
 * @param cluster the {@link MockCluster} of dataNodes.
 * @param zkConnectString ZK endpoint to establish VCR cluster
 * @param vcrClusterName the name of VCR cluster
 * @param dataNode the datanode where blobs are originally put.
 * @param clientSSLConfig the {@link SSLConfig}.
 * @param clientSSLSocketFactory the {@link SSLSocketFactory}.
 * @param notificationSystem the {@link MockNotificationSystem} to track blobs event in {@link MockCluster}.
 * @param vcrSSLProps SSL related properties for VCR. Can be {@code null}.
 * @param doTtlUpdate Do ttlUpdate request if {@code true}.
 */
static void endToEndCloudBackupTest(MockCluster cluster, String zkConnectString, String vcrClusterName, DataNodeId dataNode, SSLConfig clientSSLConfig, SSLSocketFactory clientSSLSocketFactory, MockNotificationSystem notificationSystem, Properties vcrSSLProps, boolean doTtlUpdate) throws Exception {
    int blobBackupCount = 10;
    int blobSize = 100;
    int userMetaDataSize = 100;
    ClusterAgentsFactory clusterAgentsFactory = cluster.getClusterAgentsFactory();
    // Send blobs to DataNode
    byte[] userMetadata = new byte[userMetaDataSize];
    byte[] data = new byte[blobSize];
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    long ttl = doTtlUpdate ? TimeUnit.DAYS.toMillis(1) : Utils.Infinite_Time;
    BlobProperties properties = new BlobProperties(blobSize, "serviceid1", null, null, false, ttl, cluster.time.milliseconds(), accountId, containerId, false, null, null, null);
    TestUtils.RANDOM.nextBytes(userMetadata);
    TestUtils.RANDOM.nextBytes(data);
    Port port;
    if (clientSSLConfig == null) {
        port = new Port(dataNode.getPort(), PortType.PLAINTEXT);
    } else {
        port = new Port(dataNode.getSSLPort(), PortType.SSL);
    }
    ConnectedChannel channel = getBlockingChannelBasedOnPortType(port, "localhost", clientSSLSocketFactory, clientSSLConfig);
    channel.connect();
    CountDownLatch latch = new CountDownLatch(1);
    DirectSender runnable = new DirectSender(cluster, channel, blobBackupCount, data, userMetadata, properties, null, latch);
    Thread threadToRun = new Thread(runnable);
    threadToRun.start();
    assertTrue("Did not put all blobs in 2 minutes", latch.await(2, TimeUnit.MINUTES));
    // TODO: remove this temp fix after fixing race condition in MockCluster/MockNotificationSystem
    Thread.sleep(3000);
    List<BlobId> blobIds = runnable.getBlobIds();
    for (BlobId blobId : blobIds) {
        notificationSystem.awaitBlobCreations(blobId.getID());
        if (doTtlUpdate) {
            updateBlobTtl(channel, blobId, cluster.time.milliseconds());
        }
    }
    HelixControllerManager helixControllerManager = VcrTestUtil.populateZkInfoAndStartController(zkConnectString, vcrClusterName, cluster.getClusterMap());
    // Start the VCR and CloudBackupManager
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, 12310, 12410, 12510, vcrSSLProps);
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(blobIds, clusterAgentsFactory.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), clusterAgentsFactory, notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // Waiting for backup done
    assertTrue("Did not backup all blobs in 2 minutes", latchBasedInMemoryCloudDestination.awaitUpload(2, TimeUnit.MINUTES));
    Map<String, CloudBlobMetadata> cloudBlobMetadataMap = latchBasedInMemoryCloudDestination.getBlobMetadata(blobIds);
    for (BlobId blobId : blobIds) {
        CloudBlobMetadata cloudBlobMetadata = cloudBlobMetadataMap.get(blobId.toString());
        assertNotNull("cloudBlobMetadata should not be null", cloudBlobMetadata);
        assertEquals("AccountId mismatch", accountId, cloudBlobMetadata.getAccountId());
        assertEquals("ContainerId mismatch", containerId, cloudBlobMetadata.getContainerId());
        assertEquals("Expiration time mismatch", Utils.Infinite_Time, cloudBlobMetadata.getExpirationTime());
    // TODO: verify other metadata and blob data
    }
    vcrServer.shutdown();
    helixControllerManager.syncStop();
}

Example 2 with LatchBasedInMemoryCloudDestination

use of com.github.ambry.cloud.LatchBasedInMemoryCloudDestination in project ambry by linkedin.

the class VcrBackupTest method basicTest.

/**
 * Basic test to make sure VCR can backup with HelixVcrCluster.
 */
@Test
public void basicTest() throws Exception {
    List<BlobId> blobIds = sendBlobToDataNode(dataNode, 10);
    // Start the VCR and CloudBackupManager
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, clusterMapPort, 12410, 12510, serverSSLProps, vcrHelixStateModelFactoryClass, true);
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(blobIds, mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // Waiting for backup done
    assertTrue("Did not backup all blobs in 2 minutes", latchBasedInMemoryCloudDestination.awaitUpload(2, TimeUnit.MINUTES));
    // Verify a blob by making a http2 request.
    MockClusterMap clusterMap = mockCluster.getClusterMap();
    SSLConfig clientSSLConfig = new SSLConfig(new VerifiableProperties(clientSSLProps));
    ConnectedChannel channel = ServerTestUtil.getBlockingChannelBasedOnPortType(new Port(clusterMap.getDataNodes().get(0).getHttp2Port(), PortType.HTTP2), "localhost", null, clientSSLConfig);
    BlobId blobToVerify = blobIds.get(0);
    ArrayList<BlobId> idList = new ArrayList<>(Arrays.asList(blobToVerify));
    ArrayList<PartitionRequestInfo> partitionRequestInfoList = new ArrayList<PartitionRequestInfo>();
    PartitionRequestInfo partitionRequestInfo = new PartitionRequestInfo(blobToVerify.getPartition(), idList);
    partitionRequestInfoList.add(partitionRequestInfo);
    GetRequest getRequest1 = new GetRequest(1, "clientid1", MessageFormatFlags.BlobProperties, partitionRequestInfoList, GetOption.None);
    DataInputStream stream = channel.sendAndReceive(getRequest1).getInputStream();
    GetResponse resp1 = GetResponse.readFrom(stream, clusterMap);
    try {
        BlobProperties propertyOutput = MessageFormatRecord.deserializeBlobProperties(resp1.getInputStream());
        // Do a simple check
        assertEquals(blobSize, propertyOutput.getBlobSize());
        releaseNettyBufUnderneathStream(stream);
    } catch (MessageFormatException e) {
        fail();
    }
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
}

Example 3 with LatchBasedInMemoryCloudDestination

use of com.github.ambry.cloud.LatchBasedInMemoryCloudDestination in project ambry by linkedin.

the class VcrBackupTest method multipleVcrTest.

/**
 * A multiple VCR test to test helix assignment and backup.
 */
@Test
public void multipleVcrTest() throws Exception {
    StrictMatchExternalViewVerifier helixBalanceVerifier = new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName, Collections.singleton(VcrTestUtil.helixResource), null);
    int initialNumOfVcrs = 5;
    // create a shared in memory destination.
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    // 1st phase: Start VCRs to do backup.
    List<VcrServer> vcrServers = new ArrayList<>();
    List<MockNotificationSystem> vcrNotificationSystems = new ArrayList<>();
    for (int port = 12310; port < 12310 + initialNumOfVcrs; port++) {
        Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, port, port + 100, port + 200, null, vcrHelixStateModelFactoryClass, true);
        MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
        VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
        vcrServer.startup();
        vcrServers.add(vcrServer);
        vcrNotificationSystems.add(vcrNotificationSystem);
    }
    makeSureHelixBalance(vcrServers.get(vcrServers.size() - 1), helixBalanceVerifier);
    int numOfBlobs = 100;
    sendBlobToDataNode(dataNode, numOfBlobs);
    // Make sure blobs are backed up.
    TestUtils.checkAndSleep(numOfBlobs, () -> vcrNotificationSystems.stream().mapToInt(i -> i.getBlobIds().size()).sum(), 5000);
    // verify each VCR is only replicating partitions assigned to it.
    for (int i = 0; i < initialNumOfVcrs; i++) {
        Set<PartitionId> partitionIdSet = vcrNotificationSystems.get(i).getBlobIds().stream().map(blobIdStr -> {
            try {
                return new BlobId(blobIdStr, mockCluster.getClusterMap()).getPartition();
            } catch (IOException e) {
                e.printStackTrace();
                return null;
            }
        }).collect(Collectors.toSet());
        assertTrue("Each VCR should have some assignment.", vcrServers.get(i).getVcrClusterParticipant().getAssignedPartitionIds().size() > 0);
        assertTrue("Each VCR should only backup its assigned partitions.", new HashSet<>(vcrServers.get(i).getVcrClusterParticipant().getAssignedPartitionIds()).containsAll(partitionIdSet));
    }
    logger.info("Phase 1 done.");
    // 2nd phase: Add a new VCR to cluster.
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, 12310 + initialNumOfVcrs, 12310 + initialNumOfVcrs + 100, 12310 + initialNumOfVcrs + 200, null, vcrHelixStateModelFactoryClass, true);
    MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    vcrServers.add(vcrServer);
    vcrNotificationSystems.add(vcrNotificationSystem);
    makeSureHelixBalance(vcrServers.get(vcrServers.size() - 1), helixBalanceVerifier);
    int secondNumOfBlobs = 100;
    sendBlobToDataNode(dataNode, secondNumOfBlobs);
    Assert.assertTrue("All blobs should be back up.", TestUtils.checkAndSleep(numOfBlobs + secondNumOfBlobs, () -> vcrNotificationSystems.stream().mapToInt(i -> i.getBlobIds().size()).sum(), 5000));
    logger.info("Phase 2 done.");
    // 3rd phase: Remove last VCR from cluster.
    vcrServers.get(vcrServers.size() - 1).shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServers.get(vcrServers.size() - 1).awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServers.get(vcrServers.size() - 1).getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServers.get(vcrServers.size() - 1).getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    int temp = vcrNotificationSystems.get(vcrNotificationSystems.size() - 1).getBlobIds().size();
    assertTrue("Helix balance timeout.", helixBalanceVerifier.verify(5000));
    int thirdNumOfBlobs = 100;
    sendBlobToDataNode(dataNode, thirdNumOfBlobs);
    Assert.assertTrue("All blobs should be back up.", TestUtils.checkAndSleep(numOfBlobs + secondNumOfBlobs + thirdNumOfBlobs, () -> vcrNotificationSystems.stream().mapToInt(i -> i.getBlobIds().size()).sum(), 5000));
    Assert.assertEquals("The removed vcr shouldn't have any change", temp, vcrNotificationSystems.get(vcrNotificationSystems.size() - 1).getBlobIds().size());
    logger.info("Phase 3 done.");
    // Shutdown all others.
    for (int i = 0; i < initialNumOfVcrs; i++) {
        // Error metrics should be zero.
        Assert.assertEquals("Error count should be zero", 0, vcrServers.get(i).getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
        Assert.assertEquals("Error count should be zero", 0, vcrServers.get(i).getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
        vcrServers.get(i).shutdown();
        assertTrue("VCR server shutdown timeout.", vcrServers.get(i).awaitShutdown(5000));
    }
    logger.info("Test done.");
}

Example 4 with LatchBasedInMemoryCloudDestination

use of com.github.ambry.cloud.LatchBasedInMemoryCloudDestination in project ambry by linkedin.

the class CloudAndStoreReplicationTest method setup.

/**
 * Create a cluster with one vcr node and two ambry server data nodes.
 * @throws Exception on {@link Exception}
 */
@Before
public void setup() throws Exception {
    String vcrMountPath = ClusterMapSnapshotConstants.CLOUD_REPLICA_MOUNT + "/1";
    recoveryProperties = new Properties();
    recoveryProperties.setProperty("replication.metadata.request.version", "2");
    recoveryProperties.setProperty("replication.enabled.with.vcr.cluster", "true");
    recoveryProperties.setProperty("clustermap.vcr.datacenter.name", cloudDc);
    if (!vcrRecoveryPartitionConfig.isEmpty()) {
        recoveryProperties.setProperty("vcr.recovery.partitions", vcrRecoveryPartitionConfig);
    }
    TestSSLUtils.addHttp2Properties(recoveryProperties, SSLFactory.Mode.SERVER, true);
    // create vcr node
    List<Port> vcrPortList = Arrays.asList(new Port(12310, PortType.PLAINTEXT), new Port(12410, PortType.SSL));
    MockDataNodeId vcrNode = new MockDataNodeId("localhost", vcrPortList, Collections.singletonList(vcrMountPath), cloudDc);
    // create ambry server recovery cluster
    MockClusterMap serverClusterMap = new MockClusterMap(false, true, 2, 1, 1, true, false, null);
    recoveryCluster = new MockCluster(serverClusterMap, Collections.singletonList(vcrNode), recoveryProperties);
    partitionId = recoveryCluster.getClusterMap().getWritablePartitionIds(null).get(0);
    allRecoveryNodes = serverClusterMap.getDataNodes();
    // record ambry server node which will get partition leadership notification.
    partitionLeaderRecoveryNode = allRecoveryNodes.get(0);
    MockClusterAgentsFactory leaderMockClusterAgentsFactory = new MockClusterAgentsFactory(serverClusterMap, serverClusterMap.getAllPartitionIds(null).stream().map(PartitionId::toPathString).collect(Collectors.toList()));
    // Start Helix Controller and ZK Server.
    if (!zkInfo.isZkServerStarted()) {
        zkInfo.startZkServer();
    }
    helixControllerManager = VcrTestUtil.populateZkInfoAndStartController(zkConnectString, vcrClusterName, recoveryCluster.getClusterMap());
    Properties vcrProperties = VcrTestUtil.createVcrProperties(vcrNode.getDatacenterName(), vcrClusterName, zkConnectString, 12310, 12410, 12510, null);
    vcrProperties.putAll(recoveryProperties);
    MockNotificationSystem notificationSystem = new MockNotificationSystem(recoveryCluster.getClusterMap());
    // Create blobs and data for upload to vcr.
    int blobCount = 10;
    cloudBlobIds = ServerTestUtil.createBlobIds(blobCount, recoveryCluster.getClusterMap(), accountId, containerId, partitionId);
    serverBlobIds = ServerTestUtil.createBlobIds(blobCount, recoveryCluster.getClusterMap(), accountId, containerId, partitionId);
    // Create cloud destination and start vcr server.
    latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(cloudBlobIds, recoveryCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(vcrProperties), recoveryCluster.getClusterAgentsFactory(), notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // initialize and start ambry servers
    for (MockDataNodeId serverNode : allRecoveryNodes) {
        AmbryServer server = recoveryCluster.initializeServer(serverNode, recoveryProperties, false, notificationSystem, SystemTime.getInstance(), serverNode.equals(partitionLeaderRecoveryNode) ? leaderMockClusterAgentsFactory : null);
        recoveryCluster.addServer(server);
    }
    recoveryCluster.startServers();
}

Example 5 with LatchBasedInMemoryCloudDestination

use of com.github.ambry.cloud.LatchBasedInMemoryCloudDestination in project ambry by linkedin.

the class VcrBackupTest method singleNodeUpDownTestWithPersist.

/**
 * Test single VCR up and down with persisted token.
 */
@Test
public void singleNodeUpDownTestWithPersist() throws Exception {
    StrictMatchExternalViewVerifier helixBalanceVerifier = new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName, Collections.singleton(VcrTestUtil.helixResource), null);
    int numberOfBlobs = 20;
    sendBlobToDataNode(dataNode, numberOfBlobs);
    // Create in memory cloud destination.
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    // Start the VCR with token persistor on.
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, clusterMapPort, 12410, 12510, null, vcrHelixStateModelFactoryClass, true);
    props.setProperty("replication.persist.token.on.shutdown.or.replica.remove", "true");
    MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    final MockNotificationSystem vcrNotificationSystemCopy = vcrNotificationSystem;
    assertTrue("Blob count is not correct.", TestUtils.checkAndSleep(numberOfBlobs, () -> vcrNotificationSystemCopy.getBlobIds().size(), 400));
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    assertTrue("Token is expected.", latchBasedInMemoryCloudDestination.getTokenMap().size() > 0);
    // Start VCR again with same cloud destination
    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    // Because token is reloaded, back up number is 0.
    assertEquals("Number of blobs doesn't match", 0, vcrNotificationSystem.getBlobIds().size());
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    // Start VCR again with token.
    LatchBasedInMemoryCloudDestination newLatchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    for (Map.Entry<String, byte[]> entry : latchBasedInMemoryCloudDestination.getTokenMap().entrySet()) {
        newLatchBasedInMemoryCloudDestination.getTokenMap().put(entry.getKey(), entry.getValue());
    }
    cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    final MockNotificationSystem vcrNotificationSystemCopy2 = vcrNotificationSystem;
    assertTrue("Blob count is not correct.", TestUtils.checkAndSleep(0, () -> vcrNotificationSystemCopy2.getBlobIds().size(), 400));
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
}