Examples with LatchBasedInMemoryCloudDestination com.github.ambry.cloud.LatchBasedInMemoryCloudDestination used on opensource projects

Example 6 with LatchBasedInMemoryCloudDestination

use of com.github.ambry.cloud.LatchBasedInMemoryCloudDestination in project ambry by linkedin.

the class VcrBackupTest method singleNodeUpDownTestWithoutPersist.

/**
 * Test single VCR up and down without persisted token.
 */
@Test
public void singleNodeUpDownTestWithoutPersist() throws Exception {
    StrictMatchExternalViewVerifier helixBalanceVerifier = new StrictMatchExternalViewVerifier(zkConnectString, vcrClusterName, Collections.singleton(VcrTestUtil.helixResource), null);
    int numberOfBlobs = 20;
    sendBlobToDataNode(dataNode, numberOfBlobs);
    // Create in memory cloud destination.
    LatchBasedInMemoryCloudDestination latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    // Start the VCR with token persistor off.
    Properties props = VcrTestUtil.createVcrProperties(dataNode.getDatacenterName(), vcrClusterName, zkConnectString, clusterMapPort, 12410, 12510, null, vcrHelixStateModelFactoryClass, true);
    props.setProperty("replication.persist.token.on.shutdown.or.replica.remove", "false");
    MockNotificationSystem vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    VcrServer vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    final MockNotificationSystem vcrNotificationSystemCopy = vcrNotificationSystem;
    assertTrue("Blob count is not correct.", TestUtils.checkAndSleep(numberOfBlobs, () -> vcrNotificationSystemCopy.getBlobIds().size(), 400));
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    assertEquals("No token is expected.", 0, latchBasedInMemoryCloudDestination.getTokenMap().size());
    // Start VCR again with same cloud destination
    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    // Because same cloud destination is used, getMissingKey() will filter out all keys.
    assertEquals("Number of blobs doesn't match", 0, vcrNotificationSystem.getBlobIds().size());
    vcrServer.shutdown();
    assertTrue("VCR server shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
    // Start VCR again with different cloud destination
    latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(new ArrayList<>(), mockCluster.getClusterMap());
    cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    vcrNotificationSystem = new MockNotificationSystem(mockCluster.getClusterMap());
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(props), mockCluster.getClusterAgentsFactory(), vcrNotificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    makeSureHelixBalance(vcrServer, helixBalanceVerifier);
    // Because same cloud destination is not used and no token persisted, everything will be backed again.
    final MockNotificationSystem vcrNotificationSystemCopy2 = vcrNotificationSystem;
    assertTrue("Blob count is not correct.", TestUtils.checkAndSleep(numberOfBlobs, () -> vcrNotificationSystemCopy2.getBlobIds().size(), 200));
    vcrServer.shutdown();
    assertTrue("VCR shutdown timeout.", vcrServer.awaitShutdown(5000));
    // Error metrics should be zero.
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().addPartitionErrorCount.getCount());
    Assert.assertEquals("Error count should be zero", 0, vcrServer.getVcrReplicationManager().getVcrMetrics().removePartitionErrorCount.getCount());
}

Example 7 with LatchBasedInMemoryCloudDestination

use of com.github.ambry.cloud.LatchBasedInMemoryCloudDestination in project ambry by linkedin.

the class VcrRecoveryTest method setup.

/**
 * Create a cluster with a vcr node and a recovery (ambry data) node.
 * @throws Exception on {@link Exception}
 */
@Before
public void setup() throws Exception {
    String vcrMountPath = ClusterMapSnapshotConstants.CLOUD_REPLICA_MOUNT + "/1";
    recoveryProperties = new Properties();
    recoveryProperties.setProperty("replication.metadata.request.version", "2");
    // create vcr node
    List<Port> vcrPortList = new ArrayList<>(2);
    Port vcrClusterMapPort = new Port(12310, PortType.PLAINTEXT);
    Port vcrSslPort = new Port(12410, PortType.SSL);
    vcrPortList.add(vcrClusterMapPort);
    vcrPortList.add(vcrSslPort);
    MockDataNodeId vcrNode = new MockDataNodeId("localhost", vcrPortList, Collections.singletonList(vcrMountPath), dcName);
    // create recovery node
    recoveryNodePort = new Port(12311, PortType.PLAINTEXT);
    ArrayList<Port> recoveryPortList = new ArrayList<>(2);
    recoveryPortList.add(recoveryNodePort);
    recoveryNode = MockClusterMap.createDataNode(recoveryPortList, dcName, 1);
    // create cluster for recovery
    recoveryCluster = MockCluster.createOneNodeRecoveryCluster(vcrNode, recoveryNode, dcName);
    partitionId = recoveryCluster.getClusterMap().getWritablePartitionIds(null).get(0);
    // Start ZK Server and Helix Controller.
    if (!zkInfo.isZkServerStarted()) {
        zkInfo.startZkServer();
    }
    helixControllerManager = VcrTestUtil.populateZkInfoAndStartController(zkConnectString, vcrClusterName, recoveryCluster.getClusterMap());
    Properties vcrProperties = VcrTestUtil.createVcrProperties(vcrNode.getDatacenterName(), vcrClusterName, zkConnectString, 12310, 12410, 12510, null);
    vcrProperties.putAll(recoveryProperties);
    NotificationSystem notificationSystem = new MockNotificationSystem(recoveryCluster.getClusterMap());
    // Create blobs and data for upload to vcr.
    int blobCount = 10;
    blobIds = ServerTestUtil.createBlobIds(blobCount, recoveryCluster.getClusterMap(), accountId, containerId, partitionId);
    // Create cloud destination and start vcr server.
    latchBasedInMemoryCloudDestination = new LatchBasedInMemoryCloudDestination(blobIds, recoveryCluster.getClusterMap());
    CloudDestinationFactory cloudDestinationFactory = new LatchBasedInMemoryCloudDestinationFactory(latchBasedInMemoryCloudDestination);
    vcrServer = VcrTestUtil.createVcrServer(new VerifiableProperties(vcrProperties), recoveryCluster.getClusterAgentsFactory(), notificationSystem, cloudDestinationFactory);
    vcrServer.startup();
    // start ambry server with data node
    recoveryCluster.initializeServers(notificationSystem, vcrNode, recoveryProperties);
    recoveryCluster.startServers();
}