Search in sources :

Example 1 with StoreKeyFactory

use of com.github.ambry.store.StoreKeyFactory in project ambry by linkedin.

the class MessageFormatInputStreamTest method messageFormatPutNoArgReadTest.

/**
 * Test calling the no-arg read method
 * @throws IOException
 * @throws MessageFormatException
 */
@Test
public void messageFormatPutNoArgReadTest() throws Exception, MessageFormatException {
    StoreKey key = new MockId("id1");
    StoreKeyFactory keyFactory = new MockIdFactory();
    short accountId = Utils.getRandomShort(TestUtils.RANDOM);
    short containerId = Utils.getRandomShort(TestUtils.RANDOM);
    BlobProperties prop = new BlobProperties(10, "servid", accountId, containerId, false);
    byte[] encryptionKey = new byte[100];
    new Random().nextBytes(encryptionKey);
    byte[] usermetadata = new byte[1000];
    new Random().nextBytes(usermetadata);
    int blobContentSize = 2000;
    byte[] data = new byte[blobContentSize];
    new Random().nextBytes(data);
    ByteBufferInputStream stream = new ByteBufferInputStream(ByteBuffer.wrap(data));
    MessageFormatInputStream messageFormatStream = new PutMessageFormatInputStream(key, ByteBuffer.wrap(encryptionKey), prop, ByteBuffer.wrap(usermetadata), stream, blobContentSize, BlobType.DataBlob);
    TestUtils.validateInputStreamContract(messageFormatStream);
    TestUtils.readInputStreamAndValidateSize(messageFormatStream, messageFormatStream.getSize());
}
Also used : StoreKeyFactory(com.github.ambry.store.StoreKeyFactory) Random(java.util.Random) MockIdFactory(com.github.ambry.store.MockIdFactory) ByteBufferInputStream(com.github.ambry.utils.ByteBufferInputStream) MockId(com.github.ambry.store.MockId) StoreKey(com.github.ambry.store.StoreKey) Test(org.junit.Test)

Example 2 with StoreKeyFactory

use of com.github.ambry.store.StoreKeyFactory in project ambry by linkedin.

the class BlobStoreHardDeleteTest method blobStoreHardDeleteTestUtil.

private void blobStoreHardDeleteTestUtil(short[] blobVersions, BlobType[] blobTypes) throws MessageFormatException, IOException {
    MessageStoreHardDelete hardDelete = new BlobStoreHardDelete();
    StoreKeyFactory keyFactory = new MockIdFactory();
    // create log and write to it
    ReadImp readImp = new ReadImp();
    ArrayList<Long> msgOffsets = readImp.initialize(blobVersions, blobTypes);
    Iterator<HardDeleteInfo> iter = hardDelete.getHardDeleteMessages(readImp.getMessageReadSet(), keyFactory, readImp.getRecoveryInfoList());
    List<HardDeleteInfo> hardDeletedList = new ArrayList<HardDeleteInfo>();
    while (iter.hasNext()) {
        hardDeletedList.add(iter.next());
    }
    // msg1
    HardDeleteInfo hardDeleteInfo = hardDeletedList.get(0);
    Assert.assertNotNull(hardDeleteInfo);
    HardDeleteRecoveryMetadata hardDeleteRecoveryMetadata = new HardDeleteRecoveryMetadata(hardDeleteInfo.getRecoveryInfo(), keyFactory);
    Assert.assertEquals(blobTypes[1], hardDeleteRecoveryMetadata.getBlobType());
    Assert.assertEquals(blobVersions[1], hardDeleteRecoveryMetadata.getBlobRecordVersion());
    // msg2
    hardDeleteInfo = hardDeletedList.get(1);
    Assert.assertNotNull(hardDeleteInfo);
    hardDeleteRecoveryMetadata = new HardDeleteRecoveryMetadata(hardDeleteInfo.getRecoveryInfo(), keyFactory);
    Assert.assertEquals(blobTypes[2], hardDeleteRecoveryMetadata.getBlobType());
    Assert.assertEquals(blobVersions[2], hardDeleteRecoveryMetadata.getBlobRecordVersion());
    // msg5
    hardDeleteInfo = hardDeletedList.get(2);
    Assert.assertNotNull(hardDeleteInfo);
    hardDeleteRecoveryMetadata = new HardDeleteRecoveryMetadata(hardDeleteInfo.getRecoveryInfo(), keyFactory);
    Assert.assertEquals(blobTypes[3], hardDeleteRecoveryMetadata.getBlobType());
    Assert.assertEquals(blobVersions[3], hardDeleteRecoveryMetadata.getBlobRecordVersion());
    // msg6 - NULL.
    Assert.assertNull(hardDeletedList.get(3));
}
Also used : StoreKeyFactory(com.github.ambry.store.StoreKeyFactory) MockIdFactory(com.github.ambry.store.MockIdFactory) HardDeleteInfo(com.github.ambry.store.HardDeleteInfo) ArrayList(java.util.ArrayList) MessageStoreHardDelete(com.github.ambry.store.MessageStoreHardDelete)

Example 3 with StoreKeyFactory

use of com.github.ambry.store.StoreKeyFactory in project ambry by linkedin.

the class ReplicationTest method blockDeprecatedContainerReplicationTest.

/**
 * Tests if deprecated containers have been blocked during replication.
 */
@Test
public void blockDeprecatedContainerReplicationTest() throws Exception {
    Properties properties = new Properties();
    properties.setProperty("replication.container.deletion.enabled", "true");
    replicationConfig = new ReplicationConfig(new VerifiableProperties(properties));
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    Map<StoreKey, StoreKey> conversionMap = new HashMap<>();
    storeKeyConverter.setConversionMap(conversionMap);
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    for (int i = 0; i < partitionIds.size(); i++) {
        PartitionId partitionId = partitionIds.get(i);
        BlobId b0 = generateRandomBlobId(partitionId);
        conversionMap.put(b0, b0);
        BlobId b1 = generateRandomBlobId(partitionId);
        conversionMap.put(b1, b1);
        // add 2 messages to both hosts.
        storeKeyConverter.setConversionMap(conversionMap);
        storeKeyConverter.convert(conversionMap.keySet());
        // addPutMessagesToReplicasOfPartition(Arrays.asList(b0), Arrays.asList(localHost, remoteHost));
        // add 3 messages to the remote host only
        addPutMessagesToReplicasOfPartition(Arrays.asList(b0, b1), Collections.singletonList(remoteHost));
    }
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    Transformer transformer = new BlobIdTransformer(storeKeyFactory, storeKeyConverter);
    int batchSize = 4;
    ReplicationMetrics replicationMetrics = new ReplicationMetrics(new MetricRegistry(), clusterMap.getReplicaIds(localHost.dataNodeId));
    replicationMetrics.populateSingleColoMetrics(remoteHost.dataNodeId.getDatacenterName());
    List<RemoteReplicaInfo> remoteReplicaInfoList = localHost.getRemoteReplicaInfos(remoteHost, null);
    Map<DataNodeId, List<RemoteReplicaInfo>> replicasToReplicate = Collections.singletonMap(remoteHost.dataNodeId, remoteReplicaInfoList);
    storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", clusterMap);
    Map<DataNodeId, MockHost> hosts = new HashMap<>();
    hosts.put(remoteHost.dataNodeId, remoteHost);
    MockConnectionPool connectionPool = new MockConnectionPool(hosts, clusterMap, batchSize);
    Predicate<MessageInfo> skipPredicate = new ReplicationSkipPredicate(accountService, replicationConfig);
    ReplicaThread replicaThread = new ReplicaThread("threadtest", new MockFindTokenHelper(storeKeyFactory, replicationConfig), clusterMap, new AtomicInteger(0), localHost.dataNodeId, connectionPool, replicationConfig, replicationMetrics, null, storeKeyConverter, transformer, clusterMap.getMetricRegistry(), false, localHost.dataNodeId.getDatacenterName(), new ResponseHandler(clusterMap), time, null, skipPredicate);
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfoList) {
        replicaThread.addRemoteReplicaInfo(remoteReplicaInfo);
    }
    List<RemoteReplicaInfo> remoteReplicaInfos = replicasToReplicate.get(remoteHost.dataNodeId);
    DataNodeId remoteNode = remoteReplicaInfos.get(0).getReplicaId().getDataNodeId();
    ReplicaMetadataResponse response = replicaThread.getReplicaMetadataResponse(remoteReplicaInfos, new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteNode);
    // case1 DELETE_IN_PROGRESS container with retention time qualified.
    for (int i = 0; i < 2; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getDeleteTriggerTime()).thenReturn(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(replicationConfig.replicationContainerDeletionRetentionDays + 1));
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.DELETE_IN_PROGRESS);
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All DELETE_IN_PROGRESS blobs qualified with retention time should be skipped during replication", 0, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
    // case2 DELETE_IN_PROGRESS container with retention time not qualified.
    for (int i = 2; i < 4; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.DELETE_IN_PROGRESS);
            Mockito.when(container.getDeleteTriggerTime()).thenReturn(System.currentTimeMillis());
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All DELETE_IN_PROGRESS blobs not qualified with retention time should not be skipped during replication", 2, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
    // case3 INACTIVE container
    for (int i = 4; i < 6; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.INACTIVE);
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All INACTIVE blobs should be skipped during replication", 0, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
    // case 4 ACTIVE Container
    for (int i = 6; i < 8; i++) {
        RemoteReplicaInfo remoteReplicaInfo = remoteReplicaInfos.get(i);
        ReplicaMetadataResponseInfo replicaMetadataResponseInfo = response.getReplicaMetadataResponseInfoList().get(i);
        new ResponseHandler(clusterMap).onEvent(remoteReplicaInfo.getReplicaId(), replicaMetadataResponseInfo.getError());
        for (int j = 0; j < replicaMetadataResponseInfo.getMessageInfoList().size(); j++) {
            short accountId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getAccountId();
            short containerId = replicaMetadataResponseInfo.getMessageInfoList().get(j).getContainerId();
            Container container = Mockito.mock(Container.class);
            Account account = Mockito.mock(Account.class);
            Mockito.when(account.getContainerById(containerId)).thenReturn(container);
            Mockito.when(accountService.getAccountById(accountId)).thenReturn(account);
            Mockito.when(container.getStatus()).thenReturn(Container.ContainerStatus.ACTIVE);
        }
        Set<MessageInfo> remoteMissingStoreKeys = replicaThread.getMissingStoreMessages(replicaMetadataResponseInfo, remoteNode, remoteReplicaInfo);
        assertEquals("All non-deprecated blobs should not be skipped during replication", 2, remoteMissingStoreKeys.size());
        Map<StoreKey, StoreKey> remoteKeyToLocalKeyMap = replicaThread.batchConvertReplicaMetadataResponseKeys(response);
        replicaThread.processReplicaMetadataResponse(remoteMissingStoreKeys, replicaMetadataResponseInfo, remoteReplicaInfo, remoteNode, remoteKeyToLocalKeyMap);
    }
}
Also used : Account(com.github.ambry.account.Account) ValidatingTransformer(com.github.ambry.messageformat.ValidatingTransformer) Transformer(com.github.ambry.store.Transformer) ResponseHandler(com.github.ambry.commons.ResponseHandler) HashMap(java.util.HashMap) Properties(java.util.Properties) VerifiableProperties(com.github.ambry.config.VerifiableProperties) StoreKeyFactory(com.github.ambry.store.StoreKeyFactory) Container(com.github.ambry.account.Container) List(java.util.List) ArrayList(java.util.ArrayList) ReplicaMetadataResponse(com.github.ambry.protocol.ReplicaMetadataResponse) MockStoreKeyConverterFactory(com.github.ambry.store.MockStoreKeyConverterFactory) ReplicationConfig(com.github.ambry.config.ReplicationConfig) ReplicaMetadataResponseInfo(com.github.ambry.protocol.ReplicaMetadataResponseInfo) VerifiableProperties(com.github.ambry.config.VerifiableProperties) MetricRegistry(com.codahale.metrics.MetricRegistry) MockPartitionId(com.github.ambry.clustermap.MockPartitionId) PartitionId(com.github.ambry.clustermap.PartitionId) StoreKey(com.github.ambry.store.StoreKey) BlobIdFactory(com.github.ambry.commons.BlobIdFactory) MessageInfo(com.github.ambry.store.MessageInfo) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) BlobId(com.github.ambry.commons.BlobId) DataNodeId(com.github.ambry.clustermap.DataNodeId) MockDataNodeId(com.github.ambry.clustermap.MockDataNodeId) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) Test(org.junit.Test)

Example 4 with StoreKeyFactory

use of com.github.ambry.store.StoreKeyFactory in project ambry by linkedin.

the class ReplicationTest method replicaThreadLifeVersionLocalLessThanRemote_FinalState_TtlUpdateDelete.

/**
 * Tests when the lifeVersion in local is less than the lifeVersion in remote and the final state from remote
 * is delete with ttl update.
 * @throws Exception
 */
@Test
public void replicaThreadLifeVersionLocalLessThanRemote_FinalState_TtlUpdateDelete() throws Exception {
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    MockStoreKeyConverterFactory storeKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    storeKeyConverterFactory.setConversionMap(new HashMap<>());
    storeKeyConverterFactory.setReturnInputIfAbsent(true);
    MockStoreKeyConverterFactory.MockStoreKeyConverter storeKeyConverter = storeKeyConverterFactory.getStoreKeyConverter();
    Map<StoreKey, StoreKey> conversionMap = new HashMap<>();
    storeKeyConverter.setConversionMap(conversionMap);
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    Transformer transformer = new BlobIdTransformer(storeKeyFactory, storeKeyConverter);
    Map<PartitionId, List<StoreKey>> idsByPartition = new HashMap<>();
    Map<PartitionId, StoreKey> idsToBeIgnoredByPartition = new HashMap<>();
    List<PartitionId> partitionIds = clusterMap.getWritablePartitionIds(null);
    // 5 P, D -> [U, T, D]
    for (int i = 0; i < partitionIds.size(); i++) {
        PartitionId partitionId = partitionIds.get(i);
        List<StoreKey> ids = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost), 1);
        // 1 Missing
        StoreKey id = ids.get(0);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), UPDATED_EXPIRY_TIME_MS, (short) 0);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), (short) 1, UPDATED_EXPIRY_TIME_MS);
        idsToBeIgnoredByPartition.put(partitionId, id);
        // 2 P -> [T, D]
        id = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost, localHost), 1).get(0);
        ids.add(id);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), UPDATED_EXPIRY_TIME_MS, (short) 0);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), (short) 1, UPDATED_EXPIRY_TIME_MS);
        // 3 P, T -> [D]
        id = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost, localHost), 1).get(0);
        ids.add(id);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, id, Arrays.asList(localHost, remoteHost), UPDATED_EXPIRY_TIME_MS, (short) 0);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), (short) 1, UPDATED_EXPIRY_TIME_MS);
        // 4 P, T, D -> [D]
        id = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost, localHost), 1).get(0);
        ids.add(id);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, id, Arrays.asList(localHost, remoteHost), UPDATED_EXPIRY_TIME_MS, (short) 0);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(localHost), (short) 0, UPDATED_EXPIRY_TIME_MS);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), (short) 1, UPDATED_EXPIRY_TIME_MS);
        // 5 P, D -> [U, T, D]
        id = addPutMessagesToReplicasOfPartition(partitionId, Arrays.asList(remoteHost, localHost), 1).get(0);
        ids.add(id);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(localHost), (short) 0, EXPIRY_TIME_MS);
        addTtlUpdateMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), UPDATED_EXPIRY_TIME_MS, (short) 0);
        addDeleteMessagesToReplicasOfPartition(partitionId, id, Collections.singletonList(remoteHost), (short) 1, UPDATED_EXPIRY_TIME_MS);
        ids.add(id);
        idsByPartition.put(partitionId, ids);
    }
    int batchSize = 100;
    Pair<Map<DataNodeId, List<RemoteReplicaInfo>>, ReplicaThread> replicasAndThread = getRemoteReplicasAndReplicaThread(batchSize, clusterMap, localHost, remoteHost, storeKeyConverter, transformer, null, null);
    List<RemoteReplicaInfo> remoteReplicaInfos = replicasAndThread.getFirst().get(remoteHost.dataNodeId);
    ReplicaThread replicaThread = replicasAndThread.getSecond();
    // It's all deletes, there is no missing key.
    List<ReplicaThread.ExchangeMetadataResponse> response = replicaThread.exchangeMetadata(new MockConnectionPool.MockConnection(remoteHost, batchSize), remoteReplicaInfos);
    assertEquals("Response should contain a response for each replica", remoteReplicaInfos.size(), response.size());
    for (int i = 0; i < response.size(); i++) {
        assertEquals(0, response.get(i).missingStoreMessages.size());
        remoteReplicaInfos.get(i).setToken(response.get(i).remoteToken);
    }
    // Before exchange metadata, the number of message infos in local host is 8. Exchange metadata would add another 7.
    for (Map.Entry<PartitionId, List<MessageInfo>> localInfoEntry : localHost.infosByPartition.entrySet()) {
        assertEquals("MessageInfo number mismatch", 15, localInfoEntry.getValue().size());
    }
    for (Map.Entry<PartitionId, List<StoreKey>> idsEntry : idsByPartition.entrySet()) {
        List<MessageInfo> remoteInfos = remoteHost.infosByPartition.get(idsEntry.getKey());
        List<MessageInfo> localInfos = localHost.infosByPartition.get(idsEntry.getKey());
        for (StoreKey id : idsEntry.getValue()) {
            if (!idsToBeIgnoredByPartition.get(idsEntry.getKey()).equals(id)) {
                MessageInfo localInfo = getMergedMessageInfo(id, localInfos);
                MessageInfo remoteInfo = getMergedMessageInfo(id, remoteInfos);
                assertTrue(localInfo.isDeleted());
                assertTrue(remoteInfo.isDeleted());
                assertTrue(localInfo.isTtlUpdated());
                assertTrue(remoteInfo.isTtlUpdated());
                assertEquals(localInfo.getLifeVersion(), remoteInfo.getLifeVersion());
            }
        }
    }
}
Also used : ValidatingTransformer(com.github.ambry.messageformat.ValidatingTransformer) Transformer(com.github.ambry.store.Transformer) HashMap(java.util.HashMap) StoreKeyFactory(com.github.ambry.store.StoreKeyFactory) List(java.util.List) ArrayList(java.util.ArrayList) MockStoreKeyConverterFactory(com.github.ambry.store.MockStoreKeyConverterFactory) MockPartitionId(com.github.ambry.clustermap.MockPartitionId) PartitionId(com.github.ambry.clustermap.PartitionId) StoreKey(com.github.ambry.store.StoreKey) BlobIdFactory(com.github.ambry.commons.BlobIdFactory) MessageInfo(com.github.ambry.store.MessageInfo) Map(java.util.Map) HashMap(java.util.HashMap) ClusterMap(com.github.ambry.clustermap.ClusterMap) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) Test(org.junit.Test)

Example 5 with StoreKeyFactory

use of com.github.ambry.store.StoreKeyFactory in project ambry by linkedin.

the class ReplicationTest method remoteReplicaInfoAddRemoveTest.

/**
 * Tests add/remove replicaInfo to {@link ReplicaThread}
 * @throws Exception
 */
@Test
public void remoteReplicaInfoAddRemoveTest() throws Exception {
    MockClusterMap clusterMap = new MockClusterMap();
    Pair<MockHost, MockHost> localAndRemoteHosts = getLocalAndRemoteHosts(clusterMap);
    MockHost localHost = localAndRemoteHosts.getFirst();
    MockHost remoteHost = localAndRemoteHosts.getSecond();
    StoreKeyFactory storeKeyFactory = Utils.getObj("com.github.ambry.commons.BlobIdFactory", clusterMap);
    MockStoreKeyConverterFactory mockStoreKeyConverterFactory = new MockStoreKeyConverterFactory(null, null);
    mockStoreKeyConverterFactory.setReturnInputIfAbsent(true);
    mockStoreKeyConverterFactory.setConversionMap(new HashMap<>());
    StoreKeyConverter storeKeyConverter = mockStoreKeyConverterFactory.getStoreKeyConverter();
    Transformer transformer = new ValidatingTransformer(storeKeyFactory, storeKeyConverter);
    ReplicationMetrics replicationMetrics = new ReplicationMetrics(new MetricRegistry(), clusterMap.getReplicaIds(localHost.dataNodeId));
    replicationMetrics.populateSingleColoMetrics(remoteHost.dataNodeId.getDatacenterName());
    List<RemoteReplicaInfo> remoteReplicaInfoList = localHost.getRemoteReplicaInfos(remoteHost, null);
    Map<DataNodeId, MockHost> hosts = new HashMap<>();
    hosts.put(remoteHost.dataNodeId, remoteHost);
    MockConnectionPool connectionPool = new MockConnectionPool(hosts, clusterMap, 4);
    ReplicaThread replicaThread = new ReplicaThread("threadtest", new MockFindTokenHelper(storeKeyFactory, replicationConfig), clusterMap, new AtomicInteger(0), localHost.dataNodeId, connectionPool, replicationConfig, replicationMetrics, null, mockStoreKeyConverterFactory.getStoreKeyConverter(), transformer, clusterMap.getMetricRegistry(), false, localHost.dataNodeId.getDatacenterName(), new ResponseHandler(clusterMap), time, null, null, null);
    for (RemoteReplicaInfo remoteReplicaInfo : remoteReplicaInfoList) {
        replicaThread.addRemoteReplicaInfo(remoteReplicaInfo);
    }
    List<RemoteReplicaInfo> actualRemoteReplicaInfoList = replicaThread.getRemoteReplicaInfos().get(remoteHost.dataNodeId);
    Comparator<RemoteReplicaInfo> remoteReplicaInfoComparator = Comparator.comparing(info -> info.getReplicaId().getPartitionId().toPathString());
    Collections.sort(remoteReplicaInfoList, remoteReplicaInfoComparator);
    Collections.sort(actualRemoteReplicaInfoList, remoteReplicaInfoComparator);
    assertEquals("getRemoteReplicaInfos not correct", remoteReplicaInfoList, actualRemoteReplicaInfoList);
    // Test remove remoteReplicaInfo.
    replicaThread.removeRemoteReplicaInfo(remoteReplicaInfoList.get(remoteReplicaInfoList.size() - 1));
    actualRemoteReplicaInfoList = replicaThread.getRemoteReplicaInfos().get(remoteHost.dataNodeId);
    Collections.sort(actualRemoteReplicaInfoList, remoteReplicaInfoComparator);
    remoteReplicaInfoList.remove(remoteReplicaInfoList.size() - 1);
    assertEquals("getRemoteReplicaInfos not correct", remoteReplicaInfoList, actualRemoteReplicaInfoList);
}
Also used : MockStoreKeyConverterFactory(com.github.ambry.store.MockStoreKeyConverterFactory) ValidatingTransformer(com.github.ambry.messageformat.ValidatingTransformer) Transformer(com.github.ambry.store.Transformer) ResponseHandler(com.github.ambry.commons.ResponseHandler) HashMap(java.util.HashMap) MetricRegistry(com.codahale.metrics.MetricRegistry) StoreKeyFactory(com.github.ambry.store.StoreKeyFactory) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ValidatingTransformer(com.github.ambry.messageformat.ValidatingTransformer) StoreKeyConverter(com.github.ambry.store.StoreKeyConverter) DataNodeId(com.github.ambry.clustermap.DataNodeId) MockDataNodeId(com.github.ambry.clustermap.MockDataNodeId) MockClusterMap(com.github.ambry.clustermap.MockClusterMap) Test(org.junit.Test)

Aggregations

StoreKeyFactory (com.github.ambry.store.StoreKeyFactory)27 ArrayList (java.util.ArrayList)19 HashMap (java.util.HashMap)17 MockPartitionId (com.github.ambry.clustermap.MockPartitionId)16 PartitionId (com.github.ambry.clustermap.PartitionId)16 MockStoreKeyConverterFactory (com.github.ambry.store.MockStoreKeyConverterFactory)16 Test (org.junit.Test)16 MockClusterMap (com.github.ambry.clustermap.MockClusterMap)15 BlobIdFactory (com.github.ambry.commons.BlobIdFactory)15 Transformer (com.github.ambry.store.Transformer)15 DataNodeId (com.github.ambry.clustermap.DataNodeId)14 ValidatingTransformer (com.github.ambry.messageformat.ValidatingTransformer)14 List (java.util.List)14 ClusterMap (com.github.ambry.clustermap.ClusterMap)13 Map (java.util.Map)13 StoreKey (com.github.ambry.store.StoreKey)12 MessageInfo (com.github.ambry.store.MessageInfo)9 MockDataNodeId (com.github.ambry.clustermap.MockDataNodeId)8 MetricRegistry (com.codahale.metrics.MetricRegistry)6 ClusterMapConfig (com.github.ambry.config.ClusterMapConfig)6