Examples with DataNodeId com.github.ambry.clustermap.DataNodeId used on opensource projects

Example 31 with DataNodeId

use of com.github.ambry.clustermap.DataNodeId in project ambry by linkedin.

the class ReplicationTestHelper method getRemoteNodesFromLocalAndRemoteDCs.

/**
 * Get a pair of data nodes (one from local data center and one from remote data center) which share partitions
 * with local node.
 * @return list of data nodes that share partitions with local node.
 */
protected List<DataNodeId> getRemoteNodesFromLocalAndRemoteDCs(ClusterMap clusterMap, DataNodeId localNode) {
    List<DataNodeId> remoteNodesFromLocalAndRemoteDCs = new ArrayList<>();
    String currentDataCenter = localNode.getDatacenterName();
    DataNodeId remoteNodeInLocalDC = null;
    DataNodeId remoteNodeInRemoteDC = null;
    List<? extends ReplicaId> replicaIds = clusterMap.getReplicaIds(localNode);
    MockPartitionId existingPartition = (MockPartitionId) replicaIds.get(0).getPartitionId();
    for (ReplicaId replicaId : existingPartition.getReplicaIds()) {
        if (!replicaId.getDataNodeId().equals(localNode)) {
            if (replicaId.getDataNodeId().getDatacenterName().equals(currentDataCenter)) {
                if (remoteNodeInLocalDC == null) {
                    remoteNodeInLocalDC = replicaId.getDataNodeId();
                }
            } else if (remoteNodeInRemoteDC == null) {
                remoteNodeInRemoteDC = replicaId.getDataNodeId();
            }
        }
        if (remoteNodeInLocalDC != null && remoteNodeInRemoteDC != null) {
            break;
        }
    }
    remoteNodesFromLocalAndRemoteDCs.add(remoteNodeInLocalDC);
    remoteNodesFromLocalAndRemoteDCs.add(remoteNodeInRemoteDC);
    return remoteNodesFromLocalAndRemoteDCs;
}

Example 32 with DataNodeId

use of com.github.ambry.clustermap.DataNodeId in project ambry by linkedin.

the class CloudRouterFactory method getRequestHandlerPool.

/**
 * Utility method to build  a {@link RequestHandlerPool}.
 * @param verifiableProperties the properties to use.
 * @param clusterMap the {@link ClusterMap} to use.
 * @return the constructed {@link RequestHandlerPool}.
 * @throws Exception if the construction fails.
 */
public RequestHandlerPool getRequestHandlerPool(VerifiableProperties verifiableProperties, ClusterMap clusterMap, CloudDestination cloudDestination, CloudConfig cloudConfig) throws Exception {
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    MetricRegistry registry = clusterMap.getMetricRegistry();
    DataNodeId nodeId = new CloudDataNode(cloudConfig, clusterMapConfig);
    VcrMetrics vcrMetrics = new VcrMetrics(registry);
    StoreManager cloudStorageManager = new CloudStorageManager(verifiableProperties, vcrMetrics, cloudDestination, clusterMap);
    LocalRequestResponseChannel channel = new LocalRequestResponseChannel();
    ServerMetrics serverMetrics = new ServerMetrics(registry, AmbryRequests.class);
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(routerConfig.routerStoreKeyConverterFactory, verifiableProperties, registry);
    // A null notification system is passed into AmbryRequests so that replication events are not emitted from a
    // frontend.
    AmbryRequests requests = new AmbryRequests(cloudStorageManager, channel, clusterMap, nodeId, registry, serverMetrics, null, null, null, storeKeyFactory, storeKeyConverterFactory);
    return new RequestHandlerPool(routerConfig.routerRequestHandlerNumOfThreads, channel, requests);
}

Example 33 with DataNodeId

use of com.github.ambry.clustermap.DataNodeId in project ambry by linkedin.

the class CryptoJobMetricsTracker method initializeResourceToHistogramMap.

/**
 * Initialize resource-to-latency-histogram maps based on given resource type. Here resource can be {@link PartitionId},
 * {@link DataNodeId}, etc. The resource type is defined by {@link RouterConfig#routerOperationTrackerMetricScope}.
 * @param clusterMap the {@link ClusterMap} that contains info of all resources.
 * @param routerConfig the {@link RouterConfig} that specifies histogram parameters.
 */
private void initializeResourceToHistogramMap(ClusterMap clusterMap, RouterConfig routerConfig) {
    String localDatacenterName = clusterMap.getDatacenterName(clusterMap.getLocalDatacenterId());
    switch(routerConfig.routerOperationTrackerMetricScope) {
        case Partition:
            for (PartitionId partitionId : clusterMap.getAllPartitionIds(null)) {
                getBlobLocalDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                getBlobInfoLocalDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                getBlobCrossDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                getBlobInfoCrossDcResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
                putBlobResourceToLatency.put(partitionId, createHistogram(routerConfig, false));
            }
            break;
        case DataNode:
            List<? extends DataNodeId> dataNodeIds = clusterMap.getDataNodeIds();
            for (DataNodeId dataNodeId : dataNodeIds) {
                if (dataNodeId.getDatacenterName().equals(localDatacenterName)) {
                    getBlobLocalDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                    getBlobInfoLocalDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                    // Put blob only cares abou local db data nodes.
                    putBlobResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                } else {
                    getBlobCrossDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                    getBlobInfoCrossDcResourceToLatency.put(dataNodeId, createHistogram(routerConfig, false));
                }
            }
            break;
        case Disk:
            for (PartitionId partitionId : clusterMap.getAllPartitionIds(null)) {
                for (ReplicaId replicaId : partitionId.getReplicaIds()) {
                    DiskId diskId = replicaId.getDiskId();
                    if (getBlobLocalDcResourceToLatency.containsKey(diskId) || getBlobCrossDcResourceToLatency.containsKey(diskId)) {
                        continue;
                    }
                    if (replicaId.getDataNodeId().getDatacenterName().equals(localDatacenterName)) {
                        getBlobLocalDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                        getBlobInfoLocalDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                        putBlobResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                    } else {
                        getBlobCrossDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                        getBlobInfoCrossDcResourceToLatency.put(diskId, createHistogram(routerConfig, false));
                    }
                }
            }
        default:
    }
}

Example 34 with DataNodeId

use of com.github.ambry.clustermap.DataNodeId in project ambry by linkedin.

the class NonBlockingRouterTest method testUnsuccessfulPutDataChunkDelete.

/**
 * Test that if a composite blob put fails, the successfully put data chunks are deleted.
 */
@Test
public void testUnsuccessfulPutDataChunkDelete() throws Exception {
    try {
        // Ensure there are 4 chunks.
        maxPutChunkSize = PUT_CONTENT_SIZE / 4;
        Properties props = getNonBlockingRouterProperties("DC1");
        VerifiableProperties verifiableProperties = new VerifiableProperties((props));
        RouterConfig routerConfig = new RouterConfig(verifiableProperties);
        MockClusterMap mockClusterMap = new MockClusterMap();
        MockTime mockTime = new MockTime();
        MockServerLayout mockServerLayout = new MockServerLayout(mockClusterMap);
        // Since this test wants to ensure that successfully put data chunks are deleted when the overall put operation
        // fails, it uses a notification system to track the deletions.
        final CountDownLatch deletesDoneLatch = new CountDownLatch(2);
        final Map<String, String> blobsThatAreDeleted = new HashMap<>();
        LoggingNotificationSystem deleteTrackingNotificationSystem = new LoggingNotificationSystem() {

            @Override
            public void onBlobDeleted(String blobId, String serviceId, Account account, Container container) {
                blobsThatAreDeleted.put(blobId, serviceId);
                deletesDoneLatch.countDown();
            }
        };
        router = new NonBlockingRouter(routerConfig, new NonBlockingRouterMetrics(mockClusterMap, routerConfig), new MockNetworkClientFactory(verifiableProperties, mockSelectorState, MAX_PORTS_PLAIN_TEXT, MAX_PORTS_SSL, CHECKOUT_TIMEOUT_MS, mockServerLayout, mockTime), deleteTrackingNotificationSystem, mockClusterMap, kms, cryptoService, cryptoJobHandler, accountService, mockTime, MockClusterMap.DEFAULT_PARTITION_CLASS);
        setOperationParams();
        List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
        List<ServerErrorCode> serverErrorList = new ArrayList<>();
        // There are 4 chunks for this blob.
        // All put operations make one request to each local server as there are 3 servers overall in the local DC.
        // Set the state of the mock servers so that they return success for the first 2 requests in order to succeed
        // the first two chunks.
        serverErrorList.add(ServerErrorCode.No_Error);
        serverErrorList.add(ServerErrorCode.No_Error);
        // fail requests for third and fourth data chunks including the slipped put attempts:
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        serverErrorList.add(ServerErrorCode.Unknown_Error);
        // all subsequent requests (no more puts, but there will be deletes) will succeed.
        for (DataNodeId dataNodeId : dataNodeIds) {
            MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
            server.setServerErrors(serverErrorList);
        }
        // Submit the put operation and wait for it to fail.
        try {
            router.putBlob(putBlobProperties, putUserMetadata, putChannel, new PutBlobOptionsBuilder().build()).get();
        } catch (ExecutionException e) {
            Assert.assertEquals(RouterErrorCode.AmbryUnavailable, ((RouterException) e.getCause()).getErrorCode());
        }
        // Now, wait until the deletes of the successfully put blobs are complete.
        Assert.assertTrue("Deletes should not take longer than " + AWAIT_TIMEOUT_MS, deletesDoneLatch.await(AWAIT_TIMEOUT_MS, TimeUnit.MILLISECONDS));
        for (Map.Entry<String, String> blobIdAndServiceId : blobsThatAreDeleted.entrySet()) {
            Assert.assertEquals("Unexpected service ID for deleted blob", BackgroundDeleteRequest.SERVICE_ID_PREFIX + putBlobProperties.getServiceId(), blobIdAndServiceId.getValue());
        }
    } finally {
        if (router != null) {
            router.close();
            assertClosed();
            Assert.assertEquals("All operations should have completed", 0, router.getOperationsCount());
        }
    }
}

Example 35 with DataNodeId

use of com.github.ambry.clustermap.DataNodeId in project ambry by linkedin.

the class MockReadableStreamChannel method testLaterChunkFailure.

/**
 * Tests a case where some chunks succeed and a later chunk fails and ensures that the operation fails in
 * such a scenario.
 */
@Test
public void testLaterChunkFailure() throws Exception {
    // choose a blob with two chunks, first one will succeed and second will fail.
    requestAndResultsList.clear();
    requestAndResultsList.add(new RequestAndResult(chunkSize * 2));
    List<DataNodeId> dataNodeIds = mockClusterMap.getDataNodeIds();
    // Set the state of the mock servers so that they return success for the first send issued,
    // but later ones fail. With 3 nodes, all partitions will come from the same nodes,
    // so we set the errors in such a way that the first request received by every node succeeds and later ones fail.
    List<ServerErrorCode> serverErrorList = new ArrayList<>();
    serverErrorList.add(ServerErrorCode.No_Error);
    for (int i = 0; i < 5; i++) {
        serverErrorList.add(ServerErrorCode.Unknown_Error);
    }
    for (DataNodeId dataNodeId : dataNodeIds) {
        MockServer server = mockServerLayout.getMockServer(dataNodeId.getHostname(), dataNodeId.getPort());
        server.setServerErrors(serverErrorList);
    }
    Exception expectedException = new RouterException("", RouterErrorCode.AmbryUnavailable);
    submitPutsAndAssertFailure(expectedException, true, false, true);
}