Examples with ClusterMap com.github.ambry.clustermap.ClusterMap used on opensource projects

Example 41 with ClusterMap

use of com.github.ambry.clustermap.ClusterMap in project ambry by linkedin.

the class QuotaAwareOperationControllerTest method setupMocks.

@Before
public void setupMocks() throws Exception {
    NetworkClientFactory networkClientFactory = mock(NetworkClientFactory.class);
    NetworkClient networkClient = mock(NetworkClient.class);
    when(networkClientFactory.getNetworkClient()).thenReturn(networkClient);
    ClusterMap clusterMap = mock(ClusterMap.class);
    when(clusterMap.getLocalDatacenterId()).thenReturn((byte) 1);
    when(clusterMap.getDatacenterName((byte) 1)).thenReturn("test");
    when(clusterMap.getMetricRegistry()).thenReturn(new MetricRegistry());
    MockDataNodeId mockDataNodeId = new MockDataNodeId(Collections.singletonList(new Port(80, PortType.PLAINTEXT)), Collections.singletonList("/a/b"), "test");
    List<MockDataNodeId> dataNodeIds = new ArrayList<>();
    dataNodeIds.add(mockDataNodeId);
    doReturn(dataNodeIds).when(clusterMap).getDataNodeIds();
    when(networkClient.warmUpConnections(anyList(), anyByte(), anyLong(), anyList())).thenReturn(1);
    Properties properties = new Properties();
    properties.setProperty(RouterConfig.ROUTER_DATACENTER_NAME, "test");
    properties.setProperty(RouterConfig.ROUTER_HOSTNAME, "test");
    RouterConfig routerConfig = new RouterConfig(new VerifiableProperties(properties));
    NonBlockingRouterMetrics routerMetrics = new NonBlockingRouterMetrics(clusterMap, routerConfig);
    quotaAwareOperationController = new QuotaAwareOperationController(null, null, null, networkClientFactory, clusterMap, routerConfig, null, null, routerMetrics, null, null, null, null, nonBlockingRouter);
    // closing existing put manager before setting mock to clean up the threads.
    quotaAwareOperationController.putManager.close();
    FieldSetter.setField(quotaAwareOperationController, quotaAwareOperationController.getClass().getSuperclass().getDeclaredField("putManager"), putManager);
    FieldSetter.setField(quotaAwareOperationController, quotaAwareOperationController.getClass().getSuperclass().getDeclaredField("getManager"), getManager);
    FieldSetter.setField(quotaAwareOperationController, quotaAwareOperationController.getClass().getSuperclass().getDeclaredField("deleteManager"), deleteManager);
    FieldSetter.setField(quotaAwareOperationController, quotaAwareOperationController.getClass().getSuperclass().getDeclaredField("undeleteManager"), undeleteManager);
    FieldSetter.setField(quotaAwareOperationController, quotaAwareOperationController.getClass().getSuperclass().getDeclaredField("ttlUpdateManager"), ttlUpdateManager);
    doNothing().when(getManager).poll(requestsToSend, requestsToDrop);
    doNothing().when(deleteManager).poll(requestsToSend, requestsToDrop);
    doNothing().when(ttlUpdateManager).poll(requestsToSend, requestsToDrop);
    doNothing().when(nonBlockingRouter).initiateBackgroundDeletes(anyList());
}

Example 42 with ClusterMap

use of com.github.ambry.clustermap.ClusterMap in project ambry by linkedin.

the class CompactionVerifier method main.

/**
 * Main function to trigger the verifier.
 * @param args CLI arguments
 * @throws Exception if the verifier encountered problems.
 */
public static void main(String[] args) throws Exception {
    VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
    CompactionVerifierConfig verifierConfig = new CompactionVerifierConfig(verifiableProperties);
    StoreConfig storeConfig = new StoreConfig(verifiableProperties);
    assert !storeConfig.storeEnableHardDelete : "Hard delete cannot be enabled in the properties";
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    ClusterMap clusterMap = new StaticClusterAgentsFactory(clusterMapConfig, verifierConfig.hardwareLayoutFilePath, verifierConfig.partitionLayoutFilePath).getClusterMap();
    StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
    try (CompactionVerifier compactionVerifier = new CompactionVerifier(verifierConfig, storeConfig, storeKeyFactory)) {
        compactionVerifier.verifyCompaction();
    }
    LOGGER.info("Verification completed successfully");
}

Example 43 with ClusterMap

use of com.github.ambry.clustermap.ClusterMap in project ambry by linkedin.

the class DiskReformatter method main.

public static void main(String[] args) throws Exception {
    VerifiableProperties properties = ToolUtils.getVerifiableProperties(args);
    DiskReformatterConfig config = new DiskReformatterConfig(properties);
    StoreConfig storeConfig = new StoreConfig(properties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(properties);
    ServerConfig serverConfig = new ServerConfig(properties);
    ClusterAgentsFactory clusterAgentsFactory = Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath);
    try (ClusterMap clusterMap = clusterAgentsFactory.getClusterMap()) {
        StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, properties, clusterMap.getMetricRegistry());
        StoreKeyFactory storeKeyFactory = Utils.getObj(storeConfig.storeKeyFactory, clusterMap);
        DataNodeId dataNodeId = clusterMap.getDataNodeId(config.datanodeHostname, config.datanodePort);
        if (dataNodeId == null) {
            throw new IllegalArgumentException("Did not find node in clustermap with hostname:port - " + config.datanodeHostname + ":" + config.datanodePort);
        }
        DiskReformatter reformatter = new DiskReformatter(dataNodeId, Collections.EMPTY_LIST, config.fetchSizeInBytes, storeConfig, storeKeyFactory, clusterMap, SystemTime.getInstance(), storeKeyConverterFactory.getStoreKeyConverter());
        AtomicInteger exitStatus = new AtomicInteger(0);
        CountDownLatch latch = new CountDownLatch(config.diskMountPaths.length);
        for (int i = 0; i < config.diskMountPaths.length; i++) {
            int finalI = i;
            Runnable runnable = () -> {
                try {
                    reformatter.reformat(config.diskMountPaths[finalI], new File(config.scratchPaths[finalI]));
                    latch.countDown();
                } catch (Exception e) {
                    throw new IllegalStateException(e);
                }
            };
            Thread thread = Utils.newThread(config.diskMountPaths[finalI] + "-reformatter", runnable, true);
            thread.setUncaughtExceptionHandler((t, e) -> {
                exitStatus.set(1);
                logger.error("Reformatting {} failed", config.diskMountPaths[finalI], e);
                latch.countDown();
            });
            thread.start();
        }
        latch.await();
        System.exit(exitStatus.get());
    }
}

Example 44 with ClusterMap

use of com.github.ambry.clustermap.ClusterMap in project ambry by linkedin.

the class DumpIndexTool method main.

public static void main(String[] args) throws Exception {
    final AtomicInteger exitCode = new AtomicInteger(0);
    VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
    DumpIndexToolConfig config = new DumpIndexToolConfig(verifiableProperties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    try (ClusterMap clusterMap = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath)).getClusterMap()) {
        StoreConfig storeConfig = new StoreConfig(verifiableProperties);
        // this tool supports only blob IDs. It can become generic if StoreKeyFactory provides a deserFromString method.
        BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
        StoreToolsMetrics metrics = new StoreToolsMetrics(clusterMap.getMetricRegistry());
        StoreMetrics storeMetrics = new StoreMetrics("DumpIndexTool", clusterMap.getMetricRegistry());
        ServerConfig serverConfig = new ServerConfig(verifiableProperties);
        Time time = SystemTime.getInstance();
        Throttler throttler = new Throttler(config.indexEntriesToProcessPerSec, 1000, true, time);
        StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, verifiableProperties, clusterMap.getMetricRegistry());
        DumpIndexTool dumpIndexTool = new DumpIndexTool(blobIdFactory, storeConfig, time, metrics, storeMetrics, throttler, storeKeyConverterFactory.getStoreKeyConverter());
        Set<StoreKey> filterKeySet = new HashSet<>();
        for (String key : config.filterSet) {
            filterKeySet.add(new BlobId(key, clusterMap));
        }
        switch(config.typeOfOperation) {
            case DumpIndex:
                dumpIndex(dumpIndexTool, config.pathOfInput, filterKeySet);
                break;
            case DumpIndexSegment:
                dumpIndexSegment(dumpIndexTool, config.pathOfInput, filterKeySet);
                break;
            case VerifyIndex:
                IndexProcessingResults results = dumpIndexTool.processIndex(config.pathOfInput, filterKeySet, time.milliseconds(), config.detectDuplicatesAcrossKeys);
                exitCode.set(reportVerificationResults(config.pathOfInput, results, config.failIfCraftedIdsPresent));
                break;
            case VerifyDataNode:
                DataNodeId dataNodeId = clusterMap.getDataNodeId(config.hostname, config.port);
                if (dataNodeId == null) {
                    logger.error("No data node corresponding to {}:{}", config.hostname, config.port);
                } else {
                    Set<File> replicaDirs = clusterMap.getReplicaIds(dataNodeId).stream().map(replicaId -> new File(replicaId.getMountPath())).collect(Collectors.toSet());
                    Map<File, IndexProcessingResults> resultsByReplica = dumpIndexTool.processIndex(replicaDirs, filterKeySet, config.parallelism, config.detectDuplicatesAcrossKeys);
                    replicaDirs.removeAll(resultsByReplica.keySet());
                    if (replicaDirs.size() != 0) {
                        logger.error("Results obtained missing {}", replicaDirs);
                        exitCode.set(5);
                    } else {
                        resultsByReplica.forEach((replicaDir, result) -> exitCode.set(Math.max(exitCode.get(), reportVerificationResults(replicaDir, result, config.failIfCraftedIdsPresent))));
                    }
                }
                break;
            default:
                throw new IllegalArgumentException("Unrecognized operation: " + config.typeOfOperation);
        }
    }
    System.exit(exitCode.get());
}

Example 45 with ClusterMap

use of com.github.ambry.clustermap.ClusterMap in project ambry by linkedin.

the class SafeServerShutdownTool method main.

public static void main(String[] args) throws Exception {
    VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
    SafeServerShutdownToolConfig config = new SafeServerShutdownToolConfig(verifiableProperties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    try (ClusterMap clusterMap = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath)).getClusterMap()) {
        SSLFactory sslFactory = !clusterMapConfig.clusterMapSslEnabledDatacenters.isEmpty() ? SSLFactory.getNewInstance(new SSLConfig(verifiableProperties)) : null;
        try (ServerAdminTool serverAdminTool = new ServerAdminTool(clusterMap, sslFactory, verifiableProperties)) {
            DataNodeId dataNodeId = clusterMap.getDataNodeId(config.hostname, config.port);
            if (dataNodeId == null) {
                throw new IllegalArgumentException("Could not find a data node corresponding to " + config.hostname + ":" + config.port);
            }
            SafeServerShutdownTool safeServerShutdownTool = new SafeServerShutdownTool(serverAdminTool, SystemTime.getInstance());
            int exitStatus = safeServerShutdownTool.prepareServerForShutdown(dataNodeId, config.logGrowthPauseLagThresholdBytes, config.numReplicasCaughtUpPerPartition, config.timeoutSecs, config.checkRepeatDelaySecs) ? 0 : 1;
            System.exit(exitStatus);
        }
    }
}