Search in sources :

Example 1 with Default

use of com.github.ambry.config.Default in project ambry by linkedin.

the class DumpIndexTool method main.

public static void main(String[] args) throws Exception {
    final AtomicInteger exitCode = new AtomicInteger(0);
    VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
    DumpIndexToolConfig config = new DumpIndexToolConfig(verifiableProperties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    try (ClusterMap clusterMap = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath)).getClusterMap()) {
        StoreConfig storeConfig = new StoreConfig(verifiableProperties);
        // this tool supports only blob IDs. It can become generic if StoreKeyFactory provides a deserFromString method.
        BlobIdFactory blobIdFactory = new BlobIdFactory(clusterMap);
        StoreToolsMetrics metrics = new StoreToolsMetrics(clusterMap.getMetricRegistry());
        StoreMetrics storeMetrics = new StoreMetrics("DumpIndexTool", clusterMap.getMetricRegistry());
        ServerConfig serverConfig = new ServerConfig(verifiableProperties);
        Time time = SystemTime.getInstance();
        Throttler throttler = new Throttler(config.indexEntriesToProcessPerSec, 1000, true, time);
        StoreKeyConverterFactory storeKeyConverterFactory = Utils.getObj(serverConfig.serverStoreKeyConverterFactory, verifiableProperties, clusterMap.getMetricRegistry());
        DumpIndexTool dumpIndexTool = new DumpIndexTool(blobIdFactory, storeConfig, time, metrics, storeMetrics, throttler, storeKeyConverterFactory.getStoreKeyConverter());
        Set<StoreKey> filterKeySet = new HashSet<>();
        for (String key : config.filterSet) {
            filterKeySet.add(new BlobId(key, clusterMap));
        }
        switch(config.typeOfOperation) {
            case DumpIndex:
                dumpIndex(dumpIndexTool, config.pathOfInput, filterKeySet);
                break;
            case DumpIndexSegment:
                dumpIndexSegment(dumpIndexTool, config.pathOfInput, filterKeySet);
                break;
            case VerifyIndex:
                IndexProcessingResults results = dumpIndexTool.processIndex(config.pathOfInput, filterKeySet, time.milliseconds(), config.detectDuplicatesAcrossKeys);
                exitCode.set(reportVerificationResults(config.pathOfInput, results, config.failIfCraftedIdsPresent));
                break;
            case VerifyDataNode:
                DataNodeId dataNodeId = clusterMap.getDataNodeId(config.hostname, config.port);
                if (dataNodeId == null) {
                    logger.error("No data node corresponding to {}:{}", config.hostname, config.port);
                } else {
                    Set<File> replicaDirs = clusterMap.getReplicaIds(dataNodeId).stream().map(replicaId -> new File(replicaId.getMountPath())).collect(Collectors.toSet());
                    Map<File, IndexProcessingResults> resultsByReplica = dumpIndexTool.processIndex(replicaDirs, filterKeySet, config.parallelism, config.detectDuplicatesAcrossKeys);
                    replicaDirs.removeAll(resultsByReplica.keySet());
                    if (replicaDirs.size() != 0) {
                        logger.error("Results obtained missing {}", replicaDirs);
                        exitCode.set(5);
                    } else {
                        resultsByReplica.forEach((replicaDir, result) -> exitCode.set(Math.max(exitCode.get(), reportVerificationResults(replicaDir, result, config.failIfCraftedIdsPresent))));
                    }
                }
                break;
            default:
                throw new IllegalArgumentException("Unrecognized operation: " + config.typeOfOperation);
        }
    }
    System.exit(exitCode.get());
}
Also used : Arrays(java.util.Arrays) Default(com.github.ambry.config.Default) DataNodeId(com.github.ambry.clustermap.DataNodeId) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) HashSet(java.util.HashSet) Future(java.util.concurrent.Future) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) SystemTime(com.github.ambry.utils.SystemTime) Time(com.github.ambry.utils.Time) EnumSet(java.util.EnumSet) ExecutorService(java.util.concurrent.ExecutorService) StoreConfig(com.github.ambry.config.StoreConfig) Logger(org.slf4j.Logger) ClusterAgentsFactory(com.github.ambry.clustermap.ClusterAgentsFactory) VerifiableProperties(com.github.ambry.config.VerifiableProperties) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Config(com.github.ambry.config.Config) Set(java.util.Set) ClusterMap(com.github.ambry.clustermap.ClusterMap) ServerConfig(com.github.ambry.config.ServerConfig) Utils(com.github.ambry.utils.Utils) IOException(java.io.IOException) BlobIdFactory(com.github.ambry.commons.BlobIdFactory) Collectors(java.util.stream.Collectors) File(java.io.File) Executors(java.util.concurrent.Executors) AtomicLong(java.util.concurrent.atomic.AtomicLong) ToolUtils(com.github.ambry.tools.util.ToolUtils) List(java.util.List) Throttler(com.github.ambry.utils.Throttler) TreeMap(java.util.TreeMap) ClusterMapConfig(com.github.ambry.config.ClusterMapConfig) Timer(com.codahale.metrics.Timer) Collections(java.util.Collections) BlobId(com.github.ambry.commons.BlobId) SortedMap(java.util.SortedMap) ClusterMap(com.github.ambry.clustermap.ClusterMap) SystemTime(com.github.ambry.utils.SystemTime) Time(com.github.ambry.utils.Time) ServerConfig(com.github.ambry.config.ServerConfig) ClusterAgentsFactory(com.github.ambry.clustermap.ClusterAgentsFactory) Throttler(com.github.ambry.utils.Throttler) HashSet(java.util.HashSet) VerifiableProperties(com.github.ambry.config.VerifiableProperties) ClusterMapConfig(com.github.ambry.config.ClusterMapConfig) BlobIdFactory(com.github.ambry.commons.BlobIdFactory) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) StoreConfig(com.github.ambry.config.StoreConfig) BlobId(com.github.ambry.commons.BlobId) DataNodeId(com.github.ambry.clustermap.DataNodeId) File(java.io.File)

Aggregations

Timer (com.codahale.metrics.Timer)1 ClusterAgentsFactory (com.github.ambry.clustermap.ClusterAgentsFactory)1 ClusterMap (com.github.ambry.clustermap.ClusterMap)1 DataNodeId (com.github.ambry.clustermap.DataNodeId)1 BlobId (com.github.ambry.commons.BlobId)1 BlobIdFactory (com.github.ambry.commons.BlobIdFactory)1 ClusterMapConfig (com.github.ambry.config.ClusterMapConfig)1 Config (com.github.ambry.config.Config)1 Default (com.github.ambry.config.Default)1 ServerConfig (com.github.ambry.config.ServerConfig)1 StoreConfig (com.github.ambry.config.StoreConfig)1 VerifiableProperties (com.github.ambry.config.VerifiableProperties)1 ToolUtils (com.github.ambry.tools.util.ToolUtils)1 SystemTime (com.github.ambry.utils.SystemTime)1 Throttler (com.github.ambry.utils.Throttler)1 Time (com.github.ambry.utils.Time)1 Utils (com.github.ambry.utils.Utils)1 File (java.io.File)1 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1