Examples with BlobIdFactory com.github.ambry.commons.BlobIdFactory used on opensource projects

Example 31 with BlobIdFactory

use of com.github.ambry.commons.BlobIdFactory in project ambry by linkedin.

the class IndexReadPerformance method main.

public static void main(String[] args) {
    try {
        OptionParser parser = new OptionParser();
        ArgumentAcceptingOptionSpec<String> logToReadOpt = parser.accepts("logToRead", "The log that needs to be replayed for traffic").withRequiredArg().describedAs("log_to_read").ofType(String.class);
        ArgumentAcceptingOptionSpec<String> hardwareLayoutOpt = parser.accepts("hardwareLayout", "The path of the hardware layout file").withRequiredArg().describedAs("hardware_layout").ofType(String.class);
        ArgumentAcceptingOptionSpec<String> partitionLayoutOpt = parser.accepts("partitionLayout", "The path of the partition layout file").withRequiredArg().describedAs("partition_layout").ofType(String.class);
        ArgumentAcceptingOptionSpec<Integer> numberOfReadersOpt = parser.accepts("numberOfReaders", "The number of readers that read to a random index concurrently").withRequiredArg().describedAs("The number of readers").ofType(Integer.class).defaultsTo(4);
        ArgumentAcceptingOptionSpec<Integer> readsPerSecondOpt = parser.accepts("readsPerSecond", "The rate at which reads need to be performed").withRequiredArg().describedAs("The number of reads per second").ofType(Integer.class).defaultsTo(1000);
        ArgumentAcceptingOptionSpec<Boolean> verboseLoggingOpt = parser.accepts("enableVerboseLogging", "Enables verbose logging").withOptionalArg().describedAs("Enable verbose logging").ofType(Boolean.class).defaultsTo(false);
        OptionSet options = parser.parse(args);
        ArrayList<OptionSpec> listOpt = new ArrayList<>();
        listOpt.add(logToReadOpt);
        listOpt.add(hardwareLayoutOpt);
        listOpt.add(partitionLayoutOpt);
        ToolUtils.ensureOrExit(listOpt, options, parser);
        String logToRead = options.valueOf(logToReadOpt);
        int numberOfReaders = options.valueOf(numberOfReadersOpt);
        int readsPerSecond = options.valueOf(readsPerSecondOpt);
        boolean enableVerboseLogging = options.has(verboseLoggingOpt);
        if (enableVerboseLogging) {
            System.out.println("Enabled verbose logging");
        }
        String hardwareLayoutPath = options.valueOf(hardwareLayoutOpt);
        String partitionLayoutPath = options.valueOf(partitionLayoutOpt);
        ClusterMapConfig clusterMapConfig = new ClusterMapConfig(new VerifiableProperties(new Properties()));
        ClusterMap map = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, hardwareLayoutPath, partitionLayoutPath)).getClusterMap();
        StoreKeyFactory factory = new BlobIdFactory(map);
        // Read the log and get the index directories and create the indexes
        final BufferedReader br = new BufferedReader(new FileReader(logToRead));
        final HashMap<String, IndexPayload> hashes = new HashMap<String, IndexPayload>();
        String line;
        MetricRegistry metricRegistry = new MetricRegistry();
        StoreMetrics metrics = new StoreMetrics(metricRegistry);
        ScheduledExecutorService s = Utils.newScheduler(numberOfReaders, "index", true);
        DiskSpaceAllocator diskSpaceAllocator = new DiskSpaceAllocator(false, null, 0, new StorageManagerMetrics(metricRegistry));
        Properties props = new Properties();
        props.setProperty("store.index.memory.size.bytes", "1048576");
        props.setProperty("store.segment.size.in.bytes", "1000");
        StoreConfig config = new StoreConfig(new VerifiableProperties(props));
        Log log = new Log(System.getProperty("user.dir"), 1000, diskSpaceAllocator, config, metrics, null);
        final AtomicLong totalTimeTaken = new AtomicLong(0);
        final AtomicLong totalReads = new AtomicLong(0);
        final CountDownLatch latch = new CountDownLatch(numberOfReaders);
        final AtomicBoolean shutdown = new AtomicBoolean(false);
        // attach shutdown handler to catch control-c
        Runtime.getRuntime().addShutdownHook(new Thread() {

            public void run() {
                try {
                    System.out.println("Shutdown invoked");
                    shutdown.set(true);
                    latch.await();
                    System.out.println("Total reads : " + totalReads.get() + "  Total time taken : " + totalTimeTaken.get() + " Nano Seconds  Average time taken per read " + ((double) totalReads.get() / totalTimeTaken.get()) / SystemTime.NsPerSec + " Seconds");
                } catch (Exception e) {
                    System.out.println("Error while shutting down " + e);
                }
            }
        });
        ScheduledExecutorService scheduleReadLog = Utils.newScheduler(1, true);
        while ((line = br.readLine()) != null) {
            if (line.startsWith("logdir")) {
                String[] logdirs = line.split("-");
                BlobIndexMetrics metricIndex = new BlobIndexMetrics(logdirs[1], s, log, enableVerboseLogging, totalReads, totalTimeTaken, totalReads, config, null, factory);
                hashes.put(logdirs[1], new IndexPayload(metricIndex, new HashSet<String>()));
            } else {
                break;
            }
        }
        // read next batch of ids after 2 minutes
        scheduleReadLog.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                populateIds(br, hashes);
            }
        }, 0, 120, TimeUnit.SECONDS);
        Throttler throttler = new Throttler(readsPerSecond, 100, true, SystemTime.getInstance());
        Thread[] threadIndexPerf = new Thread[numberOfReaders];
        for (int i = 0; i < numberOfReaders; i++) {
            threadIndexPerf[i] = new Thread(new IndexReadPerfRun(hashes, throttler, shutdown, latch, map));
            threadIndexPerf[i].start();
        }
        for (int i = 0; i < numberOfReaders; i++) {
            threadIndexPerf[i].join();
        }
        br.close();
    } catch (Exception e) {
        System.out.println("Exiting process with exception " + e);
    }
}

Example 32 with BlobIdFactory

use of com.github.ambry.commons.BlobIdFactory in project ambry by linkedin.

the class BlobValidator method main.

/**
 * Runs the BlobValidator
 * @param args associated arguments.
 * @throws Exception
 */
public static void main(String[] args) throws Exception {
    VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
    BlobValidatorConfig config = new BlobValidatorConfig(verifiableProperties);
    ClusterMapConfig clusterMapConfig = new ClusterMapConfig(verifiableProperties);
    ClusterMap clusterMap = ((ClusterAgentsFactory) Utils.getObj(clusterMapConfig.clusterMapClusterAgentsFactory, clusterMapConfig, config.hardwareLayoutFilePath, config.partitionLayoutFilePath)).getClusterMap();
    List<BlobId> blobIds = getBlobIds(config, clusterMap);
    SSLFactory sslFactory = !clusterMapConfig.clusterMapSslEnabledDatacenters.isEmpty() ? SSLFactory.getNewInstance(new SSLConfig(verifiableProperties)) : null;
    StoreKeyFactory storeKeyFactory = new BlobIdFactory(clusterMap);
    BlobValidator validator = new BlobValidator(clusterMap, config.replicasToContactPerSec, sslFactory, verifiableProperties);
    LOGGER.info("Validation starting");
    switch(config.typeOfOperation) {
        case ValidateBlobOnAllReplicas:
            Map<BlobId, List<String>> mismatchDetailsMap = validator.validateBlobsOnAllReplicas(blobIds, config.getOption, clusterMap, storeKeyFactory);
            logMismatches(mismatchDetailsMap);
            break;
        case ValidateBlobOnDatacenter:
            if (config.datacenter.isEmpty() || !clusterMap.hasDatacenter(config.datacenter)) {
                throw new IllegalArgumentException("Please provide a valid datacenter");
            }
            mismatchDetailsMap = validator.validateBlobsOnDatacenter(config.datacenter, blobIds, config.getOption, clusterMap, storeKeyFactory);
            logMismatches(mismatchDetailsMap);
            break;
        case ValidateBlobOnReplica:
            DataNodeId dataNodeId = clusterMap.getDataNodeId(config.hostname, config.port);
            if (dataNodeId == null) {
                throw new IllegalArgumentException("Could not find a data node corresponding to " + config.hostname + ":" + config.port);
            }
            List<ServerErrorCode> validErrorCodes = Arrays.asList(ServerErrorCode.No_Error, ServerErrorCode.Blob_Deleted, ServerErrorCode.Blob_Expired);
            Map<BlobId, ServerErrorCode> blobIdToErrorCode = validator.validateBlobsOnReplica(dataNodeId, blobIds, config.getOption, clusterMap, storeKeyFactory);
            for (Map.Entry<BlobId, ServerErrorCode> entry : blobIdToErrorCode.entrySet()) {
                ServerErrorCode errorCode = entry.getValue();
                if (!validErrorCodes.contains(errorCode)) {
                    LOGGER.error("[{}] received error code: {}", entry.getKey(), errorCode);
                }
            }
            break;
        default:
            throw new IllegalStateException("Recognized but unsupported operation: " + config.typeOfOperation);
    }
    LOGGER.info("Validation complete");
    validator.close();
    clusterMap.close();
}