Examples with CloudStorageCompactor com.github.ambry.cloud.CloudStorageCompactor used on opensource projects

Example 1 with CloudStorageCompactor

use of com.github.ambry.cloud.CloudStorageCompactor in project ambry by linkedin.

the class AzureCompactionTool method main.

public static void main(String[] args) throws Exception {
    OptionParser parser = new OptionParser();
    ArgumentAcceptingOptionSpec<String> propsFileOpt = parser.accepts(PROPS_FILE, "Properties file path").withRequiredArg().describedAs(PROPS_FILE).ofType(String.class);
    String commandName = AzureCompactionTool.class.getSimpleName();
    parser.accepts(PURGE_OPTION, "Flag to purge dead blobs from the partition");
    parser.nonOptions("The partitions to compact").ofType(String.class);
    OptionSet optionSet = parser.parse(args);
    String propsFilePath = optionSet.valueOf(propsFileOpt);
    if (propsFilePath == null) {
        printHelpAndExit(parser);
    }
    Properties properties = Utils.loadProps(propsFilePath);
    ToolUtils.addClusterMapProperties(properties);
    VerifiableProperties verifiableProperties = new VerifiableProperties(properties);
    // User needs to specify this option to actually delete blobs
    boolean testMode = !optionSet.has(PURGE_OPTION);
    List<String> partitions = (List<String>) optionSet.nonOptionArguments();
    if (!testMode && partitions.isEmpty()) {
        printHelpAndExit(parser);
    }
    Set<PartitionId> partitionIdSet = partitions.stream().map(path -> new PartitionPathId(path)).collect(Collectors.toSet());
    AzureCloudDestination azureDest = null;
    try {
        azureDest = (AzureCloudDestination) new AzureCloudDestinationFactory(verifiableProperties, new MetricRegistry(), null).getCloudDestination();
        CloudConfig cloudConfig = new CloudConfig(verifiableProperties);
        CloudStorageCompactor compactor = new CloudStorageCompactor(azureDest, cloudConfig, partitionIdSet, new VcrMetrics(new MetricRegistry()));
        // Attempt clean shutdown if someone Ctrl-C's us.
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            logger.info("Received shutdown signal. Shutting down compactor.");
            compactor.shutdown();
        }));
        if (testMode) {
            DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneId.systemDefault());
            List<Pair<String, Long>> progressList = azureDest.getAzureStorageCompactor().getAllCompactionProgress();
            progressList.forEach(pair -> {
                String progress = dateTimeFormatter.format(Instant.ofEpochMilli(pair.getSecond()));
                // TODO: write to user specified output file
                System.out.println(pair.getFirst() + "\t" + progress);
            });
        } else {
            compactor.compactPartitions();
        }
        System.exit(0);
    } catch (Exception ex) {
        logger.error("Command {} failed", commandName, ex);
        System.exit(1);
    } finally {
        if (azureDest != null) {
            azureDest.close();
        }
    }
}