Examples with Upgrader co.cask.cdap.etl.tool.config.Upgrader used on opensource projects

Example 1 with Upgrader

use of co.cask.cdap.etl.tool.config.Upgrader in project cdap by caskdata.

the class UpgradeTool method main.

public static void main(String[] args) throws Exception {
    Options options = new Options().addOption(new Option("h", "help", false, "Print this usage message.")).addOption(new Option("u", "uri", true, "CDAP instance URI to interact with in the format " + "[http[s]://]<hostname>:<port>. Defaults to localhost:11015.")).addOption(new Option("a", "accesstoken", true, "File containing the access token to use when interacting " + "with a secure CDAP instance.")).addOption(new Option("t", "timeout", true, "Timeout in milliseconds to use when interacting with the " + "CDAP RESTful APIs. Defaults to " + DEFAULT_READ_TIMEOUT_MILLIS + ".")).addOption(new Option("n", "namespace", true, "Namespace to perform the upgrade in. If none is given, " + "pipelines in all namespaces will be upgraded.")).addOption(new Option("p", "pipeline", true, "Name of the pipeline to upgrade. If specified, a namespace " + "must also be given.")).addOption(new Option("f", "configfile", true, "File containing old application details to update. " + "The file contents are expected to be in the same format as the request body for creating an " + "ETL application from one of the etl artifacts. " + "It is expected to be a JSON Object containing 'artifact' and 'config' fields." + "The value for 'artifact' must be a JSON Object that specifies the artifact scope, name, and version. " + "The value for 'config' must be a JSON Object specifies the source, transforms, and sinks of the pipeline, " + "as expected by older versions of the etl artifacts.")).addOption(new Option("o", "outputfile", true, "File to write the converted application details provided in " + "the configfile option. If none is given, results will be written to the input file + '.converted'. " + "The contents of this file can be sent directly to CDAP to update or create an application.")).addOption(new Option("e", "errorDir", true, "Optional directory to write any upgraded pipeline configs that " + "failed to upgrade. The problematic configs can then be manually edited and upgraded separately. " + "Upgrade errors may happen for pipelines that use plugins that are not backwards compatible. " + "This directory must be writable by the user that is running this tool."));
    CommandLineParser parser = new BasicParser();
    CommandLine commandLine = parser.parse(options, args);
    String[] commandArgs = commandLine.getArgs();
    // if help is an option, or if there isn't a single 'upgrade' command, print usage and exit.
    if (commandLine.hasOption("h") || commandArgs.length != 1 || !"upgrade".equalsIgnoreCase(commandArgs[0])) {
        HelpFormatter helpFormatter = new HelpFormatter();
        helpFormatter.printHelp(UpgradeTool.class.getName() + " upgrade", "Upgrades old pipelines to the current version. If the plugins used are not backward-compatible, " + "the attempted upgrade config will be written to the error directory for a manual upgrade.", options, "");
        System.exit(0);
    }
    ClientConfig clientConfig = getClientConfig(commandLine);
    if (commandLine.hasOption("f")) {
        String inputFilePath = commandLine.getOptionValue("f");
        String outputFilePath = commandLine.hasOption("o") ? commandLine.getOptionValue("o") : inputFilePath + ".new";
        convertFile(inputFilePath, outputFilePath, new Upgrader(new ArtifactClient(clientConfig)));
        System.exit(0);
    }
    File errorDir = commandLine.hasOption("e") ? new File(commandLine.getOptionValue("e")) : null;
    if (errorDir != null) {
        if (!errorDir.exists()) {
            if (!errorDir.mkdirs()) {
                LOG.error("Unable to create error directory {}.", errorDir.getAbsolutePath());
                System.exit(1);
            }
        } else if (!errorDir.isDirectory()) {
            LOG.error("{} is not a directory.", errorDir.getAbsolutePath());
            System.exit(1);
        } else if (!errorDir.canWrite()) {
            LOG.error("Unable to write to error directory {}.", errorDir.getAbsolutePath());
            System.exit(1);
        }
    }
    UpgradeTool upgradeTool = new UpgradeTool(clientConfig, errorDir);
    String namespace = commandLine.getOptionValue("n");
    String pipelineName = commandLine.getOptionValue("p");
    if (pipelineName != null) {
        if (namespace == null) {
            throw new IllegalArgumentException("Must specify a namespace when specifying a pipeline.");
        }
        ApplicationId appId = new ApplicationId(namespace, pipelineName);
        if (upgradeTool.upgrade(appId)) {
            LOG.info("Successfully upgraded {}.", appId);
        } else {
            LOG.info("{} did not need to be upgraded.", appId);
        }
        System.exit(0);
    }
    if (namespace != null) {
        printUpgraded(upgradeTool.upgrade(new NamespaceId(namespace)));
        System.exit(0);
    }
    printUpgraded(upgradeTool.upgrade());
}

Example 2 with Upgrader

use of co.cask.cdap.etl.tool.config.Upgrader in project cdap by caskdata.

the class UpgradeTool method convertFile.

private static void convertFile(String configFilePath, final String outputFilePath, Upgrader upgrader) throws Exception {
    File configFile = new File(configFilePath);
    if (!configFile.exists()) {
        throw new IllegalArgumentException(configFilePath + " does not exist.");
    }
    if (!configFile.isFile()) {
        throw new IllegalArgumentException(configFilePath + " is not a file.");
    }
    String fileContents = new String(Files.readAllBytes(configFile.toPath()), StandardCharsets.UTF_8);
    ETLAppRequest artifactFile = GSON.fromJson(fileContents, ETLAppRequest.class);
    if (!upgrader.shouldUpgrade(artifactFile.artifact)) {
        LOG.error("{} is not an artifact for an upgradeable Hydrator pipeline.", artifactFile.artifact);
        return;
    }
    upgrader.upgrade(artifactFile.artifact, artifactFile.config.toString(), new Upgrader.UpgradeAction() {

        @Override
        public boolean upgrade(AppRequest<? extends ETLConfig> appRequest) throws IOException {
            File outputFile = new File(outputFilePath);
            try (BufferedWriter writer = Files.newBufferedWriter(outputFile.toPath(), StandardCharsets.UTF_8)) {
                writer.write(GSON.toJson(appRequest));
            }
            return true;
        }
    });
    LOG.info("Successfully converted application details from file " + configFilePath + ". " + "Results have been written to " + outputFilePath);
}