Examples with DTConfiguration com.datatorrent.stram.client.DTConfiguration used on opensource projects

Example 1 with DTConfiguration

use of com.datatorrent.stram.client.DTConfiguration in project apex-core by apache.

the class ApexCliTest method testLaunchAppPackagePropertyPrecedence.

@Test
public void testLaunchAppPackagePropertyPrecedence() throws Exception {
    // set launch command options
    ApexCli.LaunchCommandLineInfo commandLineInfo = ApexCli.getLaunchCommandLineInfo(new String[] { "-D", "dt.test.1=launch-define", "-apconf", "my-app-conf1.xml", "-conf", "src/test/resources/testAppPackage/local-conf.xml" });
    // process and look at launch config
    DTConfiguration props = cli.getLaunchAppPackageProperties(ap, null, commandLineInfo, null);
    Assert.assertEquals("launch-define", props.get("dt.test.1"));
    Assert.assertEquals("local-fs-config", props.get("dt.test.2"));
    Assert.assertEquals("app-package-config", props.get("dt.test.3"));
    Assert.assertEquals("user-home-config", props.get("dt.test.4"));
    Assert.assertEquals("package-default", props.get("dt.test.5"));
    props = cli.getLaunchAppPackageProperties(ap, null, commandLineInfo, "MyFirstApplication");
    Assert.assertEquals("launch-define", props.get("dt.test.1"));
    Assert.assertEquals("local-fs-config", props.get("dt.test.2"));
    Assert.assertEquals("app-package-config", props.get("dt.test.3"));
    Assert.assertEquals("user-home-config", props.get("dt.test.4"));
    Assert.assertEquals("app-default", props.get("dt.test.5"));
    Assert.assertEquals("package-default", props.get("dt.test.6"));
}

Example 2 with DTConfiguration

use of com.datatorrent.stram.client.DTConfiguration in project apex-core by apache.

the class ApexCli method getLaunchAppPackageProperties.

DTConfiguration getLaunchAppPackageProperties(AppPackage ap, ConfigPackage cp, LaunchCommandLineInfo commandLineInfo, String appName) throws Exception {
    DTConfiguration launchProperties = new DTConfiguration();
    List<AppInfo> applications = getAppsFromPackageAndConfig(ap, cp, commandLineInfo.useConfigApps);
    AppInfo selectedApp = null;
    for (AppInfo app : applications) {
        if (app.name.equals(appName)) {
            selectedApp = app;
            break;
        }
    }
    Map<String, PropertyInfo> defaultProperties = selectedApp == null ? ap.getDefaultProperties() : selectedApp.defaultProperties;
    Set<String> requiredProperties = new TreeSet<>(selectedApp == null ? ap.getRequiredProperties() : selectedApp.requiredProperties);
    for (Map.Entry<String, PropertyInfo> entry : defaultProperties.entrySet()) {
        launchProperties.set(entry.getKey(), entry.getValue().getValue(), Scope.TRANSIENT, entry.getValue().getDescription());
        requiredProperties.remove(entry.getKey());
    }
    // settings specified in the user's environment take precedence over defaults in package.
    // since both are merged into a single -conf option below, apply them on top of the defaults here.
    File confFile = new File(StramClientUtils.getUserDTDirectory(), StramClientUtils.DT_SITE_XML_FILE);
    if (confFile.exists()) {
        Configuration userConf = new Configuration(false);
        userConf.addResource(new Path(confFile.toURI()));
        Iterator<Entry<String, String>> it = userConf.iterator();
        while (it.hasNext()) {
            Entry<String, String> entry = it.next();
            // filter relevant entries
            String key = entry.getKey();
            if (key.startsWith(StreamingApplication.DT_PREFIX) || key.startsWith(StreamingApplication.APEX_PREFIX)) {
                launchProperties.set(key, entry.getValue(), Scope.TRANSIENT, null);
                requiredProperties.remove(key);
            }
        }
    }
    if (commandLineInfo.apConfigFile != null) {
        DTConfiguration givenConfig = new DTConfiguration();
        givenConfig.loadFile(new File(ap.tempDirectory() + "/conf/" + commandLineInfo.apConfigFile));
        for (Map.Entry<String, String> entry : givenConfig) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }
    if (cp != null) {
        Map<String, String> properties = cp.getProperties(appName);
        for (Map.Entry<String, String> entry : properties.entrySet()) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    } else if (commandLineInfo.configFile != null) {
        DTConfiguration givenConfig = new DTConfiguration();
        givenConfig.loadFile(new File(commandLineInfo.configFile));
        for (Map.Entry<String, String> entry : givenConfig) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }
    if (commandLineInfo.overrideProperties != null) {
        for (Map.Entry<String, String> entry : commandLineInfo.overrideProperties.entrySet()) {
            launchProperties.set(entry.getKey(), entry.getValue(), Scope.TRANSIENT, null);
            requiredProperties.remove(entry.getKey());
        }
    }
    // now look at whether it is in default configuration
    for (Map.Entry<String, String> entry : conf) {
        if (StringUtils.isNotBlank(entry.getValue())) {
            requiredProperties.remove(entry.getKey());
        }
    }
    if (!requiredProperties.isEmpty()) {
        throw new CliException("Required properties not set: " + StringUtils.join(requiredProperties, ", "));
    }
    // StramClientUtils.evalProperties(launchProperties);
    return launchProperties;
}

Example 3 with DTConfiguration

use of com.datatorrent.stram.client.DTConfiguration in project apex-core by apache.

the class ApexCli method getLaunchAppPackageArgs.

String[] getLaunchAppPackageArgs(AppPackage ap, ConfigPackage cp, LaunchCommandLineInfo commandLineInfo, ConsoleReader reader) throws Exception {
    String matchAppName = null;
    if (commandLineInfo.args.length > 1) {
        matchAppName = commandLineInfo.args[1];
    }
    List<AppInfo> applications = new ArrayList<>(getAppsFromPackageAndConfig(ap, cp, commandLineInfo.useConfigApps));
    if (matchAppName != null) {
        Iterator<AppInfo> it = applications.iterator();
        while (it.hasNext()) {
            AppInfo ai = it.next();
            if ((commandLineInfo.exactMatch && !ai.name.equals(matchAppName)) || !ai.name.toLowerCase().matches(".*" + matchAppName.toLowerCase() + ".*")) {
                it.remove();
            }
        }
    }
    AppInfo selectedApp = null;
    if (applications.isEmpty()) {
        throw new CliException("No applications in Application Package" + (matchAppName != null ? " matching \"" + matchAppName + "\"" : ""));
    } else if (applications.size() == 1) {
        selectedApp = applications.get(0);
    } else {
        // Store the appNames sorted in alphabetical order and their position in matchingAppFactories list
        TreeMap<String, Integer> appNamesInAlphabeticalOrder = new TreeMap<>();
        // Display matching applications
        for (int i = 0; i < applications.size(); i++) {
            String appName = applications.get(i).name;
            appNamesInAlphabeticalOrder.put(appName, i);
        }
        // Create a mapping between the app display number and original index at matchingAppFactories
        int index = 1;
        HashMap<Integer, Integer> displayIndexToOriginalUnsortedIndexMap = new HashMap<>();
        for (Map.Entry<String, Integer> entry : appNamesInAlphabeticalOrder.entrySet()) {
            // Map display number of the app to original unsorted index
            displayIndexToOriginalUnsortedIndexMap.put(index, entry.getValue());
            // Display the app names
            System.out.printf("%3d. %s\n", index++, entry.getKey());
        }
        // Exit if not in interactive mode
        if (!consolePresent) {
            throw new CliException("More than one application in Application Package match '" + matchAppName + "'");
        } else {
            boolean useHistory = reader.isHistoryEnabled();
            reader.setHistoryEnabled(false);
            History previousHistory = reader.getHistory();
            History dummyHistory = new MemoryHistory();
            reader.setHistory(dummyHistory);
            List<Completer> completers = new ArrayList<>(reader.getCompleters());
            for (Completer c : completers) {
                reader.removeCompleter(c);
            }
            reader.setHandleUserInterrupt(true);
            String optionLine;
            try {
                optionLine = reader.readLine("Choose application: ");
            } finally {
                reader.setHandleUserInterrupt(false);
                reader.setHistoryEnabled(useHistory);
                reader.setHistory(previousHistory);
                for (Completer c : completers) {
                    reader.addCompleter(c);
                }
            }
            try {
                int option = Integer.parseInt(optionLine);
                if (0 < option && option <= applications.size()) {
                    int appIndex = displayIndexToOriginalUnsortedIndexMap.get(option);
                    selectedApp = applications.get(appIndex);
                }
            } catch (Exception ex) {
            // ignore
            }
        }
    }
    if (selectedApp == null) {
        throw new CliException("No application selected");
    }
    DTConfiguration launchProperties = getLaunchAppPackageProperties(ap, cp, commandLineInfo, selectedApp.name);
    String appFile = ap.tempDirectory() + "/app/" + selectedApp.file;
    List<String> launchArgs = new ArrayList<>();
    launchArgs.add("launch");
    launchArgs.add("-exactMatch");
    List<String> absClassPath = new ArrayList<>(ap.getClassPath());
    for (int i = 0; i < absClassPath.size(); i++) {
        String path = absClassPath.get(i);
        if (!path.startsWith("/")) {
            absClassPath.set(i, ap.tempDirectory() + "/" + path);
        }
    }
    if (cp != null) {
        StringBuilder files = new StringBuilder();
        for (String file : cp.getClassPath()) {
            if (files.length() != 0) {
                files.append(',');
            }
            files.append(cp.tempDirectory()).append(File.separatorChar).append(file);
        }
        if (!StringUtils.isBlank(files.toString())) {
            if (commandLineInfo.libjars != null) {
                commandLineInfo.libjars = files.toString() + "," + commandLineInfo.libjars;
            } else {
                commandLineInfo.libjars = files.toString();
            }
        }
        files.setLength(0);
        for (String file : cp.getFiles()) {
            if (files.length() != 0) {
                files.append(',');
            }
            files.append(cp.tempDirectory()).append(File.separatorChar).append(file);
        }
        if (!StringUtils.isBlank(files.toString())) {
            if (commandLineInfo.files != null) {
                commandLineInfo.files = files.toString() + "," + commandLineInfo.files;
            } else {
                commandLineInfo.files = files.toString();
            }
        }
    }
    StringBuilder libjarsVal = new StringBuilder();
    if (!absClassPath.isEmpty() || commandLineInfo.libjars != null) {
        if (!absClassPath.isEmpty()) {
            libjarsVal.append(org.apache.commons.lang3.StringUtils.join(absClassPath, ','));
        }
        if (commandLineInfo.libjars != null) {
            if (libjarsVal.length() > 0) {
                libjarsVal.append(",");
            }
            libjarsVal.append(commandLineInfo.libjars);
        }
    }
    if (appFile.endsWith(".json") || appFile.endsWith(".properties")) {
        if (libjarsVal.length() > 0) {
            libjarsVal.append(",");
        }
        libjarsVal.append(ap.tempDirectory()).append("/app/*.jar");
    }
    if (libjarsVal.length() > 0) {
        launchArgs.add("-libjars");
        launchArgs.add(libjarsVal.toString());
    }
    File launchPropertiesFile = new File(ap.tempDirectory(), "launch.xml");
    launchProperties.writeToFile(launchPropertiesFile, "");
    launchArgs.add("-conf");
    launchArgs.add(launchPropertiesFile.getCanonicalPath());
    if (commandLineInfo.localMode) {
        launchArgs.add("-local");
    }
    if (commandLineInfo.archives != null) {
        launchArgs.add("-archives");
        launchArgs.add(commandLineInfo.archives);
    }
    if (commandLineInfo.files != null) {
        launchArgs.add("-files");
        launchArgs.add(commandLineInfo.files);
    }
    if (commandLineInfo.origAppId != null) {
        launchArgs.add("-originalAppId");
        launchArgs.add(commandLineInfo.origAppId);
    }
    if (commandLineInfo.queue != null) {
        launchArgs.add("-queue");
        launchArgs.add(commandLineInfo.queue);
    }
    if (commandLineInfo.tags != null) {
        launchArgs.add("-tags");
        launchArgs.add(commandLineInfo.tags);
    }
    launchArgs.add(appFile);
    if (!appFile.endsWith(".json") && !appFile.endsWith(".properties")) {
        launchArgs.add(selectedApp.name);
    }
    LOG.debug("Launch command: {}", StringUtils.join(launchArgs, " "));
    return launchArgs.toArray(new String[] {});
}

Example 4 with DTConfiguration

use of com.datatorrent.stram.client.DTConfiguration in project apex-core by apache.

the class ApexCliTest method testLaunchAppPackagePrecedenceWithConfigPackage.

@Test
public void testLaunchAppPackagePrecedenceWithConfigPackage() throws Exception {
    // set launch command options
    ApexCli.LaunchCommandLineInfo commandLineInfo = ApexCli.getLaunchCommandLineInfo(new String[] { "-D", "dt.test.1=launch-define", "-apconf", "my-app-conf1.xml", "-conf", configFile.getAbsolutePath() });
    // process and look at launch config
    DTConfiguration props = cli.getLaunchAppPackageProperties(ap, cp, commandLineInfo, null);
    Assert.assertEquals("launch-define", props.get("dt.test.1"));
    Assert.assertEquals("config-package", props.get("dt.test.2"));
    Assert.assertEquals("app-package-config", props.get("dt.test.3"));
    Assert.assertEquals("user-home-config", props.get("dt.test.4"));
    Assert.assertEquals("package-default", props.get("dt.test.5"));
    props = cli.getLaunchAppPackageProperties(ap, cp, commandLineInfo, "MyFirstApplication");
    Assert.assertEquals("launch-define", props.get("dt.test.1"));
    Assert.assertEquals("config-package-appname", props.get("dt.test.2"));
    Assert.assertEquals("app-package-config", props.get("dt.test.3"));
    Assert.assertEquals("user-home-config", props.get("dt.test.4"));
    Assert.assertEquals("app-default", props.get("dt.test.5"));
    Assert.assertEquals("package-default", props.get("dt.test.6"));
}

Example 5 with DTConfiguration

use of com.datatorrent.stram.client.DTConfiguration in project apex-core by apache.

the class ApexCliTest method testLaunchAppPackagePrecedenceWithConfigPackageApps.

@Test
public void testLaunchAppPackagePrecedenceWithConfigPackageApps() throws Exception {
    // set launch command options
    ApexCli.LaunchCommandLineInfo commandLineInfo = ApexCli.getLaunchCommandLineInfo(new String[] { "-D", "dt.test.1=launch-define", "-apconf", "my-app-conf1.xml", "-conf", configFile.getAbsolutePath(), "-useConfigApps", "exclusive" });
    // process and look at launch config
    DTConfiguration props = cli.getLaunchAppPackageProperties(ap, cp, commandLineInfo, "testApp");
    Assert.assertEquals("launch-define", props.get("dt.test.1"));
    Assert.assertEquals("config-package", props.get("dt.test.2"));
    Assert.assertEquals("app-package-config", props.get("dt.test.3"));
    Assert.assertEquals("user-home-config", props.get("dt.test.4"));
    Assert.assertEquals("package-default", props.get("dt.test.5"));
}