Examples with Config edu.iu.dsc.tws.common.config.Config used on opensource projects

Example 46 with Config

use of edu.iu.dsc.tws.common.config.Config in project twister2 by DSC-SPIDAL.

the class MPILauncher method launch.

@Override
public boolean launch(RequestedResources resourcePlan, JobAPI.Job job) {
    LOG.log(Level.INFO, "Launching job for cluster {0}", MPIContext.clusterType(config));
    if (!setupWorkingDirectory(job)) {
        throw new RuntimeException("Failed to setup the directory");
    }
    Config newConfig = Config.newBuilder().putAll(config).put(SchedulerContext.WORKING_DIRECTORY, jobWorkingDirectory).build();
    // now start the controller, which will get the resources from
    // slurm and start the job
    IController controller = new MPIController(true);
    controller.initialize(newConfig);
    return controller.start(resourcePlan, job);
}

Example 47 with Config

use of edu.iu.dsc.tws.common.config.Config in project twister2 by DSC-SPIDAL.

the class ResourceAllocator method loadConfig.

/**
 * loadConfig from config files and also from envirobnment variables
 *
 * @param cfg the config values in this map will be put into returned Config
 */
public static Config loadConfig(Map<String, Object> cfg) {
    // first lets read the essential properties from java system properties
    String twister2Home = System.getProperty(SchedulerContext.TWISTER_2_HOME);
    String configDir = System.getProperty(SchedulerContext.CONFIG_DIR);
    String clusterType = System.getProperty(SchedulerContext.CLUSTER_TYPE);
    // lets get the job jar file from system properties or environment
    String jobJar = System.getProperty(SchedulerContext.USER_JOB_JAR_FILE);
    // now lets see weather these are overridden in environment variables
    Map<String, Object> environmentProperties = JobUtils.readCommandLineOpts();
    if (environmentProperties.containsKey(SchedulerContext.TWISTER_2_HOME)) {
        twister2Home = (String) environmentProperties.get(SchedulerContext.CONFIG_DIR);
    }
    if (environmentProperties.containsKey(SchedulerContext.CONFIG_DIR)) {
        configDir = (String) environmentProperties.get(SchedulerContext.CONFIG_DIR);
    }
    if (environmentProperties.containsKey(SchedulerContext.CLUSTER_TYPE)) {
        clusterType = (String) environmentProperties.get(SchedulerContext.CLUSTER_TYPE);
    }
    if (environmentProperties.containsKey(SchedulerContext.USER_JOB_JAR_FILE)) {
        jobJar = (String) environmentProperties.get(SchedulerContext.USER_JOB_JAR_FILE);
    }
    if (configDir == null) {
        configDir = twister2Home + "/conf";
    }
    LOG.log(Level.INFO, String.format("Loading configuration with twister2_home: %s and " + "configuration: %s and cluster: %s", twister2Home, configDir, clusterType));
    Config config = ConfigLoader.loadConfig(twister2Home, configDir + "/" + clusterType);
    return Config.newBuilder().putAll(config).put(MPIContext.TWISTER2_HOME.getKey(), twister2Home).put(MPIContext.TWISTER2_CLUSTER_TYPE, clusterType).put(MPIContext.USER_JOB_JAR_FILE, jobJar).putAll(environmentProperties).putAll(cfg).build();
}

Example 48 with Config

use of edu.iu.dsc.tws.common.config.Config in project twister2 by DSC-SPIDAL.

the class AuroraJobSubmitter method main.

public static void main(String[] args) {
    Options cmdOptions = null;
    try {
        // get command line parameters
        cmdOptions = setupOptions();
        CommandLineParser parser = new DefaultParser();
        CommandLine cmd = parser.parse(cmdOptions, args);
        // load the configurations from config files
        // we are loading the configuration for all the components
        Config config = loadConfigurations(cmd);
        System.out.println("all config entries");
        System.out.println("number of config parameters: " + config.size());
        System.out.println(config);
        // construct the controller to submit the job to Aurora Scheduler
        String cluster = AuroraContext.auroraClusterName(config);
        String role = AuroraContext.role(config);
        String env = AuroraContext.environment(config);
        String jobName = SchedulerContext.jobName(config);
        AuroraClientController controller = new AuroraClientController(cluster, role, env, jobName, true);
        // get aurora file name to execute when submitting the job
        String auroraFilename = AuroraContext.auroraScript(config);
        // get environment variables from config
        Map<AuroraField, String> bindings = AuroraLauncher.constructEnvVariables(config);
        // print all environment variables for debugging
        AuroraLauncher.printEnvs(bindings);
        boolean jobSubmitted = controller.createJob(bindings, auroraFilename);
        if (jobSubmitted) {
            LOG.log(Level.INFO, "job submission is successfull ...");
        } else {
            LOG.log(Level.SEVERE, "job submission to Aurora failed ...");
        }
    } catch (ParseException e) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("AuroraJobSubmitter", cmdOptions);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
}

Example 49 with Config

use of edu.iu.dsc.tws.common.config.Config in project twister2 by DSC-SPIDAL.

the class AuroraJobSubmitter method loadConfigurations.

/**
 * read config parameters from configuration files
 * all config files are in a single directory
 *
 * @return Config object that has values from config files and from command line
 */
private static Config loadConfigurations(CommandLine cmd) {
    String twister2Home = cmd.getOptionValue("twister2_home");
    String configDir = cmd.getOptionValue("config_dir");
    String clusterType = cmd.getOptionValue("cluster_type");
    // String packagePath = cmd.getOptionValue("package_path");
    // String packageFile = cmd.getOptionValue("package_file");
    LOG.log(Level.INFO, String.format("Initializing process with " + "twister_home: %s config_dir: %s cluster_type: %s", twister2Home, configDir, clusterType));
    try {
        // Reflection.initialize(Class.forName(
        // "edu.iu.dsc.tws.rsched.schedulers.aurora.AuroraContext"));
        Class.forName(AuroraContext.class.getName());
    } catch (ClassNotFoundException e) {
        e.printStackTrace();
    }
    Config config = ConfigLoader.loadConfig(twister2Home, configDir + "/" + clusterType);
    return Config.newBuilder().putAll(config).put(SchedulerContext.TWISTER2_HOME.getKey(), twister2Home).put(SchedulerContext.TWISTER2_CLUSTER_TYPE, clusterType).build();
}

Example 50 with Config

use of edu.iu.dsc.tws.common.config.Config in project twister2 by DSC-SPIDAL.

the class AuroraWorker method loadConfig.

/**
 * loadConfig from config files
 * @return
 */
public void loadConfig() {
    // first lets read the essential properties from java system properties
    String twister2Home = Paths.get("").toAbsolutePath().toString();
    String clusterType = System.getProperty(SchedulerContext.CLUSTER_TYPE);
    String configDir = twister2Home + "/" + DIR_PREFIX_FOR_JOB_ARCHIVE + clusterType;
    LOG.log(Level.INFO, String.format("Loading configuration with twister2_home: %s and " + "configuration: %s", twister2Home, configDir));
    Config conf = ConfigLoader.loadConfig(twister2Home, configDir);
    config = Config.newBuilder().putAll(conf).put(Context.TWISTER2_HOME.getKey(), twister2Home).put(Context.TWISTER2_CONF.getKey(), configDir).put(Context.TWISTER2_CLUSTER_TYPE, clusterType).build();
    LOG.log(Level.INFO, "Config files are read from directory: " + configDir);
}