Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 41 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class RuntimeManagerMain method main.

public static void main(String[] args) {
    setupOptions();
    Options cmdOptions = null;
    try {
        cmdOptions = setupOptions();
        CommandLineParser parser = new DefaultParser();
        // parse the help options first.
        CommandLine cmd = parser.parse(cmdOptions, args);
        // load the configuration
        // we are loading the configuration for all the components
        Config config = loadConfigurations(cmd);
        // normal worker
        LOG.log(Level.INFO, "The runtime controller...");
        String command = cmd.getOptionValue("command");
        executeCommand(config, command);
    } catch (ParseException e) {
        HelpFormatter formatter = new HelpFormatter();
        formatter.printHelp("SubmitterMain", cmdOptions);
        throw new RuntimeException("Error parsing command line options: ", e);
    } catch (Throwable t) {
        String msg = "Un-expected error";
        LOG.log(Level.SEVERE, msg, t);
        throw new RuntimeException(msg, t);
    }
}

Example 42 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class RuntimeManagerMain method loadConfigurations.

private static Config loadConfigurations(CommandLine cmd) {
    String twister2Home = cmd.getOptionValue("twister2_home");
    String configDir = cmd.getOptionValue("config_path");
    String cluster = cmd.getOptionValue("cluster");
    String jobID = cmd.getOptionValue("job_id");
    String command = cmd.getOptionValue("command");
    LOG.log(Level.INFO, String.format("Initializing process with " + "twister_home: %s command: %s config_dir: %s cluster_type: %s", twister2Home, command, configDir, cluster));
    Config config = ConfigLoader.loadConfig(twister2Home, configDir, cluster);
    return Config.newBuilder().putAll(config).put(Context.TWISTER2_HOME.getKey(), twister2Home).put(SchedulerContext.CONFIG_DIR, configDir).put(Context.JOB_ID, jobID).put(Context.TWISTER2_CLUSTER_TYPE, cluster).build();
}

Example 43 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class Twister2Submitter method restartJob.

/**
 * Restart a Twister2 job
 */
public static Twister2JobState restartJob(String jobID, Config config) {
    // job package filename from failed submission
    String prevJobDir = FsContext.uploaderJobDirectory(config) + File.separator + jobID;
    String jobPackage = prevJobDir + File.separator + SchedulerContext.jobPackageFileName(config);
    Path jobPackageFile = Paths.get(jobPackage);
    if (Files.notExists(jobPackageFile)) {
        LOG.severe("Job Package File does not exist: " + jobPackage);
        return new Twister2JobState(false);
    }
    // unpack the previous job package to a temp directory
    Path tempDirPath;
    try {
        tempDirPath = Files.createTempDirectory(jobID);
    } catch (IOException e) {
        throw new Twister2RuntimeException("Can not create temp directory", e);
    }
    // todo: we can exclude user-job-file from being unpacked
    // usually that is the lastest file, so we can be more efficient
    TarGzipPacker.unpack(jobPackageFile, tempDirPath);
    // load Job object
    String unpackedJobDir = tempDirPath + File.separator + Context.JOB_ARCHIVE_DIRECTORY;
    String jobFile = unpackedJobDir + File.separator + SchedulerContext.createJobDescriptionFileName(jobID);
    JobAPI.Job job = JobUtils.readJobFile(jobFile);
    // load previous configurations
    Config prevConfig = ConfigLoader.loadConfig(Context.twister2Home(config), unpackedJobDir, Context.clusterType(config));
    // delete temp directory
    try {
        Files.delete(tempDirPath);
        LOG.info("Unpacked job directory deleted: " + tempDirPath);
    } catch (IOException e) {
        LOG.warning("Exception when deleting temp directory: " + tempDirPath);
    }
    URI packageURI = null;
    try {
        packageURI = new URI(prevJobDir);
    } catch (URISyntaxException e) {
        throw new Twister2RuntimeException("Can not ceate URI for directory: " + prevJobDir, e);
    }
    // add restore parameter
    // local packages path
    prevConfig = Config.newBuilder().putAll(prevConfig).put(CheckpointingContext.CHECKPOINTING_RESTORE_JOB, true).put(SchedulerContext.TEMPORARY_PACKAGES_PATH, prevJobDir).put(SchedulerContext.USER_JOB_FILE, job.getJobFormat().getJobFile()).put(SchedulerContext.JOB_PACKAGE_URI, packageURI).put(Context.TWISTER2_HOME.getKey(), Context.twister2Home(config)).put(Context.JOB_ID, jobID).put(Context.TWISTER2_CLUSTER_TYPE, Context.clusterType(config)).build();
    writeJobIdToFile(jobID);
    printJobInfo(job, prevConfig);
    // launch the launcher
    ResourceAllocator resourceAllocator = new ResourceAllocator(prevConfig, job);
    return resourceAllocator.resubmitJob();
}

Example 44 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class PythonWorker method main.

public static void main(String[] args) throws InterruptedException {
    String pythonFile = System.getProperty("python_file");
    String mainFile = System.getProperty("main_file");
    boolean zip = pythonFile.endsWith("zip");
    if (zip) {
        try {
            String unzipDir = PythonWorkerUtils.unzip(pythonFile);
            pythonFile = new File(unzipDir, mainFile).getAbsolutePath();
        } catch (IOException e) {
            LOG.log(Level.SEVERE, "Failed when extracting zip file.", e);
            return;
        }
    }
    Config config = ResourceAllocator.loadConfig(Collections.emptyMap());
    final BootstrapPoint bootstrapPoint = new BootstrapPoint();
    String mainPy = new File(pythonFile).getAbsolutePath();
    final GatewayServer gatewayServer = initJavaServer(config.getIntegerValue(PYTHON_PORT_OFFSET) - 1, bootstrapPoint, mainPy, true, args, false, 1, config);
    LOG.info("Exchanging configurations...");
    gatewayServer.start();
    bootstrapPoint.waitForCompletion();
    JobConfig jobConfig = new JobConfig();
    jobConfig.putAll(bootstrapPoint.getConfigs());
    jobConfig.put("python_file", new File(pythonFile).getName());
    jobConfig.put("args", args);
    Twister2Job.Twister2JobBuilder twister2JobBuilder = Twister2Job.newBuilder().setJobName(bootstrapPoint.getJobName()).setWorkerClass(CheckpointingContext.isCheckpointingEnabled(config) ? CheckpointablePythonWorker.class : PythonWorker.class).setConfig(jobConfig);
    if (!bootstrapPoint.getComputeResources().isEmpty()) {
        bootstrapPoint.getComputeResources().forEach(computeResource -> {
            twister2JobBuilder.addComputeResource(computeResource.getCpu(), computeResource.getRam(), computeResource.getInstances());
        });
    } else {
        LOG.warning("Computer resources not specified. Using default resource configurations.");
        twister2JobBuilder.addComputeResource(1, 512, 1);
    }
    Twister2Submitter.submitJob(twister2JobBuilder.build(), config);
}

Example 45 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class AuroraWorkerStarter method loadConfig.

/**
 * loadConfig from config files
 */
public void loadConfig() {
    // first lets read the essential properties from java system properties
    String twister2Home = Paths.get("").toAbsolutePath().toString();
    String clusterType = System.getProperty(SchedulerContext.CLUSTER_TYPE);
    String configDir = twister2Home + "/" + JOB_ARCHIVE_DIRECTORY + "/" + clusterType;
    LOG.log(Level.INFO, String.format("Loading configuration with twister2_home: %s and " + "configuration: %s", twister2Home, configDir));
    Config conf = ConfigLoader.loadConfig(twister2Home, JOB_ARCHIVE_DIRECTORY, clusterType);
    config = Config.newBuilder().putAll(conf).put(Context.TWISTER2_HOME.getKey(), twister2Home).put(Context.TWISTER2_CONF.getKey(), configDir).put(Context.TWISTER2_CLUSTER_TYPE, clusterType).build();
    LOG.log(Level.INFO, "Config files are read from directory: " + configDir);
}