Examples with Config edu.iu.dsc.tws.api.config.Config used on opensource projects

Example 91 with Config

use of edu.iu.dsc.tws.api.config.Config in project beam by apache.

the class Twister2Runner method run.

@Override
public PipelineResult run(Pipeline pipeline) {
    // create a worker and pass in the pipeline and then do the translation
    Twister2PipelineExecutionEnvironment env = new Twister2PipelineExecutionEnvironment(options);
    LOG.info("Translating pipeline to Twister2 program.");
    pipeline.replaceAll(getDefaultOverrides());
    // TODO(BEAM-10670): Use SDF read as default when we address performance issue.
    if (!ExperimentalOptions.hasExperiment(pipeline.getOptions(), "beam_fn_api")) {
        SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(pipeline);
    }
    env.translate(pipeline);
    setupSystem(options);
    Map configMap = new HashMap();
    JobConfig jobConfig = new JobConfig();
    if (isLocalMode(options)) {
        options.setParallelism(1);
        configMap.put(SIDEINPUTS, extractNames(env.getSideInputs()));
        configMap.put(LEAVES, extractNames(env.getLeaves()));
        configMap.put(GRAPH, env.getTSetGraph());
        configMap.put("twister2.network.buffer.size", 32000);
        configMap.put("twister2.network.sendBuffer.count", options.getParallelism());
        LOG.warning("Twister2 Local Mode currently only supports single worker");
    } else {
        jobConfig.put(SIDEINPUTS, extractNames(env.getSideInputs()));
        jobConfig.put(LEAVES, extractNames(env.getLeaves()));
        jobConfig.put(GRAPH, env.getTSetGraph());
    }
    Config config = ResourceAllocator.loadConfig(configMap);
    int workers = options.getParallelism();
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName(options.getJobName()).setWorkerClass(BeamBatchWorker.class).addComputeResource(options.getWorkerCPUs(), options.getRamMegaBytes(), workers).setConfig(jobConfig).build();
    Twister2JobState jobState;
    if (isLocalMode(options)) {
        jobState = LocalSubmitter.submitJob(twister2Job, config);
    } else {
        jobState = Twister2Submitter.submitJob(twister2Job, config);
    }
    Twister2PipelineResult result = new Twister2PipelineResult(jobState);
    return result;
}

Example 92 with Config

use of edu.iu.dsc.tws.api.config.Config in project beam by apache.

the class Twister2Runner method runTest.

public PipelineResult runTest(Pipeline pipeline) {
    // create a worker and pass in the pipeline and then do the translation
    Twister2PipelineExecutionEnvironment env = new Twister2PipelineExecutionEnvironment(options);
    LOG.info("Translating pipeline to Twister2 program.");
    pipeline.replaceAll(getDefaultOverrides());
    // TODO(BEAM-10670): Use SDF read as default when we address performance issue.
    if (!ExperimentalOptions.hasExperiment(pipeline.getOptions(), "beam_fn_api")) {
        SplittableParDo.convertReadBasedSplittableDoFnsToPrimitiveReadsIfNecessary(pipeline);
    }
    env.translate(pipeline);
    setupSystemTest(options);
    Map configMap = new HashMap();
    configMap.put(SIDEINPUTS, extractNames(env.getSideInputs()));
    configMap.put(LEAVES, extractNames(env.getLeaves()));
    configMap.put(GRAPH, env.getTSetGraph());
    configMap.put("twister2.network.buffer.size", 32000);
    configMap.put("twister2.network.sendBuffer.count", options.getParallelism());
    Config config = ResourceAllocator.loadConfig(configMap);
    JobConfig jobConfig = new JobConfig();
    int workers = options.getParallelism();
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName(options.getJobName()).setWorkerClass(BeamBatchWorker.class).addComputeResource(options.getWorkerCPUs(), options.getRamMegaBytes(), workers).setConfig(jobConfig).build();
    Twister2JobState jobState = LocalSubmitter.submitJob(twister2Job, config);
    Twister2PipelineResult result = new Twister2PipelineResult(jobState);
    // TODO: Need to fix the check for "RUNNING" once fix for this is done on Twister2 end.
    if (result.state == PipelineResult.State.FAILED) {
        throw new RuntimeException("Pipeline execution failed", jobState.getCause());
    }
    return result;
}

Example 93 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class ConfigLoader method loadConfig.

/**
 * Loads raw configurations from files under the twister2 Home and configPath. The returned config
 * must be converted to either local or cluster mode to trigger pattern substitution of wildcards
 * tokens.
 */
public static Config loadConfig(String twister2Home, String configPath) {
    Config defaultConfig = loadDefaults(twister2Home, configPath);
    // to token-substitute the conf paths
    Config localConfig = Config.transform(defaultConfig);
    // now load the configurations
    Config.Builder cb = Config.newBuilder().putAll(localConfig).putAll(loadConfig(Context.taskConfigurationFile(localConfig))).putAll(loadConfig(Context.resourceSchedulerConfigurationFile(localConfig))).putAll(loadConfig(Context.networkConfigurationFile(localConfig))).putAll(loadConfig(Context.systemConfigurationFile(localConfig))).putAll(loadConfig(Context.dataConfigurationFile(localConfig))).putAll(loadConfig(Context.checkpointConfigurationFile(localConfig)));
    Config config = cb.build();
    return Config.transform(config);
}

Example 94 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class JobMasterExample method main.

/**
 * this main method is for locally testing only
 * A JobMaster instance is started locally on the default port:
 * edu.iu.dsc.tws.master.JobMasterContext.JOB_MASTER_PORT_DEFAULT = 11011
 * <p>
 * numberOfWorkers to join is expected as a parameter
 * <p>
 * When all workers joined and all have sent completed messages,
 * this server also completes and exits
 * <p>
 * En example usage of JobMaster can be seen in:
 * edu.iu.dsc.tws.rsched.schedulers.k8s.master.JobMasterStarter
 */
public static void main(String[] args) {
    if (args.length != 1) {
        LOG.info("usage: java JobMasterExample numberOfWorkers");
        return;
    }
    int numberOfWorkers = Integer.parseInt(args[0]);
    String host = "0.0.0.0";
    // we assume that the twister2Home is the current directory
    // String configDir = "../twister2/config/src/yaml/";
    String configDir = "";
    String twister2Home = Paths.get(configDir).toAbsolutePath().toString();
    Config config = ConfigLoader.loadConfig(twister2Home, "conf", "kubernetes");
    config = JobMasterClientExample.updateConfig(config, config, host);
    LOG.info("Loaded: " + config.size() + " configuration parameters.");
    // Twister2Job twister2Job = Twister2Job.loadTwister2Job(config, null);
    Twister2Job twister2Job = Twister2Job.newBuilder().setJobName("hello-world-job").setWorkerClass(HelloWorld.class).addComputeResource(.2, 128, numberOfWorkers).build();
    twister2Job.setUserName(System.getProperty("user.name"));
    JobAPI.Job job = twister2Job.serialize();
    LOG.info("JobID: " + job.getJobId());
    JobMasterAPI.JobMasterState initialState = JobMasterAPI.JobMasterState.JM_STARTED;
    JobMasterStarter.job = job;
    if (ZKContext.isZooKeeperServerUsed(config)) {
        if ("start".equalsIgnoreCase(args[0])) {
            JobMasterStarter.initializeZooKeeper(config, job.getJobId(), host, initialState);
        } else if ("restart".equalsIgnoreCase(args[0])) {
            initialState = JobMasterAPI.JobMasterState.JM_RESTARTED;
            JobMasterStarter.initializeZooKeeper(config, job.getJobId(), host, initialState);
            job = JobMasterStarter.job;
        } else {
            LOG.info("usage: java JobMasterExample start/restart");
            return;
        }
    }
    // write jobID to file
    String dir = System.getProperty("user.home") + "/.twister2";
    if (!FileUtils.isDirectoryExists(dir)) {
        FileUtils.createDirectory(dir);
    }
    String filename = dir + "/last-job-id.txt";
    FileUtils.writeToFile(filename, (job.getJobId() + "").getBytes(), true);
    LOG.info("Written jobID to file: " + job.getJobId());
    String ip = null;
    try {
        ip = Inet4Address.getLocalHost().getHostAddress();
    } catch (UnknownHostException e) {
        LOG.log(Level.SEVERE, e.getMessage(), e);
        return;
    }
    JobMasterAPI.NodeInfo jobMasterNode = NodeInfoUtils.createNodeInfo(ip, null, null);
    KubernetesController controller = KubernetesController.init("default");
    K8sScaler k8sScaler = new K8sScaler(config, job, controller);
    IJobTerminator jobTerminator = new NullTerminator();
    JobMaster jobMaster = new JobMaster(config, host, jobTerminator, job, jobMasterNode, k8sScaler, initialState);
    try {
        // jobMaster.startJobMasterThreaded();
        jobMaster.startJobMasterBlocking();
    } catch (Twister2Exception e) {
        LOG.log(Level.SEVERE, "Exception when starting Job master: ", e);
        throw new RuntimeException(e);
    }
    LOG.info("Threaded Job Master started:" + "\nnumberOfWorkers: " + job.getNumberOfWorkers() + "\njobID: " + job.getJobId());
}

Example 95 with Config

use of edu.iu.dsc.tws.api.config.Config in project twister2 by DSC-SPIDAL.

the class ZKJobMasterRegistrarExample method main.

/**
 * we assume that we have the Job Master IP address and the port number
 * We will register this pair of information on a ZooKeeper server
 * Workers will discover the Job Master address by querying this ZooKeeper server
 * <p>
 * If there is already a znode on the ZooKeeper with the same name,
 * we delete that znode. It must be from a previous registration session
 * <p>
 * Parameters:
 * the only parameter is the ZooKeeper server address
 * <p>
 * This class is used together with ZKJobMasterFinderExample.java
 * This class registers the Job Master and that class discovers it
 */
public static void main(String[] args) {
    if (args.length != 1) {
        printUsage();
        return;
    }
    String zkAddress = args[0];
    String jobID = "test-job";
    Config cnfg = buildConfig(zkAddress);
    String jobMasterIP = "x.y.z.t";
    // get the default port
    int jobMasterPort = JobMasterContext.jobMasterPort(cnfg);
    ZKJobMasterRegistrar registrar = new ZKJobMasterRegistrar(cnfg, jobMasterIP, jobMasterPort, jobID);
    boolean initialized = registrar.initialize();
    if (!initialized && registrar.sameZNodeExist()) {
        registrar.deleteJobMasterZNode();
        registrar.initialize();
    }
    try {
        long waitDuration = 30;
        LOG.info("Waiting " + waitDuration + "seconds. Will exit afterwards...");
        Thread.sleep(waitDuration * 1000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    registrar.close();
    LOG.info("Done, exiting ...");
}