Examples with WorkerEnvironment edu.iu.dsc.tws.api.resource.WorkerEnvironment used on opensource projects

Example 31 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class BroadcastExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    SourceTSet<Integer> src = dummySource(env, COUNT, 1);
    ReplicateTLink<Integer> replicate = src.replicate(PARALLELISM);
    LOG.info("test foreach");
    replicate.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    replicate.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    replicate.flatmap((i, c) -> c.collect(i.toString() + "##")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    replicate.compute((ComputeFunc<Iterator<Integer>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        return "sum" + sum;
    }).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    replicate.compute((ComputeCollectorFunc<Iterator<Integer>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next();
        }
        output.collect("sum" + sum);
    }).direct().forEach(s -> LOG.info("computec: " + s));
    LOG.info("test sink");
    SinkTSet<Iterator<Integer>> sink = replicate.sink((SinkFunc<Iterator<Integer>>) value -> {
        while (value.hasNext()) {
            LOG.info("val =" + value.next());
        }
        return true;
    });
    env.run(sink);
}

Example 32 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class AllReduceExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    LOG.info("test foreach");
    src.allReduce(Integer::sum).forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    src.allReduce(Integer::sum).map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test flat map");
    src.allReduce(Integer::sum).flatmap((i, c) -> c.collect(i.toString() + "$$")).direct().forEach(s -> LOG.info("flat:" + s));
    LOG.info("test compute");
    src.allReduce(Integer::sum).compute(i -> i * 2).direct().forEach(i -> LOG.info("comp: " + i));
    LOG.info("test computec");
    src.allReduce(Integer::sum).compute((ComputeCollectorFunc<Integer, String>) (input, output) -> output.collect("sum=" + input)).direct().forEach(s -> LOG.info("computec: " + s));
}

Example 33 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class ComputeExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM).setName("src").withSchema(PrimitiveSchemas.INTEGER);
    ComputeTSet<Integer> sum = src.direct().compute((ComputeFunc<Iterator<Integer>, Integer>) input -> {
        int s = 0;
        while (input.hasNext()) {
            s += input.next();
        }
        return s;
    }).withSchema(PrimitiveSchemas.INTEGER).setName("sum");
    sum.direct().forEach(data -> LOG.info("val: " + data));
    sum.reduce(Integer::sum).forEach(i -> LOG.info("red: " + i));
}

Example 34 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class Twister2WorkerStarter method execute.

@Override
public void execute(Config config, JobAPI.Job job, IWorkerController workerController, IPersistentVolume persistentVolume, IVolatileVolume volatileVolume) {
    int workerID = workerController.getWorkerInfo().getWorkerID();
    WorkerEnvironment workerEnv = WorkerEnvironment.init(config, job, workerController, persistentVolume, volatileVolume);
    String workerClass = job.getWorkerClassName();
    Twister2Worker worker;
    try {
        Object object = ReflectionUtils.newInstance(workerClass);
        worker = (Twister2Worker) object;
        LOG.info("loaded worker class: " + workerClass);
        worker.execute(workerEnv);
    } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
        LOG.severe(String.format("failed to load the worker class %s", workerClass));
        throw new RuntimeException(e);
    }
    // If the execute returns without any errors we assume that the job completed properly
    if (JobMasterContext.isJobMasterUsed(config) && !job.getDriverClassName().isEmpty()) {
        ISenderToDriver senderToDriver = WorkerRuntime.getSenderToDriver();
        JobExecutionState.WorkerJobState workerState = JobExecutionState.WorkerJobState.newBuilder().setFailure(false).setJobName(config.getStringValue(Context.JOB_ID)).setWorkerMessage("Worker Completed").build();
        senderToDriver.sendToDriver(workerState);
    }
}

Example 35 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class SGatherExample method compute.

@Override
protected void compute(WorkerEnvironment workerEnv) {
    if (jobParameters.getTargets() != 1) {
        LOG.warning("Setting targets to 1. Found, " + jobParameters.getTargets());
        jobParameters.getTaskStages().set(1, 1);
    }
    LogicalPlanBuilder logicalPlanBuilder = LogicalPlanBuilder.plan(jobParameters.getSources(), jobParameters.getTargets(), workerEnv);
    // create the communication
    gather = new SGather(workerEnv.getCommunicator(), logicalPlanBuilder, MessageTypes.INTEGER_ARRAY, new FinalReduceReceiver());
    Set<Integer> tasksOfExecutor = logicalPlanBuilder.getSourcesOnThisWorker();
    for (int t : tasksOfExecutor) {
        finishedSources.put(t, false);
    }
    if (tasksOfExecutor.size() == 0) {
        sourcesDone = true;
    }
    if (!logicalPlan.getLogicalIdsOfWorker(workerId).contains(logicalPlanBuilder.getTargets().iterator().next())) {
        gatherDone = true;
    }
    this.resultsVerifier = new ResultsVerifier<>(inputDataArray, (dataArray, args) -> {
        List<Tuple<Integer, int[]>> listOfArrays = new ArrayList<>();
        for (int i = 0; i < logicalPlanBuilder.getSources().size(); i++) {
            listOfArrays.add(new Tuple<>(i, dataArray));
        }
        return listOfArrays.iterator();
    }, new IteratorComparator<>(new TupleComparator<>(IntComparator.getInstance(), IntArrayComparator.getInstance())));
    // now initialize the workers
    for (int t : tasksOfExecutor) {
        // the map thread where data is produced
        Thread mapThread = new Thread(new BenchWorker.MapWorker(t));
        mapThread.start();
    }
}