Search in sources :

Example 26 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class KeyedOperationsExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    KeyedSourceTSet<String, Integer> kSource = dummyKeyedSource(env, COUNT, PARALLELISM);
    KeyedDirectTLink<String, Integer> kDirect = kSource.keyedDirect();
    kDirect.forEach(i -> LOG.info("d_fe: " + i.toString()));
    KeyedCachedTSet<String, Integer> cache = kDirect.cache();
    cache.keyedDirect().forEach(i -> LOG.info("c_d_fe: " + i.toString()));
    cache.keyedReduce(Integer::sum).forEach(i -> LOG.info("c_r_fe: " + i.toString()));
    cache.keyedGather().forEach((ApplyFunc<Tuple<String, Iterator<Integer>>>) data -> {
        StringBuilder sb = new StringBuilder();
        sb.append("c_g_fe: key ").append(data.getKey()).append("->");
        while (data.getValue().hasNext()) {
            sb.append(" ").append(data.getValue().next());
        }
        LOG.info(sb.toString());
    });
}
Also used : Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) Iterator(java.util.Iterator) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) KeyedDirectTLink(edu.iu.dsc.tws.tset.links.batch.KeyedDirectTLink) Logger(java.util.logging.Logger) KeyedCachedTSet(edu.iu.dsc.tws.tset.sets.batch.KeyedCachedTSet) JobConfig(edu.iu.dsc.tws.api.JobConfig) KeyedSourceTSet(edu.iu.dsc.tws.tset.sets.batch.KeyedSourceTSet) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) ApplyFunc(edu.iu.dsc.tws.api.tset.fn.ApplyFunc) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 27 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class JoinExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int para = 2;
    int workerID = env.getWorkerID();
    SourceTSet<Integer> src0 = dummySource(env, COUNT, para).setName("src0");
    KeyedTSet<Integer, Integer> left = src0.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("left");
    left.keyedDirect().forEach(i -> LOG.info(workerID + "L " + i.toString()));
    SourceTSet<Integer> src1 = dummySource(env, COUNT, para).setName("src1");
    KeyedTSet<Integer, Integer> right = src1.mapToTuple(i -> new Tuple<>(i % 2, i)).setName("right");
    right.keyedDirect().forEach(i -> LOG.info(workerID + "R " + i.toString()));
    JoinTLink<Integer, Integer, Integer> join = left.join(right, CommunicationContext.JoinType.INNER, Integer::compareTo).setName("join");
    join.forEach(t -> LOG.info(workerID + "out: " + t.toString()));
}
Also used : CommunicationContext(edu.iu.dsc.tws.api.comms.CommunicationContext) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) KeyedTSet(edu.iu.dsc.tws.tset.sets.batch.KeyedTSet) JobConfig(edu.iu.dsc.tws.api.JobConfig) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) JoinTLink(edu.iu.dsc.tws.tset.links.batch.JoinTLink) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 28 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class GatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    GatherTLink<Integer> gather = src.gather();
    LOG.info("test foreach");
    gather.forEach(i -> LOG.info("foreach: " + i));
    LOG.info("test map");
    gather.map(i -> i.toString() + "$$").withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    gather.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        return "sum=" + sum;
    }).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("compute: " + s));
    LOG.info("test computec");
    gather.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        int sum = 0;
        while (input.hasNext()) {
            sum += input.next().getValue();
        }
        output.collect("sum=" + sum);
    }).withSchema(PrimitiveSchemas.STRING).direct().forEach(s -> LOG.info("computec: " + s));
    gather.mapToTuple(i -> new Tuple<>(i % 2, i)).keyedDirect().forEach(i -> LOG.info("mapToTuple: " + i.toString()));
}
Also used : Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) Iterator(java.util.Iterator) ComputeCollectorFunc(edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) GatherTLink(edu.iu.dsc.tws.tset.links.batch.GatherTLink) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) ComputeFunc(edu.iu.dsc.tws.api.tset.fn.ComputeFunc) PrimitiveSchemas(edu.iu.dsc.tws.api.tset.schema.PrimitiveSchemas) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) ComputeCollectorFunc(edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc) ComputeFunc(edu.iu.dsc.tws.api.tset.fn.ComputeFunc) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 29 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class KGatherExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    KeyedGatherTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedGather();
    LOG.info("test foreach");
    klink.forEach((ApplyFunc<Tuple<Integer, Iterator<Integer>>>) data -> LOG.info(data.getKey() + " -> " + iterToString(data.getValue())));
    LOG.info("test map");
    klink.map((MapFunc<Tuple<Integer, Iterator<Integer>>, String>) input -> {
        int s = 0;
        while (input.getValue().hasNext()) {
            s += input.getValue().next();
        }
        return input.getKey() + " -> " + s;
    }).direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    klink.compute((ComputeFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) input -> {
        StringBuilder s = new StringBuilder();
        while (input.hasNext()) {
            Tuple<Integer, Iterator<Integer>> next = input.next();
            s.append(" [").append(next.getKey()).append(" -> ").append(iterToString(next.getValue())).append("] ");
        }
        return s.toString();
    }).direct().forEach(s -> LOG.info("compute: concat " + s));
    LOG.info("test computec");
    klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Iterator<Integer>>>, String>) (input, output) -> {
        while (input.hasNext()) {
            Tuple<Integer, Iterator<Integer>> next = input.next();
            output.collect(next.getKey() + " -> " + iterToString(next.getValue()));
        }
    }).direct().forEach(s -> LOG.info("computec: " + s));
    // Test byte[] key value pairs for KeyedGather
    SourceTSet<String> srcString = dummyStringSource(env, 25, PARALLELISM);
    KeyedGatherTLink<byte[], Integer> keyedGatherLink = srcString.mapToTuple(s -> new Tuple<>(s.getBytes(), 1)).keyedGather();
    LOG.info("test foreach");
    keyedGatherLink.forEach((ApplyFunc<Tuple<byte[], Iterator<Integer>>>) data -> LOG.info(new String(data.getKey()) + " -> " + iterToString(data.getValue())));
}
Also used : Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) Iterator(java.util.Iterator) ComputeCollectorFunc(edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) MapFunc(edu.iu.dsc.tws.api.tset.fn.MapFunc) KeyedGatherTLink(edu.iu.dsc.tws.tset.links.batch.KeyedGatherTLink) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) ComputeFunc(edu.iu.dsc.tws.api.tset.fn.ComputeFunc) ApplyFunc(edu.iu.dsc.tws.api.tset.fn.ApplyFunc) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) Iterator(java.util.Iterator) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Example 30 with WorkerEnvironment

use of edu.iu.dsc.tws.api.resource.WorkerEnvironment in project twister2 by DSC-SPIDAL.

the class KPartitionExample method execute.

@Override
public void execute(WorkerEnvironment workerEnv) {
    BatchEnvironment env = TSetEnvironment.initBatch(workerEnv);
    int start = env.getWorkerID() * 100;
    SourceTSet<Integer> src = dummySource(env, start, COUNT, PARALLELISM);
    KeyedPartitionTLink<Integer, Integer> klink = src.mapToTuple(i -> new Tuple<>(i % 10, i)).keyedPartition(new LoadBalancePartitioner<>());
    LOG.info("test foreach");
    klink.forEach(t -> LOG.info(t.getKey() + "_" + t.getValue()));
    LOG.info("test map");
    klink.map(i -> i.toString() + "$$").direct().forEach(s -> LOG.info("map: " + s));
    LOG.info("test compute");
    klink.compute((ComputeFunc<Iterator<Tuple<Integer, Integer>>, String>) input -> {
        StringBuilder s = new StringBuilder();
        while (input.hasNext()) {
            s.append(input.next().toString()).append(" ");
        }
        return s.toString();
    }).direct().forEach(s -> LOG.info("compute: concat " + s));
    LOG.info("test computec");
    klink.compute((ComputeCollectorFunc<Iterator<Tuple<Integer, Integer>>, String>) (input, output) -> {
        while (input.hasNext()) {
            output.collect(input.next().toString());
        }
    }).direct().forEach(s -> LOG.info("computec: " + s));
}
Also used : Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple) Iterator(java.util.Iterator) ComputeCollectorFunc(edu.iu.dsc.tws.api.tset.fn.ComputeCollectorFunc) SourceTSet(edu.iu.dsc.tws.tset.sets.batch.SourceTSet) ResourceAllocator(edu.iu.dsc.tws.rsched.core.ResourceAllocator) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) LoadBalancePartitioner(edu.iu.dsc.tws.tset.fn.LoadBalancePartitioner) HashMap(java.util.HashMap) Config(edu.iu.dsc.tws.api.config.Config) Logger(java.util.logging.Logger) JobConfig(edu.iu.dsc.tws.api.JobConfig) WorkerEnvironment(edu.iu.dsc.tws.api.resource.WorkerEnvironment) TSetEnvironment(edu.iu.dsc.tws.tset.env.TSetEnvironment) ComputeFunc(edu.iu.dsc.tws.api.tset.fn.ComputeFunc) KeyedPartitionTLink(edu.iu.dsc.tws.tset.links.batch.KeyedPartitionTLink) BatchEnvironment(edu.iu.dsc.tws.tset.env.BatchEnvironment) Iterator(java.util.Iterator) Tuple(edu.iu.dsc.tws.api.comms.structs.Tuple)

Aggregations

WorkerEnvironment (edu.iu.dsc.tws.api.resource.WorkerEnvironment)49 Logger (java.util.logging.Logger)46 Config (edu.iu.dsc.tws.api.config.Config)42 Iterator (java.util.Iterator)27 TSetEnvironment (edu.iu.dsc.tws.tset.env.TSetEnvironment)26 JobConfig (edu.iu.dsc.tws.api.JobConfig)25 Tuple (edu.iu.dsc.tws.api.comms.structs.Tuple)24 ResourceAllocator (edu.iu.dsc.tws.rsched.core.ResourceAllocator)23 BatchEnvironment (edu.iu.dsc.tws.tset.env.BatchEnvironment)23 SourceTSet (edu.iu.dsc.tws.tset.sets.batch.SourceTSet)23 HashMap (java.util.HashMap)22 LogicalPlanBuilder (edu.iu.dsc.tws.comms.utils.LogicalPlanBuilder)21 MessageTypes (edu.iu.dsc.tws.api.comms.messaging.types.MessageTypes)20 Set (java.util.Set)20 ResultsVerifier (edu.iu.dsc.tws.examples.verification.ResultsVerifier)19 IntArrayComparator (edu.iu.dsc.tws.examples.verification.comparators.IntArrayComparator)19 BenchmarkUtils (edu.iu.dsc.tws.examples.utils.bench.BenchmarkUtils)18 Timing (edu.iu.dsc.tws.examples.utils.bench.Timing)18 BenchWorker (edu.iu.dsc.tws.examples.comms.BenchWorker)14 BenchmarkConstants (edu.iu.dsc.tws.examples.utils.bench.BenchmarkConstants)13