Examples with LocalCompleteTextInputPartitioner edu.iu.dsc.tws.data.api.formatters.LocalCompleteTextInputPartitioner used on opensource projects

Example 1 with LocalCompleteTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalCompleteTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class PointDataSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    if ("txt".equals(fileType)) {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        }
    } else {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        }
    }
}

Example 2 with LocalCompleteTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalCompleteTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class DataFileReplicatedReadSource method prepare.

public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    this.source = runtime.createInput(cfg, context, new LocalCompleteTextInputPartitioner(new Path(getDataDirectory()), context.getParallelism(), config));
}