Examples with LocalCSVInputPartitioner edu.iu.dsc.tws.data.api.formatters.LocalCSVInputPartitioner used on opensource projects

Example 1 with LocalCSVInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalCSVInputPartitioner in project twister2 by DSC-SPIDAL.

the class PointDataSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    if ("txt".equals(fileType)) {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        }
    } else {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        }
    }
}

Example 2 with LocalCSVInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalCSVInputPartitioner in project twister2 by DSC-SPIDAL.

the class CSVInputFormatTest method testUniqueSchedules.

/**
 * To test the CSV Input Format
 */
@Test
public void testUniqueSchedules() throws IOException {
    Config config = getConfig();
    Path path = new Path("/tmp/dinput/");
    createOutputFile(path, config);
    LocalCSVInputPartitioner csvInputPartitioner = new LocalCSVInputPartitioner(path, 4, config);
    csvInputPartitioner.configure(config);
    FileInputSplit[] inputSplits = csvInputPartitioner.createInputSplits(2);
    LOG.info("input split values are:" + Arrays.toString(inputSplits));
    InputSplitAssigner inputSplitAssigner = csvInputPartitioner.getInputSplitAssigner(inputSplits);
    InputSplit inputSplit = inputSplitAssigner.getNextInputSplit("localhost", 0);
    inputSplit.open(config);
    do {
        inputSplit.nextRecord(null);
    } while (!inputSplit.reachedEnd());
}

Example 3 with LocalCSVInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalCSVInputPartitioner in project twister2 by DSC-SPIDAL.

the class DataObjectCSVSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    this.source = runtime.createInput(cfg, context, new LocalCSVInputPartitioner(new Path(getDataDirectory()), context.getParallelism(), cfg));
}

Example 4 with LocalCSVInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalCSVInputPartitioner in project twister2 by DSC-SPIDAL.

the class TextBasedSourceFunction method prepare.

@Override
public void prepare(TSetContext context) {
    super.prepare(context);
    this.ctx = context;
    Config cfg = ctx.getConfig();
    if ("complete".equals(partitionerType)) {
        this.dataSource = new DataSource(cfg, new LocalCompleteCSVInputPartitioner(new Path(datainputDirectory), context.getParallelism(), dataSize, cfg), parallel);
    } else {
        this.dataSource = new DataSource(cfg, new LocalCSVInputPartitioner(new Path(datainputDirectory), parallel, dataSize, cfg), parallel);
    }
    this.dataSplit = this.dataSource.getNextSplit(context.getIndex());
}