Examples with LocalTextInputPartitioner edu.iu.dsc.tws.data.api.formatters.LocalTextInputPartitioner used on opensource projects

Example 1 with LocalTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class PointDataSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    if ("txt".equals(fileType)) {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        }
    } else {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        }
    }
}

Example 2 with LocalTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class DataObjectSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(getDataDirectory()), context.getParallelism(), cfg));
}

Example 3 with LocalTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class DataParallelTask method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    String directory = cfg.getStringValue(Constants.ARGS_INPUT_DIRECTORY);
    ExecutionRuntime runtime = (ExecutionRuntime) config.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    String outDir = cfg.getStringValue(Constants.ARGS_OUTPUT_DIRECTORY);
    boolean shared = cfg.getBooleanValue(Constants.ARGS_SHARED_FILE_SYSTEM);
    if (!shared) {
        this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(directory), context.getParallelism()));
    } else {
        this.source = runtime.createInput(cfg, context, new SharedTextInputPartitioner(new Path(directory)));
    }
    this.sink = new DataSink<String>(cfg, new TextOutputWriter(FileSystem.WriteMode.OVERWRITE, new Path(outDir)));
}

Example 4 with LocalTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class DataFileSplittedReadSource method prepare.

/**
 * Retrieve the data input directory using the DataObjectConstants and the config file.
 * @param cfg
 * @param context
 */
@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(getDataDirectory()), context.getParallelism(), config));
}

Example 5 with LocalTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.LocalTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class KMeansDataGeneratorTest method testUniqueSchedules1.

@Test
public void testUniqueSchedules1() throws IOException {
    Config config = getConfig();
    String dinputDirectory = "/tmp/testdinput";
    int numFiles = 1;
    int dsize = 20;
    int dimension = 2;
    int parallelismValue = 2;
    KMeansDataGenerator.generateData("txt", new Path(dinputDirectory), numFiles, dsize, 100, dimension, config);
    ComputeGraphBuilder computeGraphBuilder = ComputeGraphBuilder.newBuilder(config);
    computeGraphBuilder.setTaskGraphName("kmeans");
    DataObjectSource sourceTask = new DataObjectSource("direct", dinputDirectory);
    DataObjectSink sinkTask = new DataObjectSink();
    computeGraphBuilder.addSource("source", sourceTask, parallelismValue);
    ComputeConnection computeConnection1 = computeGraphBuilder.addCompute("sink", sinkTask, parallelismValue);
    computeConnection1.direct("source").viaEdge("direct").withDataType(MessageTypes.OBJECT);
    computeGraphBuilder.setMode(OperationMode.BATCH);
    LocalTextInputPartitioner localTextInputPartitioner = new LocalTextInputPartitioner(new Path(dinputDirectory), parallelismValue, config);
    DataSource<String, ?> source = new DataSource<>(config, localTextInputPartitioner, parallelismValue);
    InputSplit<String> inputSplit;
    for (int i = 0; i < parallelismValue; i++) {
        inputSplit = source.getNextSplit(i);
        Assert.assertNotNull(inputSplit);
    }
}