Examples with SharedTextInputPartitioner edu.iu.dsc.tws.data.api.formatters.SharedTextInputPartitioner used on opensource projects

Example 1 with SharedTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.SharedTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class TestLocalFileSystem method main.

public static void main(String[] args) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/home/pulasthi/git/twister2/twister2/data/src/test" + "/resources/TextInputFormatTestFile.text");
    Config txtFileConf = builder.build();
    Path path = new Path("/home/pulasthi/git/twister2/twister2/data/src/test/resources" + "/TextInputFormatTestFile.text");
    InputPartitioner txtInput = new SharedTextInputPartitioner(path);
    txtInput.configure(txtFileConf);
    int minSplits = 8;
    try {
        InputSplit[] inputSplits = txtInput.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = txtInput.getInputSplitAssigner(inputSplits);
        InputSplit cur = inputSplitAssigner.getNextInputSplit(null, 0);
        cur.open();
        String line = "";
        line = (String) cur.nextRecord(line);
        System.out.println(line);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 2 with SharedTextInputPartitioner

use of edu.iu.dsc.tws.data.api.formatters.SharedTextInputPartitioner in project twister2 by DSC-SPIDAL.

the class DataParallelTask method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    String directory = cfg.getStringValue(Constants.ARGS_INPUT_DIRECTORY);
    ExecutionRuntime runtime = (ExecutionRuntime) config.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    String outDir = cfg.getStringValue(Constants.ARGS_OUTPUT_DIRECTORY);
    boolean shared = cfg.getBooleanValue(Constants.ARGS_SHARED_FILE_SYSTEM);
    if (!shared) {
        this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(directory), context.getParallelism()));
    } else {
        this.source = runtime.createInput(cfg, context, new SharedTextInputPartitioner(new Path(directory)));
    }
    this.sink = new DataSink<String>(cfg, new TextOutputWriter(FileSystem.WriteMode.OVERWRITE, new Path(outDir)));
}