Search in sources :

Example 1 with InputSplit

use of edu.iu.dsc.tws.data.fs.io.InputSplit in project twister2 by DSC-SPIDAL.

the class TestBinaryFileFormatter method main.

public static void main(String[] args) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/home/pulasthi/git/twister2/twister2/" + "data/src/test/resources/2000.bin");
    Config txtFileConf = builder.build();
    Path path = new Path("/home/pulasthi/git/twister2/twister2/data/src/test/resources/2000.bin");
    InputFormat binaryInputFormatter = new BinaryInputFormatter(path, 2000 * Short.BYTES);
    binaryInputFormatter.configure(txtFileConf);
    int minSplits = 8;
    double expectedSum = 1.97973979E8;
    double newSum = 0.0;
    int count = 0;
    Buffer buffer = null;
    try {
        InputSplit[] inputSplits = binaryInputFormatter.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = binaryInputFormatter.getInputSplitAssigner(inputSplits);
        InputSplit currentSplit;
        byte[] line = new byte[4000];
        ByteBuffer byteBuffer = ByteBuffer.allocate(4000);
        byteBuffer.order(ByteOrder.LITTLE_ENDIAN);
        while ((currentSplit = inputSplitAssigner.getNextInputSplit(null, 0)) != null) {
            binaryInputFormatter.open(currentSplit);
            while (binaryInputFormatter.nextRecord(line) != null) {
                byteBuffer.clear();
                byteBuffer.put(line);
                byteBuffer.flip();
                buffer = byteBuffer.asShortBuffer();
                short[] shortArray = new short[2000];
                ((ShortBuffer) buffer).get(shortArray);
                for (short i : shortArray) {
                    newSum += i;
                    count++;
                }
            }
        }
        System.out.println(newSum);
        System.out.println(count);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : Path(edu.iu.dsc.tws.data.fs.Path) Buffer(java.nio.Buffer) ShortBuffer(java.nio.ShortBuffer) ByteBuffer(java.nio.ByteBuffer) InputSplitAssigner(edu.iu.dsc.tws.data.fs.io.InputSplitAssigner) Config(edu.iu.dsc.tws.common.config.Config) ByteBuffer(java.nio.ByteBuffer) BinaryInputFormatter(edu.iu.dsc.tws.data.api.formatters.BinaryInputFormatter) InputFormat(edu.iu.dsc.tws.data.api.InputFormat) InputSplit(edu.iu.dsc.tws.data.fs.io.InputSplit) ShortBuffer(java.nio.ShortBuffer)

Example 2 with InputSplit

use of edu.iu.dsc.tws.data.fs.io.InputSplit in project twister2 by DSC-SPIDAL.

the class TestLocalFileSystem method main.

public static void main(String[] args) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/home/pulasthi/git/twister2/twister2/data/src/test" + "/resources/TextInputFormatTestFile.text");
    Config txtFileConf = builder.build();
    Path path = new Path("/home/pulasthi/git/twister2/twister2/data/src/test/resources" + "/TextInputFormatTestFile.text");
    InputFormat txtInput = new TextInputFormatter(path);
    txtInput.configure(txtFileConf);
    int minSplits = 8;
    try {
        InputSplit[] inputSplits = txtInput.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = txtInput.getInputSplitAssigner(inputSplits);
        InputSplit cur = inputSplitAssigner.getNextInputSplit(null, 0);
        txtInput.open(cur);
        String line = "";
        line = (String) txtInput.nextRecord(line);
        System.out.println(line);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : Path(edu.iu.dsc.tws.data.fs.Path) TextInputFormatter(edu.iu.dsc.tws.data.api.formatters.TextInputFormatter) InputSplitAssigner(edu.iu.dsc.tws.data.fs.io.InputSplitAssigner) Config(edu.iu.dsc.tws.common.config.Config) InputFormat(edu.iu.dsc.tws.data.api.InputFormat) InputSplit(edu.iu.dsc.tws.data.fs.io.InputSplit)

Example 3 with InputSplit

use of edu.iu.dsc.tws.data.fs.io.InputSplit in project twister2 by DSC-SPIDAL.

the class WordCountExample method init.

@Override
public void init(Config config, int id, ResourcePlan resourcePlan) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/home/pulasthi/git/twister2/twister2" + "/data/src/test/resources/TextInputFormatTestFile.text");
    Config txtFileConf = builder.build();
    Path path = new Path("/home/pulasthi/git/twister2/twister2/data/src/test/" + "resources/TextInputFormatTestFile.text");
    @SuppressWarnings("unchecked") InputFormat txtInput = new TextInputFormatter(path);
    txtInput.configure(txtFileConf);
    int minSplits = 8;
    try {
        InputSplit[] inputSplits = txtInput.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = txtInput.getInputSplitAssigner(inputSplits);
        MapperSource mapperSource = new MapperSource(txtInput, inputSplitAssigner.getNextInputSplit(null, id));
        Mapper wordCountMapper = new Mapper();
        wordCountMapper.setInputSource(mapperSource);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : Path(edu.iu.dsc.tws.data.fs.Path) TextInputFormatter(edu.iu.dsc.tws.data.api.formatters.TextInputFormatter) InputSplitAssigner(edu.iu.dsc.tws.data.fs.io.InputSplitAssigner) Config(edu.iu.dsc.tws.common.config.Config) IOException(java.io.IOException) InputFormat(edu.iu.dsc.tws.data.api.InputFormat) InputSplit(edu.iu.dsc.tws.data.fs.io.InputSplit) FileInputSplit(edu.iu.dsc.tws.data.fs.FileInputSplit)

Aggregations

Config (edu.iu.dsc.tws.common.config.Config)3 InputFormat (edu.iu.dsc.tws.data.api.InputFormat)3 Path (edu.iu.dsc.tws.data.fs.Path)3 InputSplit (edu.iu.dsc.tws.data.fs.io.InputSplit)3 InputSplitAssigner (edu.iu.dsc.tws.data.fs.io.InputSplitAssigner)3 TextInputFormatter (edu.iu.dsc.tws.data.api.formatters.TextInputFormatter)2 BinaryInputFormatter (edu.iu.dsc.tws.data.api.formatters.BinaryInputFormatter)1 FileInputSplit (edu.iu.dsc.tws.data.fs.FileInputSplit)1 IOException (java.io.IOException)1 Buffer (java.nio.Buffer)1 ByteBuffer (java.nio.ByteBuffer)1 ShortBuffer (java.nio.ShortBuffer)1