Examples with InputPartitioner edu.iu.dsc.tws.data.api.InputPartitioner used on opensource projects

Example 1 with InputPartitioner

use of edu.iu.dsc.tws.data.api.InputPartitioner in project twister2 by DSC-SPIDAL.

the class TestLocalFileSystem method main.

public static void main(String[] args) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/home/pulasthi/git/twister2/twister2/data/src/test" + "/resources/TextInputFormatTestFile.text");
    Config txtFileConf = builder.build();
    Path path = new Path("/home/pulasthi/git/twister2/twister2/data/src/test/resources" + "/TextInputFormatTestFile.text");
    InputPartitioner txtInput = new SharedTextInputPartitioner(path);
    txtInput.configure(txtFileConf);
    int minSplits = 8;
    try {
        InputSplit[] inputSplits = txtInput.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = txtInput.getInputSplitAssigner(inputSplits);
        InputSplit cur = inputSplitAssigner.getNextInputSplit(null, 0);
        cur.open();
        String line = "";
        line = (String) cur.nextRecord(line);
        System.out.println(line);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 2 with InputPartitioner

use of edu.iu.dsc.tws.data.api.InputPartitioner in project twister2 by DSC-SPIDAL.

the class TestBinaryFileFormatter method main.

public static void main(String[] args) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/tmp/2000.bin");
    builder.put("RECORD_LENGTH", 1000 * Short.BYTES);
    Config txtFileConf = builder.build();
    Path path = new Path("/tmp/2000.bin");
    InputPartitioner binaryInputPartitioner = new BinaryInputPartitioner(path, 1000 * Short.BYTES);
    binaryInputPartitioner.configure(txtFileConf);
    int count = 0;
    int minSplits = 4;
    double expectedSum = 1.6375350724E1;
    double newSum = 0.0;
    Buffer buffer;
    try {
        InputSplit[] inputSplits = binaryInputPartitioner.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = binaryInputPartitioner.getInputSplitAssigner(inputSplits);
        InputSplit currentSplit;
        byte[] line = new byte[2000];
        ByteBuffer byteBuffer = ByteBuffer.allocate(2000);
        byteBuffer.order(ByteOrder.BIG_ENDIAN);
        while ((currentSplit = inputSplitAssigner.getNextInputSplit("localhost", 0)) != null) {
            currentSplit.open(txtFileConf);
            while (currentSplit.nextRecord(line) != null) {
                byteBuffer.clear();
                byteBuffer.put(line);
                byteBuffer.flip();
                buffer = byteBuffer.asShortBuffer();
                short[] shortArray = new short[1000];
                ((ShortBuffer) buffer).get(shortArray);
                for (short i : shortArray) {
                    newSum += i;
                    count++;
                }
            }
        }
        LOG.info("Sum and count values are:" + newSum + "\t" + count);
    } catch (Exception e) {
        e.printStackTrace();
    }
}