Examples with Path edu.iu.dsc.tws.api.data.Path used on opensource projects

Example 11 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class PointDataSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    if ("txt".equals(fileType)) {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteTextInputPartitioner(new Path(dataDirectory), context.getParallelism(), cfg));
        }
    } else {
        if ("points".equals(inputKey)) {
            this.source = runtime.createInput(cfg, context, new LocalCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        } else {
            this.source = runtime.createInput(cfg, context, new LocalCompleteCSVInputPartitioner(new Path(dataDirectory), context.getParallelism(), datasize, cfg));
        }
    }
}

Example 12 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class BufferedCollectionPartition method getConsumer.

@Override
public DataPartitionConsumer<T> getConsumer() {
    final Iterator<T> inMemoryIterator = this.dataList.iterator();
    final Iterator<Path> fileIterator = this.filesList.iterator();
    final Iterator<byte[]> buffersIterator = this.buffers.iterator();
    return new DataPartitionConsumer<T>() {

        private Queue<byte[]> bufferFromDisk = new LinkedList<>();

        @Override
        public boolean hasNext() {
            return inMemoryIterator.hasNext() || fileIterator.hasNext() || buffersIterator.hasNext() || !bufferFromDisk.isEmpty();
        }

        @Override
        public T next() {
            if (!this.bufferFromDisk.isEmpty()) {
                return (T) dataType.getDataPacker().unpackFromByteArray(this.bufferFromDisk.poll());
            } else if (inMemoryIterator.hasNext()) {
                return inMemoryIterator.next();
            } else if (fileIterator.hasNext()) {
                Path nextFile = fileIterator.next();
                try {
                    DataInputStream reader = new DataInputStream(fileSystem.open(nextFile));
                    long noOfFrames = reader.readLong();
                    for (long i = 0; i < noOfFrames; i++) {
                        int size = reader.readInt();
                        byte[] data = new byte[size];
                        int readSoFar = 0;
                        while (readSoFar < size) {
                            int readSize = reader.read(data, readSoFar, data.length - readSoFar);
                            if (readSize == -1) {
                                throw new Twister2RuntimeException("Reached the EOF unexpectedly");
                            }
                            readSoFar += readSize;
                        }
                        this.bufferFromDisk.add(data);
                    }
                    return next();
                } catch (IOException e) {
                    throw new Twister2RuntimeException("Failed to read value from the temp file : " + nextFile.toString(), e);
                }
            } else if (buffersIterator.hasNext()) {
                return (T) dataType.getDataPacker().unpackFromByteArray(buffersIterator.next());
            }
            throw new Twister2RuntimeException("No more frames available in this partition");
        }
    };
}

Example 13 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class BufferedCollectionPartition method get.

public T get(int index) {
    // read from memory
    if (index < this.dataList.size()) {
        return this.dataList.get(index);
    } else {
        // iterate over files
        long currentSize = this.dataList.size();
        for (int fileIndex = 0; fileIndex < this.filesList.size(); fileIndex++) {
            Path nextFile = this.filesList.get(fileIndex);
            try {
                DataInputStream reader = new DataInputStream(fileSystem.open(nextFile));
                long noOfFrames = reader.readLong();
                if (index < currentSize + noOfFrames) {
                    if (cachedFileIndex != fileIndex) {
                        cachedFileIndex = fileIndex;
                        this.currentFileCache = new ArrayList<>();
                        // read from this file
                        for (long i = 0; i < noOfFrames; i++) {
                            int size = reader.readInt();
                            byte[] data = new byte[size];
                            reader.read(data);
                            this.currentFileCache.add(data);
                        }
                    }
                    // not we have this file in cache
                    return (T) dataType.getDataPacker().unpackFromByteArray(this.currentFileCache.get((int) (index - this.dataList.size() - currentSize)));
                } else {
                    currentSize += noOfFrames;
                }
            } catch (IOException ioex) {
                throw new Twister2RuntimeException("Failed to read from file : " + nextFile);
            }
        }
        return (T) dataType.getDataPacker().unpackFromByteArray(this.buffers.get((int) (index - currentSize)));
    }
}

Example 14 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class BufferedCollectionPartition method clear.

@Override
public void clear() {
    // cleanup files
    for (Path path : this.filesList) {
        try {
            this.fileSystem.delete(path, true);
        } catch (IOException e) {
            throw new Twister2RuntimeException("Failed to delete the temporary file : " + path.toString(), e);
        }
    }
    super.clear();
    this.filesList.clear();
    this.buffers.clear();
    this.bufferedBytes = 0;
    this.fileCounter = 0;
}

Example 15 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class TestLocalFileSystem method main.

public static void main(String[] args) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/home/pulasthi/git/twister2/twister2/data/src/test" + "/resources/TextInputFormatTestFile.text");
    Config txtFileConf = builder.build();
    Path path = new Path("/home/pulasthi/git/twister2/twister2/data/src/test/resources" + "/TextInputFormatTestFile.text");
    InputPartitioner txtInput = new SharedTextInputPartitioner(path);
    txtInput.configure(txtFileConf);
    int minSplits = 8;
    try {
        InputSplit[] inputSplits = txtInput.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = txtInput.getInputSplitAssigner(inputSplits);
        InputSplit cur = inputSplitAssigner.getNextInputSplit(null, 0);
        cur.open();
        String line = "";
        line = (String) cur.nextRecord(line);
        System.out.println(line);
    } catch (Exception e) {
        e.printStackTrace();
    }
}