Examples with Path edu.iu.dsc.tws.api.data.Path used on opensource projects

Example 16 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class TestBinaryFileFormatter method main.

public static void main(String[] args) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/tmp/2000.bin");
    builder.put("RECORD_LENGTH", 1000 * Short.BYTES);
    Config txtFileConf = builder.build();
    Path path = new Path("/tmp/2000.bin");
    InputPartitioner binaryInputPartitioner = new BinaryInputPartitioner(path, 1000 * Short.BYTES);
    binaryInputPartitioner.configure(txtFileConf);
    int count = 0;
    int minSplits = 4;
    double expectedSum = 1.6375350724E1;
    double newSum = 0.0;
    Buffer buffer;
    try {
        InputSplit[] inputSplits = binaryInputPartitioner.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = binaryInputPartitioner.getInputSplitAssigner(inputSplits);
        InputSplit currentSplit;
        byte[] line = new byte[2000];
        ByteBuffer byteBuffer = ByteBuffer.allocate(2000);
        byteBuffer.order(ByteOrder.BIG_ENDIAN);
        while ((currentSplit = inputSplitAssigner.getNextInputSplit("localhost", 0)) != null) {
            currentSplit.open(txtFileConf);
            while (currentSplit.nextRecord(line) != null) {
                byteBuffer.clear();
                byteBuffer.put(line);
                byteBuffer.flip();
                buffer = byteBuffer.asShortBuffer();
                short[] shortArray = new short[1000];
                ((ShortBuffer) buffer).get(shortArray);
                for (short i : shortArray) {
                    newSum += i;
                    count++;
                }
            }
        }
        LOG.info("Sum and count values are:" + newSum + "\t" + count);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 17 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class LocalFileSystem method open.

@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
    Path directoryPath = path;
    final File file = pathToFile(directoryPath, bufferSize);
    return new LocalDataInputStream(file);
}

Example 18 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class LocalFileSystem method create.

@Override
public FSDataOutputStream create(Path filePath) throws IOException {
    final Path parent = filePath.getParent();
    if (parent != null && !mkdirs(parent)) {
        throw new IOException("Mkdirs failed to create " + parent);
    }
    final File file = pathToFile(filePath);
    return new LocalDataOutputStream(file);
}

Example 19 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class DataNodeLocatorUtils method findDataNodesLocation.

/**
 * This method receives the input data list for each vertex and find the location of the
 * datanodes in the HDFS and returns the data node list.
 */
public List<String> findDataNodesLocation(List<String> inputFileList) {
    List<String> dataNodes = new ArrayList<>();
    FileSystem fileSystem;
    try {
        for (String anInputFileList : inputFileList) {
            Path path = new Path(anInputFileList);
            fileSystem = FileSystemUtils.get(path.toUri(), config);
            this.datasetName = anInputFileList;
            if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_HDFS_FILESYSTEM)) {
                FileStatus fileStatus = fileSystem.getFileStatus(new Path(datasetName));
                if (!fileStatus.getPath().isNullOrEmpty()) {
                    // dataNodes = getDataNodes(new String[]{this.datasetName});
                    dataNodes = getDataNodes();
                }
            } else if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_LOCAL_FILESYSTEM)) {
                FileStatus fileStatus = fileSystem.getFileStatus(new Path(datasetName));
                if (!fileStatus.getPath().isNullOrEmpty()) {
                    String datanodeName = InetAddress.getLocalHost().getHostName();
                    dataNodes.add(datanodeName);
                }
            }
        }
    } catch (IOException ioe) {
        throw new RuntimeException("IOException Occured");
    }
    return dataNodes;
}

Example 20 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class TextOutputWriter method createOutput.

public void createOutput() {
    try {
        if (fs.exists(path)) {
            fs.delete(path, true);
        }
        outputStream = fs.create(new Path(path, generateRandom(10) + ".txt"));
        pw = new PrintWriter(outputStream);
    } catch (IOException e) {
        throw new RuntimeException("IOException Occured");
    }
}