Examples with Path edu.iu.dsc.tws.api.data.Path used on opensource projects

Example 51 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class FileOutputWriter method write.

public void write(int partition, T out) {
    FSDataOutputStream fsOut;
    if (!openStreams.containsKey(partition)) {
        Path path = new Path(outPath, "part-" + partition);
        try {
            fsOut = fs.create(path);
            // lets ask user to create its own output method
            createOutput(partition, fsOut);
            openStreams.put(partition, fsOut);
        } catch (IOException e) {
            throw new RuntimeException("Failed to create output stream for file: " + path, e);
        }
    }
    writeRecord(partition, out);
}

Example 52 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class FileOutputWriter method write.

public void write(T out) {
    FSDataOutputStream fsOut;
    try {
        if (fs.exists(outPath)) {
            fs.delete(outPath, true);
        }
        fsOut = fs.create(new Path(outPath, generateRandom(10) + ".csv"));
        pw = new PrintWriter(fsOut);
    } catch (IOException e) {
        throw new RuntimeException("IOException Occured");
    }
    writeRecord(out);
}

Example 53 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class Twister2ArrowFileWriter method setUpTwister2ArrowWrite.

public boolean setUpTwister2ArrowWrite(int workerId) throws Exception {
    LOG.fine("%%%%%%%%% worker id details:" + workerId + "\t" + arrowFile);
    this.root = VectorSchemaRoot.create(Schema.fromJSON(arrowSchema), this.rootAllocator);
    Path path = new Path(arrowFile);
    this.fileSystem = FileSystemUtils.get(path);
    this.fsDataOutputStream = fileSystem.create(path);
    this.twister2ArrowOutputStream = new Twister2ArrowOutputStream(this.fsDataOutputStream);
    DictionaryProvider.MapDictionaryProvider provider = new DictionaryProvider.MapDictionaryProvider();
    if (!flag) {
        this.arrowFileWriter = new ArrowFileWriter(root, provider, this.fsDataOutputStream.getChannel());
    } else {
        this.arrowFileWriter = new ArrowFileWriter(root, provider, this.twister2ArrowOutputStream);
    }
    LOG.info("root schema fields:" + root.getSchema().getFields());
    for (Field field : root.getSchema().getFields()) {
        FieldVector vector = root.getVector(field.getName());
        if (vector.getMinorType().equals(Types.MinorType.INT)) {
            this.generatorMap.put(vector, new IntVectorGenerator());
        } else if (vector.getMinorType().equals(Types.MinorType.BIGINT)) {
            this.generatorMap.put(vector, new BigIntVectorGenerator());
        } else if (vector.getMinorType().equals(Types.MinorType.FLOAT4)) {
            this.generatorMap.put(vector, new FloatVectorGenerator());
        } else {
            throw new RuntimeException("unsupported arrow write type");
        }
    }
    return true;
}

Example 54 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class BufferedCollectionPartition method flush.

public void flush() {
    if (this.buffers.isEmpty()) {
        return;
    }
    Path filePath = new Path(this.rootPath, (this.fileCounter++) + EXTENSION);
    try (DataOutputStream outputStream = new DataOutputStream(this.fileSystem.create(filePath))) {
        outputStream.writeLong(this.buffers.size());
        Iterator<byte[]> bufferIt = this.buffers.iterator();
        while (bufferIt.hasNext()) {
            byte[] next = bufferIt.next();
            outputStream.writeInt(next.length);
            outputStream.write(next);
        }
    } catch (IOException e) {
        throw new Twister2RuntimeException("Couldn't flush partitions to the disk", e);
    }
    this.filesList.add(filePath);
    this.buffers.clear();
    this.bufferedBytes = 0;
}

Example 55 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class BufferedCollectionPartition method loadFromFS.

/**
 * This method loads existing frames on disk
 */
private void loadFromFS() {
    try {
        FileStatus[] fileStatuses = this.fileSystem.listFiles(this.rootPath);
        this.filesList = Arrays.stream(fileStatuses).map(FileStatus::getPath).filter(p -> p.getName().contains(EXTENSION)).sorted(Comparator.comparingLong(path -> Long.parseLong(path.getName().replace(EXTENSION, "")))).collect(Collectors.toList());
        this.fileCounter = fileStatuses.length;
    } catch (IOException e) {
        throw new Twister2RuntimeException("Failed to load frames from file system", e);
    }
}