Examples with Path edu.iu.dsc.tws.api.data.Path used on opensource projects

Example 1 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class GraphDataSource method prepare.

@Override
public void prepare(Config cfg, TaskContext context) {
    super.prepare(cfg, context);
    ExecutionRuntime runtime = (ExecutionRuntime) cfg.get(ExecutorContext.TWISTER2_RUNTIME_OBJECT);
    this.source = runtime.createInput(cfg, context, new LocalFixedInputPartitioner(new Path(getDataDirectory()), context.getParallelism(), cfg, dsize));
}

Example 2 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class DataLocalityBatchTaskScheduler method getInputFilesList.

private List<String> getInputFilesList() {
    List<String> inputDataList = new ArrayList<>();
    String directory = null;
    if (config.get(DataObjectConstants.DINPUT_DIRECTORY) != null) {
        directory = String.valueOf(config.get(DataObjectConstants.DINPUT_DIRECTORY));
    }
    final Path path = new Path(directory);
    final FileSystem fileSystem;
    try {
        fileSystem = FileSystemUtils.get(path);
        if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_HDFS_FILESYSTEM)) {
            final FileStatus pathFile = fileSystem.getFileStatus(path);
            inputDataList.add(String.valueOf(pathFile.getPath()));
        } else if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_LOCAL_FILESYSTEM)) {
            for (FileStatus file : fileSystem.listFiles(path)) {
                String filename = String.valueOf(file.getPath());
                if (filename != null) {
                    inputDataList.add(filename);
                }
            }
        }
    } catch (IOException e) {
        throw new RuntimeException("IOException Occured");
    }
    return inputDataList;
}

Example 3 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class DataLocalityStreamingTaskScheduler method getInputFilesList.

private List<String> getInputFilesList() {
    List<String> inputDataList = new ArrayList<>();
    String directory = null;
    if (config.get(DataObjectConstants.DINPUT_DIRECTORY) != null) {
        directory = String.valueOf(config.get(DataObjectConstants.DINPUT_DIRECTORY));
    }
    final Path path = new Path(directory);
    final FileSystem fileSystem;
    try {
        fileSystem = FileSystemUtils.get(path);
        if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_HDFS_FILESYSTEM)) {
            final FileStatus pathFile = fileSystem.getFileStatus(path);
            inputDataList.add(String.valueOf(pathFile.getPath()));
        } else if (config.get(DataObjectConstants.FILE_SYSTEM).equals(DataContext.TWISTER2_LOCAL_FILESYSTEM)) {
            for (FileStatus file : fileSystem.listFiles(path)) {
                String filename = String.valueOf(file.getPath());
                if (filename != null) {
                    inputDataList.add(filename);
                }
            }
        }
    } catch (IOException e) {
        throw new TaskSchedulerException("Not able to get the input files", e);
    }
    return inputDataList;
}

Example 4 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class HDFSFileStateStore method put.

@Override
public void put(String key, byte[] data) throws IOException {
    Path pathForKey = this.getPathForKey(key);
    FSDataOutputStream hadoopDataOutputStream = this.hdfs.create(pathForKey);
    IOUtils.copyBytes(new ByteArrayInputStream(data), hadoopDataOutputStream, data.length, true);
}

Example 5 with Path

use of edu.iu.dsc.tws.api.data.Path in project twister2 by DSC-SPIDAL.

the class HDFSFileStateStore method init.

@Override
public void init(Config config, String... path) {
    String finalPath = HDFS_PROTO + String.join(File.separator, FileSystemContext.persistentStorageRoot(config), String.join(File.separator, path));
    this.parentPath = finalPath;
    try {
        this.hdfs = FileSystemUtils.get(URI.create(finalPath), config);
        this.hdfs.mkdirs(new Path(finalPath));
    } catch (IOException e) {
        throw new RuntimeException("Couldn't initial HDFS Store. " + "Failed to create the root directory, " + this.parentPath, e);
    }
}