Examples with Path edu.iu.dsc.tws.data.fs.Path used on opensource projects

Example 1 with Path

use of edu.iu.dsc.tws.data.fs.Path in project twister2 by DSC-SPIDAL.

the class BasicMemoryManagerContainer method testPrimitivesBuffered.

/**
 * test primitives with Buffered memory manager
 */
public boolean testPrimitivesBuffered() {
    LOG.info("## Running BufferedMemoryManager primitives test ##");
    boolean allPassed = true;
    Path dataPath = new Path("/home/pulasthi/work/twister2/lmdbdatabase2");
    MemoryManager memoryManager = new BufferedMemoryManager(dataPath);
    int opID = 1;
    OperationMemoryManager op = memoryManager.addOperation(opID, DataMessageType.INTEGER);
    // Test single integer operation
    ByteBuffer key = ByteBuffer.allocateDirect(4);
    ByteBuffer value = ByteBuffer.allocateDirect(4);
    key.putInt(1);
    int testInt = 1231212121;
    byte[] val = Ints.toByteArray(testInt);
    value.put(val);
    op.put(key, value);
    ByteBuffer results = op.get(key);
    int res = results.getInt();
    if (res != testInt) {
        allPassed = false;
    }
    if (allPassed) {
        System.out.println("Passed BufferedMemoryManager int test");
    }
    // test int array, put should replace the current value
    int[] testarray = { 234, 14123, 534, 6345 };
    value = ByteBuffer.allocateDirect(16);
    ByteBuffer value2 = ByteBuffer.allocateDirect(16);
    ByteBuffer value3 = ByteBuffer.allocateDirect(16);
    ByteBuffer value4 = ByteBuffer.allocateDirect(16);
    for (int i : testarray) {
        value.putInt(i);
    }
    op.put(key, value);
    results = op.get(key);
    for (int i : testarray) {
        if (i != results.getInt()) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed BufferedMemoryManager int array test");
    }
    // get retuls with iterator
    Iterator<Object> iter = op.iterator();
    int[] dataset = (int[]) iter.next();
    for (int i = 0; i < dataset.length; i++) {
        if (dataset[i] != testarray[i]) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed BufferedMemoryManager int array iterator test," + " number of values returned by" + "iterator : " + 1);
    }
    op.delete(key);
    // iterator with more than 1 key will test that keys are sorted properly
    int[][] datamultiarray = { { 1, 11, 111, 1111 }, { 2, 22, 222, 2222 }, { 3, 33, 333, 3333 }, { 4, 44, 444, 4444 } };
    key.clear();
    value.clear();
    key.putInt(4);
    for (int i : datamultiarray[3]) {
        value.putInt(i);
    }
    op.put(key, value);
    key.clear();
    key.putInt(1);
    for (int i : datamultiarray[0]) {
        value2.putInt(i);
    }
    op.put(key, value2);
    key.clear();
    key.putInt(3);
    for (int i : datamultiarray[2]) {
        value3.putInt(i);
    }
    op.put(key, value3);
    key.clear();
    key.putInt(2);
    for (int i : datamultiarray[1]) {
        value4.putInt(i);
    }
    op.put(key, value4);
    Iterator<Object> itermulti = op.iterator();
    int itercount = 0;
    itercount = 0;
    while (itermulti.hasNext()) {
        if (itercount > 3) {
            break;
        }
        dataset = (int[]) itermulti.next();
        for (int i = 0; i < 4; i++) {
            if (dataset[i] != datamultiarray[itercount][i]) {
                allPassed = false;
            }
        }
        itercount++;
    }
    if (allPassed) {
        System.out.println("Passed BufferedMemoryManager int multi array iterator test," + " number of values returned by" + "iterator : " + itercount);
    }
    // test append function
    key.clear();
    value2.clear();
    key.putInt(6);
    for (int i : datamultiarray[0]) {
        value2.putInt(i);
    }
    op.put(key, value2);
    value3.clear();
    for (int i : datamultiarray[1]) {
        value3.putInt(i);
    }
    op.append(key, value3);
    results = op.get(key);
    for (int i : datamultiarray[0]) {
        if (i != results.getInt()) {
            allPassed = false;
        }
    }
    for (int i : datamultiarray[1]) {
        if (i != results.getInt()) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed BufferedMemoryManager int array append test");
    }
    return allPassed;
}

Example 2 with Path

use of edu.iu.dsc.tws.data.fs.Path in project twister2 by DSC-SPIDAL.

the class BasicMemoryManagerContainer method testPrimitivesLMDB.

/**
 * test primitives with LMDB memory manager
 */
public boolean testPrimitivesLMDB() {
    LOG.info("## Running LMDB primitives test ##");
    boolean allPassed = true;
    Path dataPath = new Path("/home/pulasthi/work/twister2/lmdbdatabase");
    MemoryManager memoryManager = new LMDBMemoryManager(dataPath);
    int opID = 1;
    OperationMemoryManager op = memoryManager.addOperation(opID, DataMessageType.INTEGER);
    // Test single integer operation
    ByteBuffer key = ByteBuffer.allocateDirect(4);
    ByteBuffer value = ByteBuffer.allocateDirect(4);
    key.putInt(1);
    int testInt = 1231212121;
    byte[] val = Ints.toByteArray(testInt);
    value.put(val);
    op.put(key, value);
    ByteBuffer results = op.get(key);
    int res = results.getInt();
    if (res != testInt) {
        allPassed = false;
    }
    if (allPassed) {
        System.out.println("Passed LMDB int test");
    }
    // test int array, put should replace the current value
    int[] testarray = { 234, 14123, 534, 6345 };
    value = ByteBuffer.allocateDirect(16);
    for (int i : testarray) {
        value.putInt(i);
    }
    op.put(key, value);
    results = op.get(key);
    for (int i : testarray) {
        if (i != results.getInt()) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed LMDB int array test");
    }
    // get retuls with iterator
    Iterator<Object> iter = op.iterator();
    int[] dataset = (int[]) iter.next();
    for (int i = 0; i < dataset.length; i++) {
        if (dataset[i] != testarray[i]) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed LMDB int array iterator test, number of values returned by" + "iterator : " + 1);
    }
    // iterator with more than 1 key will test that keys are sorted properly
    int[][] datamultiarray = { { 1, 11, 111, 1111 }, { 2, 22, 222, 2222 }, { 3, 33, 333, 3333 }, { 4, 44, 444, 4444 } };
    ByteBuffer value2 = ByteBuffer.allocateDirect(16);
    ByteBuffer value3 = ByteBuffer.allocateDirect(16);
    ByteBuffer value4 = ByteBuffer.allocateDirect(16);
    key.clear();
    value.clear();
    key.putInt(4);
    for (int i : datamultiarray[3]) {
        value.putInt(i);
    }
    op.put(key, value);
    key.clear();
    value.clear();
    key.putInt(1);
    for (int i : datamultiarray[0]) {
        value.putInt(i);
    }
    op.put(key, value);
    key.clear();
    value.clear();
    key.putInt(3);
    for (int i : datamultiarray[2]) {
        value.putInt(i);
    }
    op.put(key, value);
    key.clear();
    value.clear();
    key.putInt(2);
    for (int i : datamultiarray[1]) {
        value.putInt(i);
    }
    op.put(key, value);
    Iterator<Object> itermulti = op.iterator();
    int itercount = 0;
    itercount = 0;
    while (itermulti.hasNext()) {
        if (itercount > 3) {
            break;
        }
        dataset = (int[]) itermulti.next();
        for (int i = 0; i < 4; i++) {
            if (dataset[i] != datamultiarray[itercount][i]) {
                allPassed = false;
            }
        }
        itercount++;
    }
    if (allPassed) {
        System.out.println("Passed LMDB int multi array iterator test, number of values returned by" + "iterator : " + itercount);
    }
    // test append function
    key.clear();
    value.clear();
    key.putInt(6);
    for (int i : datamultiarray[0]) {
        value.putInt(i);
    }
    op.put(key, value);
    value.clear();
    for (int i : datamultiarray[1]) {
        value.putInt(i);
    }
    op.append(key, value);
    results = op.get(key);
    for (int i : datamultiarray[0]) {
        int itemp = results.getInt();
        if (i != itemp) {
            allPassed = false;
        }
    }
    for (int i : datamultiarray[1]) {
        int itemp = results.getInt();
        if (i != itemp) {
            allPassed = false;
        }
    }
    if (allPassed) {
        System.out.println("Passed LMDB int array append test");
    }
    return allPassed;
}

Example 3 with Path

use of edu.iu.dsc.tws.data.fs.Path in project twister2 by DSC-SPIDAL.

the class WordCountExample method init.

@Override
public void init(Config config, int id, ResourcePlan resourcePlan) {
    Config.Builder builder = new Config.Builder();
    builder.put("input.file.path", "/home/pulasthi/git/twister2/twister2" + "/data/src/test/resources/TextInputFormatTestFile.text");
    Config txtFileConf = builder.build();
    Path path = new Path("/home/pulasthi/git/twister2/twister2/data/src/test/" + "resources/TextInputFormatTestFile.text");
    @SuppressWarnings("unchecked") InputFormat txtInput = new TextInputFormatter(path);
    txtInput.configure(txtFileConf);
    int minSplits = 8;
    try {
        InputSplit[] inputSplits = txtInput.createInputSplits(minSplits);
        InputSplitAssigner inputSplitAssigner = txtInput.getInputSplitAssigner(inputSplits);
        MapperSource mapperSource = new MapperSource(txtInput, inputSplitAssigner.getNextInputSplit(null, id));
        Mapper wordCountMapper = new Mapper();
        wordCountMapper.setInputSource(mapperSource);
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 4 with Path

use of edu.iu.dsc.tws.data.fs.Path in project twister2 by DSC-SPIDAL.

the class FileInputFormat method createInputSplits.

/**
 * Computes the input splits for the file. By default, one file block is one split. If more splits
 * are requested than blocks are available, then a split may be a fraction of a
 * block and splits may cross block boundaries.
 *
 * @param minNumSplits The minimum desired number of file splits.
 * @return The computed file splits.
 */
@Override
public FileInputSplit[] createInputSplits(int minNumSplits) throws Exception {
    if (minNumSplits < 1) {
        throw new IllegalArgumentException("Number of input splits has to be at least 1.");
    }
    // take the desired number of splits into account
    int curminNumSplits = Math.max(minNumSplits, this.numSplits);
    final Path path = this.filePath;
    final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>(curminNumSplits);
    // get all the files that are involved in the splits
    List<FileStatus> files = new ArrayList<FileStatus>();
    long totalLength = 0;
    final FileSystem fs = path.getFileSystem();
    final FileStatus pathFile = fs.getFileStatus(path);
    if (pathFile.isDir()) {
        totalLength += sumFilesInDir(path, files, true);
    } else {
        // TODO L3: implement test for unsplittable
        // testForUnsplittable(pathFile);
        files.add(pathFile);
        totalLength += pathFile.getLen();
    }
    // TODO L3: Handle if unsplittable
    // TODO L1: check if we can add the i j method when making splits so that the last split is not
    // larger than the other splits
    final long maxSplitSize = totalLength / curminNumSplits + (totalLength % curminNumSplits == 0 ? 0 : 1);
    // Generate the splits
    int splitNum = 0;
    for (final FileStatus file : files) {
        final long len = file.getLen();
        final long blockSize = file.getBlockSize();
        final long localminSplitSize;
        if (this.minSplitSize <= blockSize) {
            localminSplitSize = this.minSplitSize;
        } else {
            LOG.log(Level.WARNING, "Minimal split size of " + this.minSplitSize + " is larger than the block size of " + blockSize + ". Decreasing minimal split size to block size.");
            localminSplitSize = blockSize;
        }
        final long splitSize = Math.max(localminSplitSize, Math.min(maxSplitSize, blockSize));
        final long halfSplit = splitSize >>> 1;
        final long maxBytesForLastSplit = (long) (splitSize * MAX_SPLIT_SIZE_DISCREPANCY);
        if (len > 0) {
            // get the block locations and make sure they are in order with respect to their offset
            final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len);
            Arrays.sort(blocks);
            long bytesUnassigned = len;
            long position = 0;
            int blockIndex = 0;
            while (bytesUnassigned > maxBytesForLastSplit) {
                // get the block containing the majority of the data
                blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex);
                // create a new split
                FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, splitSize, blocks[blockIndex].getHosts());
                inputSplits.add(fis);
                // adjust the positions
                position += splitSize;
                bytesUnassigned -= splitSize;
            }
            if (bytesUnassigned > 0) {
                blockIndex = getBlockIndexForPosition(blocks, position, halfSplit, blockIndex);
                final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), position, bytesUnassigned, blocks[blockIndex].getHosts());
                inputSplits.add(fis);
            }
        } else {
            // special case with a file of zero bytes size
            final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, 0);
            String[] hosts;
            if (blocks.length > 0) {
                hosts = blocks[0].getHosts();
            } else {
                hosts = new String[0];
            }
            final FileInputSplit fis = new FileInputSplit(splitNum++, file.getPath(), 0, 0, hosts);
            inputSplits.add(fis);
        }
    }
    return inputSplits.toArray(new FileInputSplit[inputSplits.size()]);
}

Example 5 with Path

use of edu.iu.dsc.tws.data.fs.Path in project twister2 by DSC-SPIDAL.

the class MPIDataFlowOperation method setStoreBased.

public void setStoreBased(boolean storeBased) {
    isStoreBased = storeBased;
    if (isStoreBased) {
        // TODO : need to load this from config file, both the type of memory manager and the datapath
        // TODO : need to make the memory manager available globally
        opertionID = (int) System.currentTimeMillis();
        this.kryoSerializer = new KryoSerializer();
        // Path dataPath = new Path("/scratch/pulasthi/terasort/lmdbdatabase_" + this.executor);
        Path dataPath = new Path(MPIContext.networkStoragePath(config) + " /lmdbdatabase_" + this.executor);
        this.memoryManager = new LMDBMemoryManager(dataPath);
        if (!isKeyed) {
            this.operationMemoryManager = memoryManager.addOperation(opertionID, MessageTypeUtils.toDataMessageType(type));
        } else {
            this.operationMemoryManager = memoryManager.addOperation(opertionID, MessageTypeUtils.toDataMessageType(type), MessageTypeUtils.toDataMessageType(keyType));
        }
    }
}