Examples with FileInStream alluxio.client.file.FileInStream used on opensource projects

Example 91 with FileInStream

use of alluxio.client.file.FileInStream in project alluxio by Alluxio.

the class RemoteBlockInStreamIntegrationTest method seekAroundLocalBlock.

/**
   * Tests that seeking around a file cached locally works.
   */
@Test
public void seekAroundLocalBlock() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    // The number of bytes per remote block read should be set to 100 in the before function
    FileSystemTestUtils.createByteFile(mFileSystem, uniqPath, 200, mWriteAlluxio);
    FileInStream is = mFileSystem.openFile(new AlluxioURI(uniqPath), mReadNoCache);
    Assert.assertEquals(0, is.read());
    is.seek(199);
    Assert.assertEquals(199, is.read());
    is.seek(99);
    Assert.assertEquals(99, is.read());
    is.close();
}

Example 92 with FileInStream

use of alluxio.client.file.FileInStream in project alluxio by Alluxio.

the class UnderFileSystemBlockInStreamIntegrationTest method skip.

/**
   * Tests {@link UnderFileSystemBlockInStream#skip(long)}.
   */
@Test
public void skip() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteUnderStore, k);
        FileInStream is = mFileSystem.openFile(uri, mReadCache);
        Assert.assertEquals(k / 2, is.skip(k / 2));
        Assert.assertEquals(k / 2, is.read());
        is.close();
        Assert.assertEquals(100, mFileSystem.getStatus(uri).getInMemoryPercentage());
        if (k >= 3) {
            is = mFileSystem.openFile(uri, mReadCache);
            int t = k / 3;
            Assert.assertEquals(t, is.skip(t));
            Assert.assertEquals(t, is.read());
            Assert.assertEquals(t, is.skip(t));
            Assert.assertEquals(2 * t + 1, is.read());
            is.close();
            Assert.assertTrue(mFileSystem.getStatus(uri).getInMemoryPercentage() == 100);
        }
    }
}

Example 93 with FileInStream

use of alluxio.client.file.FileInStream in project alluxio by Alluxio.

the class UnderFileSystemBlockInStreamIntegrationTest method read.

/**
   * Tests {@link UnderFileSystemBlockInStream#read()}. Read from underfs.
   */
@Test
public void read() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteUnderStore, k);
        FileInStream is = mFileSystem.openFile(uri, mReadNoCache);
        byte[] ret = new byte[k];
        int value = is.read();
        int cnt = 0;
        while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
        }
        Assert.assertEquals(cnt, k);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
        if (k == 0) {
            Assert.assertEquals(100, mFileSystem.getStatus(uri).getInMemoryPercentage());
        } else {
            Assert.assertNotEquals(100, mFileSystem.getStatus(uri).getInMemoryPercentage());
        }
        is = mFileSystem.openFile(uri, mReadCache);
        ret = new byte[k];
        value = is.read();
        cnt = 0;
        while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
        }
        Assert.assertEquals(cnt, k);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
        Assert.assertEquals(100, mFileSystem.getStatus(uri).getInMemoryPercentage());
        is = mFileSystem.openFile(uri, mReadCache);
        ret = new byte[k];
        value = is.read();
        cnt = 0;
        while (value != -1) {
            Assert.assertTrue(value >= 0);
            Assert.assertTrue(value < 256);
            ret[cnt++] = (byte) value;
            value = is.read();
        }
        Assert.assertEquals(cnt, k);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
        is.close();
        Assert.assertEquals(100, mFileSystem.getStatus(uri).getInMemoryPercentage());
    }
}

Example 94 with FileInStream

use of alluxio.client.file.FileInStream in project alluxio by Alluxio.

the class UnderFileSystemBlockInStreamIntegrationTest method concurrentUfsRead.

/**
   * Tests {@link UnderFileSystemBlockInStream#read()} concurrency. Read from underfs.
   */
@Test
public void concurrentUfsRead() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    final AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + MAX_LEN);
    FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteUnderStore, MAX_LEN);
    ExecutorService executorService = Executors.newFixedThreadPool(100);
    final AtomicInteger count = new AtomicInteger(0);
    final Random random = new Random();
    int expectedCount = 100;
    for (int i = 0; i < expectedCount; i++) {
        final int index = i;
        executorService.submit(new Runnable() {

            @Override
            public void run() {
                try {
                    // Add some randomness here to avoid opening too many connections too fast.
                    Thread.sleep(random.nextInt(100));
                    FileInStream is = mFileSystem.openFile(uri, mReadCache);
                    // Sleep here so that we can make sure the file is not cached too fast.
                    Thread.sleep(100);
                    byte[] ret = new byte[MAX_LEN];
                    int value = is.read();
                    int cnt = 0;
                    while (value != -1) {
                        Assert.assertTrue(value >= 0);
                        Assert.assertTrue(value < 256);
                        ret[cnt++] = (byte) value;
                        value = is.read();
                    }
                    is.close();
                    Assert.assertEquals(cnt, MAX_LEN);
                    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MAX_LEN, ret));
                    while (mFileSystem.getStatus(uri).getInMemoryPercentage() < 100) {
                        Thread.sleep(1000);
                    }
                    Assert.assertEquals(100, mFileSystem.getStatus(uri).getInMemoryPercentage());
                    count.incrementAndGet();
                } catch (Throwable e) {
                    LOG.error("Failed to read file {}.", index, e);
                }
            }
        });
    }
    executorService.shutdown();
    executorService.awaitTermination(Constants.MINUTE_MS * 5, TimeUnit.MILLISECONDS);
    // There can be some errors due because SASL handsake failure or opening the connections too
    // fast. If that happens, consider loosing the condition a bit.
    Assert.assertTrue(expectedCount >= count.get() - 1);
}

Example 95 with FileInStream

use of alluxio.client.file.FileInStream in project alluxio by Alluxio.

the class UnderFileSystemBlockInStreamIntegrationTest method readMultiBlockFile.

/**
   * Tests that reading a file consisting of more than one block from the underfs works.
   */
@Test
public void readMultiBlockFile() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    int blockSizeByte = 10;
    int numBlocks = 10;
    AlluxioURI uri = new AlluxioURI(uniqPath);
    FileOutStream os = mFileSystem.createFile(uri, mWriteUnderStore);
    for (int i = 0; i < numBlocks; i++) {
        for (int j = 0; j < blockSizeByte; j++) {
            os.write((byte) (i * blockSizeByte + j));
        }
    }
    os.close();
    FileInStream is = mFileSystem.openFile(uri, mReadCache);
    for (int i = 0; i < blockSizeByte * numBlocks; i++) {
        Assert.assertEquals((byte) i, is.read());
    }
    is.close();
    Assert.assertTrue(mFileSystem.getStatus(uri).getInMemoryPercentage() == 100);
}