Examples with SeekableUnderFileInputStream alluxio.underfs.SeekableUnderFileInputStream used on opensource projects

Example 1 with SeekableUnderFileInputStream

use of alluxio.underfs.SeekableUnderFileInputStream in project alluxio by Alluxio.

the class HdfsUnderFileSystemTest method verifyPread.

/**
 * Tests the dynamic switching between pread and read calls to underlying stream.
 */
@Test
public void verifyPread() throws Exception {
    File file = mTemporaryFolder.newFile("test.txt");
    byte[] data = new byte[4 * MOVEMENT_LIMIT];
    for (int i = 0; i < 4 * MOVEMENT_LIMIT; i++) {
        data[i] = (byte) i;
    }
    FileUtils.writeByteArrayToFile(file, data);
    SeekableUnderFileInputStream in = (SeekableUnderFileInputStream) mHdfsUnderFileSystem.open(file.getAbsolutePath(), OpenOptions.defaults().setPositionShort(true));
    FSDataInputStream dataInput = Whitebox.getInternalState(in, "in");
    PreadSeekableStream stream = new PreadSeekableStream(dataInput);
    PreadSeekableStream spyStream = spy(stream);
    Whitebox.setInternalState(in, "in", spyStream);
    int readPos = 0;
    checkDataValid(in.read(), readPos++);
    checkDataValid(in.read(), readPos++);
    checkDataValid(in.read(), readPos++);
    checkDataValid(in.read(), readPos++);
    in.skip(2);
    in.skip(2);
    readPos += 4;
    checkDataValid(in.read(), readPos++);
    in.skip(2);
    readPos += 2;
    checkDataValid(in.read(), readPos++);
    // we are in sequential read mode, therefore all reads are normal reads and never preads
    verify(spyStream, never()).read(anyInt(), any(byte[].class), anyInt(), anyInt());
    verify(spyStream, times(6)).read(any(byte[].class), anyInt(), anyInt());
    in.skip(MOVEMENT_LIMIT + 1);
    readPos += MOVEMENT_LIMIT + 1;
    for (int i = 0; i < SEQUENTIAL_READ_LIMIT; i++) {
        checkDataValid(in.read(), readPos++);
    }
    // because we skipped over more than MOVEMENT_LIMIT, we switched to pread mode, the next
    // three reads are preads
    verify(spyStream, times(SEQUENTIAL_READ_LIMIT)).read(anyLong(), any(byte[].class), anyInt(), anyInt());
    verify(spyStream, times(6)).read(any(byte[].class), anyInt(), anyInt());
    // we performed more than SEQUENTIAL_READ_LIMIT reads without seeking beyond movement limit,
    // thus we switch back to sequential read mode
    checkDataValid(in.read(), readPos++);
    verify(spyStream, times(SEQUENTIAL_READ_LIMIT)).read(anyLong(), any(byte[].class), anyInt(), anyInt());
    verify(spyStream, times(7)).read(any(byte[].class), anyInt(), anyInt());
    in.seek(MOVEMENT_LIMIT * 3);
    readPos = MOVEMENT_LIMIT * 3;
    checkDataValid(in.read(), readPos++);
    // we performed seek to a far location, we should switch back to pread mode
    verify(spyStream, times(SEQUENTIAL_READ_LIMIT + 1)).read(anyLong(), any(byte[].class), anyInt(), anyInt());
    verify(spyStream, times(7)).read(any(byte[].class), anyInt(), anyInt());
    in.close();
}

Example 2 with SeekableUnderFileInputStream

use of alluxio.underfs.SeekableUnderFileInputStream in project alluxio by Alluxio.

the class UfsInputStreamCacheTest method acquireAndRelease.

@Test
public void acquireAndRelease() throws Exception {
    SeekableUnderFileInputStream mockedStream = mock(SeekableUnderFileInputStream.class);
    when(mUfs.openExistingFile(eq(FILE_NAME), any(OpenOptions.class))).thenReturn(mockedStream).thenThrow(new IllegalStateException("Should be called once"));
    // acquire a stream
    InputStream instream1 = mManager.acquire(mUfs, FILE_NAME, FILE_ID, OpenOptions.defaults().setOffset(2));
    // release
    mManager.release(instream1);
    // acquire a stream again
    InputStream instream2 = mManager.acquire(mUfs, FILE_NAME, FILE_ID, OpenOptions.defaults().setOffset(4));
    Assert.assertEquals(instream1, instream2);
    // ensure the second time the released instream is the same one but repositioned
    verify(mockedStream).seek(4);
}

Example 3 with SeekableUnderFileInputStream

use of alluxio.underfs.SeekableUnderFileInputStream in project alluxio by Alluxio.

the class UfsInputStreamCacheTest method notSeekable.

@Test
public void notSeekable() throws Exception {
    when(mUfs.isSeekable()).thenReturn(false);
    SeekableUnderFileInputStream mockedStream = mock(SeekableUnderFileInputStream.class);
    when(mUfs.openExistingFile(eq(FILE_NAME), any(OpenOptions.class))).thenReturn(mockedStream).thenThrow(new IllegalStateException("Should be called once"));
    // acquire a stream
    InputStream instream1 = mManager.acquire(mUfs, FILE_NAME, FILE_ID, OpenOptions.defaults().setOffset(2));
    // release
    mManager.release(instream1);
    // ensure the second time the released instream is the same one but repositioned
    verify(mockedStream).close();
}

Example 4 with SeekableUnderFileInputStream

use of alluxio.underfs.SeekableUnderFileInputStream in project alluxio by Alluxio.

the class UfsInputStreamCacheTest method before.

@Before
public void before() throws Exception {
    mSeekableInStreams = new SeekableUnderFileInputStream[mNumOfInputStreams];
    mUfs = mock(UnderFileSystem.class);
    when(mUfs.isSeekable()).thenReturn(true);
    for (int i = 0; i < mNumOfInputStreams; i++) {
        SeekableUnderFileInputStream instream = mock(SeekableUnderFileInputStream.class);
        mSeekableInStreams[i] = instream;
    }
    when(mUfs.openExistingFile(eq(FILE_NAME), any(OpenOptions.class))).thenReturn(mSeekableInStreams[0], Arrays.copyOfRange(mSeekableInStreams, 1, mNumOfInputStreams));
    mManager = new UfsInputStreamCache();
}