Examples with BlockInStream alluxio.client.block.stream.BlockInStream used on opensource projects

Example 1 with BlockInStream

use of alluxio.client.block.stream.BlockInStream in project alluxio by Alluxio.

the class RemoteReadIntegrationTest method readTest5.

/**
 * Tests the batch read API from a remote location when the data is only in an Alluxio worker.
 */
@Test
public void readTest5() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        URIStatus status = mFileSystem.getStatus(uri);
        InStreamOptions options = new InStreamOptions(status, ServerConfiguration.global());
        long blockId = status.getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create(FileSystemContext.create(ServerConfiguration.global())).getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        BlockInStream is = BlockInStream.create(mFsContext, options.getBlockInfo(blockId), workerAddr, BlockInStreamSource.REMOTE, options);
        byte[] ret = new byte[k];
        int read = is.read(ret);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(read, Arrays.copyOfRange(ret, 0, read)));
        is.close();
        FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uri, 100);
    }
}

Example 2 with BlockInStream

use of alluxio.client.block.stream.BlockInStream in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamInAlluxioWhenCreateStreamIsFailed.

@Test
public void getInStreamInAlluxioWhenCreateStreamIsFailed() throws Exception {
    int workerCount = 5;
    boolean persisted = false;
    int[] blockLocations = new int[] { 2, 3, 4 };
    Map<Integer, Long> failedWorkers = ImmutableMap.of(0, 3L, 1, 1L, 3, 2L);
    int expectedWorker = 2;
    WorkerNetAddress[] workers = new WorkerNetAddress[workerCount];
    for (int i = 0; i < workers.length - 1; i++) {
        workers[i] = new WorkerNetAddress().setHost(String.format("worker-%d", i));
    }
    workers[workers.length - 1] = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    when(mContext.acquireBlockWorkerClient(WORKER_NET_ADDRESS_LOCAL)).thenThrow(new UnavailableException("failed to connect to " + WORKER_NET_ADDRESS_LOCAL.getHost()));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(persisted).setBlockIds(Collections.singletonList(BLOCK_ID)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
    when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg.getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next().getNetAddress());
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    options.setUfsReadLocationPolicy(mockPolicy);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers).map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList())));
    Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream().map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    BlockInStream inStream = null;
    int i = 2;
    while (i-- > 0) {
        try {
            inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses);
        } catch (Exception e) {
        // do nothing
        }
    }
    assertEquals(workers[expectedWorker], inStream.getAddress());
}

Example 3 with BlockInStream

use of alluxio.client.block.stream.BlockInStream in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamProcessLocal.

@Test
public void getInStreamProcessLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.hasProcessLocalWorker()).thenReturn(true);
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf));
    assertEquals(local, stream.getAddress());
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 4 with BlockInStream

use of alluxio.client.block.stream.BlockInStream in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method testGetInStreamFallback.

private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception {
    WorkerNetAddress[] workers = new WorkerNetAddress[workerCount];
    Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i)));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted).setBlockIds(Collections.singletonList(BLOCK_ID)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
    when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg.getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next().getNetAddress());
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    options.setUfsReadLocationPolicy(mockPolicy);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers).map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList())));
    Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream().map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses);
    assertEquals(workers[expectedWorker], inStream.getAddress());
}

Example 5 with BlockInStream

use of alluxio.client.block.stream.BlockInStream in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method blockInStreamOutOfSync.

/**
 * Tests that when the underlying blocks are inconsistent with the metadata in terms of block
 * length, an exception is thrown rather than client hanging indefinitely. This case may happen if
 * the file in Alluxio and UFS is out of sync.
 */
@Test
public void blockInStreamOutOfSync() throws Exception {
    when(mBlockStore.getInStream(any(BlockInfo.class), any(InStreamOptions.class), any())).thenAnswer(new Answer<BlockInStream>() {

        @Override
        public BlockInStream answer(InvocationOnMock invocation) throws Throwable {
            return new TestBlockInStream(new byte[1], 0, BLOCK_LENGTH, false, mBlockSource);
        }
    });
    byte[] buffer = new byte[(int) BLOCK_LENGTH];
    try {
        mTestStream.read(buffer, 0, (int) BLOCK_LENGTH);
        fail("BlockInStream is inconsistent, an Exception is expected");
    } catch (IllegalStateException e) {
    // expect an exception to throw
    }
}