Examples with InStreamOptions alluxio.client.file.options.InStreamOptions used on opensource projects

Example 1 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamLocal.

@Test
public void getInStreamLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    // Mock away gRPC usage.
    OpenLocalBlockResponse response = OpenLocalBlockResponse.newBuilder().setPath("/tmp").build();
    when(mWorkerClient.openLocalBlock(any(StreamObserver.class))).thenAnswer(invocation -> {
        mResponseObserver = invocation.getArgument(0, StreamObserver.class);
        return mStreamObserver;
    });
    doAnswer(invocation -> {
        mResponseObserver.onNext(response);
        mResponseObserver.onCompleted();
        return null;
    }).when(mStreamObserver).onNext(any(OpenLocalBlockRequest.class));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    assertEquals(local, mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)).getAddress());
}

Example 2 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamInAlluxioWhenCreateStreamIsFailed.

@Test
public void getInStreamInAlluxioWhenCreateStreamIsFailed() throws Exception {
    int workerCount = 5;
    boolean persisted = false;
    int[] blockLocations = new int[] { 2, 3, 4 };
    Map<Integer, Long> failedWorkers = ImmutableMap.of(0, 3L, 1, 1L, 3, 2L);
    int expectedWorker = 2;
    WorkerNetAddress[] workers = new WorkerNetAddress[workerCount];
    for (int i = 0; i < workers.length - 1; i++) {
        workers[i] = new WorkerNetAddress().setHost(String.format("worker-%d", i));
    }
    workers[workers.length - 1] = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    when(mContext.acquireBlockWorkerClient(WORKER_NET_ADDRESS_LOCAL)).thenThrow(new UnavailableException("failed to connect to " + WORKER_NET_ADDRESS_LOCAL.getHost()));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(persisted).setBlockIds(Collections.singletonList(BLOCK_ID)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
    when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg.getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next().getNetAddress());
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    options.setUfsReadLocationPolicy(mockPolicy);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers).map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList())));
    Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream().map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    BlockInStream inStream = null;
    int i = 2;
    while (i-- > 0) {
        try {
            inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses);
        } catch (Exception e) {
        // do nothing
        }
    }
    assertEquals(workers[expectedWorker], inStream.getAddress());
}

Example 3 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamMissingBlock.

@Test
public void getInStreamMissingBlock() throws Exception {
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(false).setBlockIds(Collections.singletonList(0L)));
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
    Exception e = assertThrows(UnavailableException.class, () -> mBlockStore.getInStream(BLOCK_ID, options).getAddress());
    assertTrue(e.getMessage().contains("unavailable in both Alluxio and UFS"));
}

Example 4 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamProcessLocal.

@Test
public void getInStreamProcessLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote), new BlockLocation().setWorkerAddress(local)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.hasProcessLocalWorker()).thenReturn(true);
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf));
    assertEquals(local, stream.getAddress());
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 5 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamUfsMockLocaltion.

@Test
public void getInStreamUfsMockLocaltion() throws Exception {
    try (Closeable c = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY, MockBlockLocationPolicy.class.getTypeName(), sConf).toResource()) {
        WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1");
        WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2");
        BlockInfo info = new BlockInfo().setBlockId(0);
        URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
        OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build();
        InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
        ((MockBlockLocationPolicy) options.getUfsReadLocationPolicy()).setHosts(Arrays.asList(worker1, worker2));
        when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
        when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(worker1, -1, -1), new BlockWorkerInfo(worker2, -1, -1)));
        // Location policy chooses worker1 first.
        assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
        // Location policy chooses worker2 second.
        assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
    }
}