Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 46 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamUfsMockLocaltion.

@Test
public void getInStreamUfsMockLocaltion() throws Exception {
    try (Closeable c = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY, MockBlockLocationPolicy.class.getTypeName(), sConf).toResource()) {
        WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1");
        WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2");
        BlockInfo info = new BlockInfo().setBlockId(0);
        URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
        OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build();
        InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
        ((MockBlockLocationPolicy) options.getUfsReadLocationPolicy()).setHosts(Arrays.asList(worker1, worker2));
        when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
        when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(worker1, -1, -1), new BlockWorkerInfo(worker2, -1, -1)));
        // Location policy chooses worker1 first.
        assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
        // Location policy chooses worker2 second.
        assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
    }
}

Example 47 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method testGetInStreamFallback.

private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception {
    WorkerNetAddress[] workers = new WorkerNetAddress[workerCount];
    Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i)));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted).setBlockIds(Collections.singletonList(BLOCK_ID)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
    when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg.getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next().getNetAddress());
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    options.setUfsReadLocationPolicy(mockPolicy);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers).map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList())));
    Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream().map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses);
    assertEquals(workers[expectedWorker], inStream.getAddress());
}

Example 48 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class FileOutStreamTest method before.

/**
 * Sets up the different contexts and clients before a test runs.
 */
@Before
public void before() throws Exception {
    GroupMappingServiceTestUtils.resetCache();
    ClientTestUtils.setSmallBufferSizes(sConf);
    mClientContext = ClientContext.create(sConf);
    // PowerMock enums and final classes
    mFileSystemContext = PowerMockito.mock(FileSystemContext.class);
    when(mFileSystemContext.getClientContext()).thenReturn(mClientContext);
    when(mFileSystemContext.getClusterConf()).thenReturn(sConf);
    when(mFileSystemContext.getPathConf(any(AlluxioURI.class))).thenReturn(sConf);
    mBlockStore = PowerMockito.mock(AlluxioBlockStore.class);
    mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mFileSystemContext)).thenReturn(mBlockStore);
    when(mFileSystemContext.acquireMasterClientResource()).thenReturn(new DummyCloseableResource<>(mFileSystemMasterClient));
    when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class), any(GetStatusPOptions.class))).thenReturn(new URIStatus(new FileInfo()));
    // Return sequentially increasing numbers for new block ids
    when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)).thenAnswer(new Answer<Long>() {

        private long mCount = 0;

        @Override
        public Long answer(InvocationOnMock invocation) throws Throwable {
            return mCount++;
        }
    });
    // Set up out streams. When they are created, add them to outStreamMap
    final Map<Long, TestBlockOutStream> outStreamMap = new HashMap<>();
    when(mBlockStore.getOutStream(anyLong(), eq(BLOCK_LENGTH), any(OutStreamOptions.class))).thenAnswer(new Answer<TestBlockOutStream>() {

        @Override
        public TestBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
            Long blockId = invocation.getArgument(0, Long.class);
            if (!outStreamMap.containsKey(blockId)) {
                TestBlockOutStream newStream = new TestBlockOutStream(ByteBuffer.allocate(1000), BLOCK_LENGTH);
                outStreamMap.put(blockId, newStream);
            }
            return outStreamMap.get(blockId);
        }
    });
    BlockWorkerInfo workerInfo = new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost").setTieredIdentity(TieredIdentityFactory.fromString("node=localhost", sConf)).setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0);
    when(mFileSystemContext.getCachedWorkers()).thenReturn(Lists.newArrayList(workerInfo));
    mAlluxioOutStreamMap = outStreamMap;
    // Create an under storage stream so that we can check whether it has been flushed
    final AtomicBoolean underStorageFlushed = new AtomicBoolean(false);
    mUnderStorageOutputStream = new TestUnderFileSystemFileOutStream(ByteBuffer.allocate(5000)) {

        @Override
        public void flush() throws IOException {
            super.flush();
            underStorageFlushed.set(true);
        }
    };
    mUnderStorageFlushed = underStorageFlushed;
    PowerMockito.mockStatic(UnderFileSystemFileOutStream.class);
    PowerMockito.when(UnderFileSystemFileOutStream.create(any(FileSystemContext.class), any(WorkerNetAddress.class), any(OutStreamOptions.class))).thenReturn(mUnderStorageOutputStream);
    OutStreamOptions options = OutStreamOptions.defaults(mClientContext).setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.CACHE_THROUGH).setUfsPath(FILE_NAME.getPath());
    mTestStream = createTestStream(FILE_NAME, options);
}

Example 49 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class BlockInStreamTest method createShortCircuit.

@Test
public void createShortCircuit() throws Exception {
    WorkerNetAddress dataSource = new WorkerNetAddress();
    BlockInStream.BlockInStreamSource dataSourceType = BlockInStream.BlockInStreamSource.NODE_LOCAL;
    BlockInStream stream = BlockInStream.create(mMockContext, mInfo, dataSource, dataSourceType, mOptions);
    assertEquals(LocalFileDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 50 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class BufferCachingGrpcDataReaderTest method before.

@Before
public void before() throws Exception {
    WorkerNetAddress address = new WorkerNetAddress();
    BlockWorkerClient client = Mockito.mock(BlockWorkerClient.class);
    GrpcBlockingStream<ReadRequest, ReadResponse> unusedStream = new GrpcBlockingStream<>(client::readBlock, 5, "test message");
    ReadRequest readRequest = ReadRequest.newBuilder().setOffset(0).setLength(BLOCK_SIZE).setChunkSize(CHUNK_SIZE).setBlockId(1L).build();
    mDataReader = new TestBufferCachingGrpcDataReader(address, new NoopClosableResource<>(client), TIMEOUT, readRequest, unusedStream, CHUNK_SIZE, BLOCK_SIZE);
}