Examples with BlockWorkerInfo alluxio.client.block.BlockWorkerInfo used on opensource projects

Example 1 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class MostAvailableFirstPolicy method getWorkerForNextBlock.

@Override
public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
    long mostAvailableBytes = -1;
    WorkerNetAddress result = null;
    for (BlockWorkerInfo workerInfo : workerInfoList) {
        if (workerInfo.getCapacityBytes() - workerInfo.getUsedBytes() > mostAvailableBytes) {
            mostAvailableBytes = workerInfo.getCapacityBytes() - workerInfo.getUsedBytes();
            result = workerInfo.getNetAddress();
        }
    }
    return result;
}

Example 2 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class LocalFirstPolicy method getWorkerForNextBlock.

@Override
public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
    // first try the local host
    for (BlockWorkerInfo workerInfo : workerInfoList) {
        if (workerInfo.getNetAddress().getHost().equals(mLocalHostName) && workerInfo.getCapacityBytes() >= blockSizeBytes) {
            return workerInfo.getNetAddress();
        }
    }
    // otherwise randomly pick a worker that has enough availability
    List<BlockWorkerInfo> shuffledWorkers = Lists.newArrayList(workerInfoList);
    Collections.shuffle(shuffledWorkers);
    for (BlockWorkerInfo workerInfo : workerInfoList) {
        if (workerInfo.getCapacityBytes() >= blockSizeBytes) {
            return workerInfo.getNetAddress();
        }
    }
    return null;
}

Example 3 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class FileOutStreamTest method before.

/**
   * Sets up the different contexts and clients before a test runs.
   */
@Before
public void before() throws Exception {
    GroupMappingServiceTestUtils.resetCache();
    ClientTestUtils.setSmallBufferSizes();
    // PowerMock enums and final classes
    mFileSystemContext = PowerMockito.mock(FileSystemContext.class);
    mBlockStore = PowerMockito.mock(AlluxioBlockStore.class);
    mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class);
    mFactory = PowerMockito.mock(UnderFileSystemFileOutStream.Factory.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mFileSystemContext)).thenReturn(mBlockStore);
    when(mFileSystemContext.acquireMasterClientResource()).thenReturn(new DummyCloseableResource<>(mFileSystemMasterClient));
    when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class))).thenReturn(new URIStatus(new FileInfo()));
    // Worker file client mocking
    mWorkerClient = PowerMockito.mock(FileSystemWorkerClient.class);
    when(mFileSystemContext.createFileSystemWorkerClient()).thenReturn(mWorkerClient);
    when(mWorkerClient.createUfsFile(any(AlluxioURI.class), any(CreateUfsFileOptions.class))).thenReturn(UFS_FILE_ID);
    // Return sequentially increasing numbers for new block ids
    when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)).thenAnswer(new Answer<Long>() {

        private long mCount = 0;

        @Override
        public Long answer(InvocationOnMock invocation) throws Throwable {
            return mCount++;
        }
    });
    // Set up out streams. When they are created, add them to outStreamMap
    final Map<Long, TestBufferedBlockOutStream> outStreamMap = new HashMap<>();
    when(mBlockStore.getOutStream(anyLong(), eq(BLOCK_LENGTH), any(OutStreamOptions.class))).thenAnswer(new Answer<BufferedBlockOutStream>() {

        @Override
        public BufferedBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
            Long blockId = invocation.getArgumentAt(0, Long.class);
            if (!outStreamMap.containsKey(blockId)) {
                TestBufferedBlockOutStream newStream = new TestBufferedBlockOutStream(blockId, BLOCK_LENGTH, mFileSystemContext);
                outStreamMap.put(blockId, newStream);
            }
            return outStreamMap.get(blockId);
        }
    });
    BlockWorkerInfo workerInfo = new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost").setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0);
    when(mBlockStore.getWorkerInfoList()).thenReturn(Lists.newArrayList(workerInfo));
    mAlluxioOutStreamMap = outStreamMap;
    // Create an under storage stream so that we can check whether it has been flushed
    final AtomicBoolean underStorageFlushed = new AtomicBoolean(false);
    mUnderStorageOutputStream = new ByteArrayOutputStream() {

        @Override
        public void flush() {
            underStorageFlushed.set(true);
        }
    };
    mUnderStorageFlushed = underStorageFlushed;
    when(mFactory.create(any(FileSystemContext.class), any(InetSocketAddress.class), anyLong())).thenReturn(mUnderStorageOutputStream);
    // Set up underFileStorage so that we can test UnderStorageType.SYNC_PERSIST
    mUnderFileSystem = ClientMockUtils.mockUnderFileSystem();
    when(mUnderFileSystem.create(anyString())).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.create(anyString(), any(CreateOptions.class))).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.isDirectory(anyString())).thenReturn(true);
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.CACHE_THROUGH).setUfsPath(FILE_NAME.getPath());
    mTestStream = createTestStream(FILE_NAME, options);
}

Example 4 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicyTest method getLocalFirst.

/**
   * Tests that the local host is returned first.
   */
@Test
public void getLocalFirst() {
    String localhostName = NetworkAddressUtils.getLocalHostName();
    LocalFirstAvoidEvictionPolicy policy = new LocalFirstAvoidEvictionPolicy();
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName).setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    Assert.assertEquals(localhostName, policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost());
}

Example 5 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class LocalFirstPolicyTest method getLocalFirst.

/**
   * Tests that the local host is returned first.
   */
@Test
public void getLocalFirst() {
    String localhostName = NetworkAddressUtils.getLocalHostName();
    LocalFirstPolicy policy = new LocalFirstPolicy();
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName).setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    Assert.assertEquals(localhostName, policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost());
}