Examples with BlockWorkerInfo alluxio.client.block.BlockWorkerInfo used on opensource projects

Example 6 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class DeterministicHashPolicy method getWorker.

@Override
public WorkerNetAddress getWorker(GetWorkerOptions options) {
    List<BlockWorkerInfo> workerInfos = Lists.newArrayList(options.getBlockWorkerInfos());
    Collections.sort(workerInfos, new Comparator<BlockWorkerInfo>() {

        @Override
        public int compare(BlockWorkerInfo o1, BlockWorkerInfo o2) {
            return o1.getNetAddress().toString().compareToIgnoreCase(o2.getNetAddress().toString());
        }
    });
    HashMap<WorkerNetAddress, BlockWorkerInfo> blockWorkerInfoMap = new HashMap<>();
    for (BlockWorkerInfo workerInfo : options.getBlockWorkerInfos()) {
        blockWorkerInfoMap.put(workerInfo.getNetAddress(), workerInfo);
    }
    List<WorkerNetAddress> workers = new ArrayList<>();
    // Try the next one if the worker mapped from the blockId doesn't work until all the workers
    // are examined.
    int index = (int) (options.getBlockId() % (long) workerInfos.size());
    for (BlockWorkerInfo blockWorkerInfoUnused : workerInfos) {
        WorkerNetAddress candidate = workerInfos.get(index).getNetAddress();
        BlockWorkerInfo workerInfo = blockWorkerInfoMap.get(candidate);
        if (workerInfo != null && workerInfo.getCapacityBytes() >= options.getBlockSize()) {
            workers.add(candidate);
            if (workers.size() >= mShards) {
                break;
            }
        }
        index = (index + 1) % workerInfos.size();
    }
    return workers.isEmpty() ? null : workers.get(mRandom.nextInt(workers.size()));
}

Example 7 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class RoundRobinPolicy method getWorkerForNextBlock.

/**
   * The policy uses the first fetch of worker info list as the base, and visits each of them in a
   * round-robin manner in the subsequent calls. The policy doesn't assume the list of worker info
   * in the subsequent calls has the same order from the first, and it will skip the workers that
   * are no longer active.
   *
   * @param workerInfoList the info of the active workers
   * @param blockSizeBytes the size of the block in bytes
   * @return the address of the worker to write to
   */
@Override
public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
    if (!mInitialized) {
        mWorkerInfoList = Lists.newArrayList(workerInfoList);
        Collections.shuffle(mWorkerInfoList);
        mIndex = 0;
        mInitialized = true;
    }
    // at most try all the workers
    for (int i = 0; i < mWorkerInfoList.size(); i++) {
        WorkerNetAddress candidate = mWorkerInfoList.get(mIndex).getNetAddress();
        BlockWorkerInfo workerInfo = findBlockWorkerInfo(workerInfoList, candidate);
        mIndex = (mIndex + 1) % mWorkerInfoList.size();
        if (workerInfo != null && workerInfo.getCapacityBytes() >= blockSizeBytes) {
            return candidate;
        }
    }
    return null;
}

Example 8 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class DeterministicHashPolicyTest method before.

@Before
public void before() {
    mWorkerInfos.clear();
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0));
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0));
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker4").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0));
}

Example 9 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class FileInStreamTest method before.

/**
   * Sets up the context and streams before a test runs.
   *
   * @throws AlluxioException when the worker ufs operations fail
   * @throws IOException when the read and write streams fail
   */
@Before
public void before() throws Exception {
    mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(FILE_LENGTH);
    mDelegateUfsOps = Configuration.getBoolean(PropertyKey.USER_UFS_DELEGATION_ENABLED);
    ClientTestUtils.setSmallBufferSizes();
    mContext = PowerMockito.mock(FileSystemContext.class);
    mBlockStore = Mockito.mock(AlluxioBlockStore.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mContext)).thenReturn(mBlockStore);
    PowerMockito.when(mBlockStore.getWorkerInfoList()).thenReturn(new ArrayList<BlockWorkerInfo>());
    Mockito.mock(StreamFactory.class);
    PowerMockito.mockStatic(StreamFactory.class);
    // Set up BufferedBlockInStreams and caching streams
    mCacheStreams = new ArrayList<>();
    List<Long> blockIds = new ArrayList<>();
    for (int i = 0; i < NUM_STREAMS; i++) {
        blockIds.add((long) i);
        mCacheStreams.add(new TestBufferedBlockOutStream(i, getBlockLength(i), mContext));
        Mockito.when(mBlockStore.getInStream(Mockito.eq((long) i), Mockito.any(InStreamOptions.class))).thenAnswer(new Answer<BufferedBlockInStream>() {

            @Override
            public BufferedBlockInStream answer(InvocationOnMock invocation) throws Throwable {
                long i = (Long) invocation.getArguments()[0];
                byte[] input = BufferUtils.getIncreasingByteArray((int) (i * BLOCK_LENGTH), (int) getBlockLength((int) i));
                return new TestBufferedBlockInStream(i, input);
            }
        });
        Mockito.when(mBlockStore.getOutStream(Mockito.eq((long) i), Mockito.anyLong(), Mockito.any(WorkerNetAddress.class), Mockito.any(OutStreamOptions.class))).thenAnswer(new Answer<BufferedBlockOutStream>() {

            @Override
            public BufferedBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
                long i = (Long) invocation.getArguments()[0];
                return mCacheStreams.get((int) i).isClosed() ? null : mCacheStreams.get((int) i);
            }
        });
    }
    mInfo.setBlockIds(blockIds);
    mStatus = new URIStatus(mInfo);
    mTestStream = new FileInStream(mStatus, InStreamOptions.defaults().setReadType(ReadType.CACHE_PROMOTE).setCachePartiallyReadBlock(false), mContext);
}

Example 10 with BlockWorkerInfo

use of alluxio.client.block.BlockWorkerInfo in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicy method getWorkerForNextBlock.

@Override
public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
    // try the local host first
    WorkerNetAddress localWorkerNetAddress = null;
    for (BlockWorkerInfo workerInfo : workerInfoList) {
        if (workerInfo.getNetAddress().getHost().equals(mLocalHostName)) {
            localWorkerNetAddress = workerInfo.getNetAddress();
            if (getAvailableBytes(workerInfo) >= blockSizeBytes) {
                return localWorkerNetAddress;
            }
        }
    }
    // otherwise randomly pick a worker that has enough availability
    List<BlockWorkerInfo> shuffledWorkers = Lists.newArrayList(workerInfoList);
    Collections.shuffle(shuffledWorkers);
    for (BlockWorkerInfo workerInfo : shuffledWorkers) {
        if (getAvailableBytes(workerInfo) >= blockSizeBytes) {
            return workerInfo.getNetAddress();
        }
    }
    if (localWorkerNetAddress == null && shuffledWorkers.size() > 0) {
        return shuffledWorkers.get(0).getNetAddress();
    }
    return localWorkerNetAddress;
}