Search in sources :

Example 21 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class RoundRobinPolicy method getWorkerForNextBlock.

/**
   * The policy uses the first fetch of worker info list as the base, and visits each of them in a
   * round-robin manner in the subsequent calls. The policy doesn't assume the list of worker info
   * in the subsequent calls has the same order from the first, and it will skip the workers that
   * are no longer active.
   *
   * @param workerInfoList the info of the active workers
   * @param blockSizeBytes the size of the block in bytes
   * @return the address of the worker to write to
   */
@Override
public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
    if (!mInitialized) {
        mWorkerInfoList = Lists.newArrayList(workerInfoList);
        Collections.shuffle(mWorkerInfoList);
        mIndex = 0;
        mInitialized = true;
    }
    // at most try all the workers
    for (int i = 0; i < mWorkerInfoList.size(); i++) {
        WorkerNetAddress candidate = mWorkerInfoList.get(mIndex).getNetAddress();
        BlockWorkerInfo workerInfo = findBlockWorkerInfo(workerInfoList, candidate);
        mIndex = (mIndex + 1) % mWorkerInfoList.size();
        if (workerInfo != null && workerInfo.getCapacityBytes() >= blockSizeBytes) {
            return candidate;
        }
    }
    return null;
}
Also used : WorkerNetAddress(alluxio.wire.WorkerNetAddress) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo)

Example 22 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class DeterministicHashPolicyTest method before.

@Before
public void before() {
    mWorkerInfos.clear();
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0));
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0));
    mWorkerInfos.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker4").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0));
}
Also used : WorkerNetAddress(alluxio.wire.WorkerNetAddress) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) Before(org.junit.Before)

Example 23 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicy method getWorkerForNextBlock.

@Override
public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
    // try the local host first
    WorkerNetAddress localWorkerNetAddress = null;
    for (BlockWorkerInfo workerInfo : workerInfoList) {
        if (workerInfo.getNetAddress().getHost().equals(mLocalHostName)) {
            localWorkerNetAddress = workerInfo.getNetAddress();
            if (getAvailableBytes(workerInfo) >= blockSizeBytes) {
                return localWorkerNetAddress;
            }
        }
    }
    // otherwise randomly pick a worker that has enough availability
    List<BlockWorkerInfo> shuffledWorkers = Lists.newArrayList(workerInfoList);
    Collections.shuffle(shuffledWorkers);
    for (BlockWorkerInfo workerInfo : shuffledWorkers) {
        if (getAvailableBytes(workerInfo) >= blockSizeBytes) {
            return workerInfo.getNetAddress();
        }
    }
    if (localWorkerNetAddress == null && shuffledWorkers.size() > 0) {
        return shuffledWorkers.get(0).getNetAddress();
    }
    return localWorkerNetAddress;
}
Also used : WorkerNetAddress(alluxio.wire.WorkerNetAddress) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo)

Example 24 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class RoundRobinPolicyTest method getWorker.

/**
   * Tests that the correct workers are chosen when round robin is used.
   */
@Test
public void getWorker() {
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0));
    RoundRobinPolicy policy = new RoundRobinPolicy();
    Assert.assertNotEquals(policy.getWorkerForNextBlock(workerInfoList, 2 * (long) Constants.GB).getHost(), policy.getWorkerForNextBlock(workerInfoList, 2 * (long) Constants.GB).getHost());
}
Also used : WorkerNetAddress(alluxio.wire.WorkerNetAddress) ArrayList(java.util.ArrayList) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) Test(org.junit.Test)

Example 25 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class SpecificHostPolicyTest method noMatchingHost.

/**
   * Tests that no worker is chosen when the worker specified in the policy is not part of the
   * worker list.
   */
@Test
public void noMatchingHost() {
    SpecificHostPolicy policy = new SpecificHostPolicy("worker3");
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    Assert.assertNull(policy.getWorkerForNextBlock(workerInfoList, Constants.MB));
}
Also used : WorkerNetAddress(alluxio.wire.WorkerNetAddress) ArrayList(java.util.ArrayList) BlockWorkerInfo(alluxio.client.block.BlockWorkerInfo) Test(org.junit.Test)

Aggregations

WorkerNetAddress (alluxio.wire.WorkerNetAddress)30 BlockWorkerInfo (alluxio.client.block.BlockWorkerInfo)15 Test (org.junit.Test)15 ArrayList (java.util.ArrayList)13 IOException (java.io.IOException)7 AlluxioURI (alluxio.AlluxioURI)6 AlluxioException (alluxio.exception.AlluxioException)5 BlockInfo (alluxio.wire.BlockInfo)5 RemoteBlockInStream (alluxio.client.block.RemoteBlockInStream)4 InetSocketAddress (java.net.InetSocketAddress)4 AlluxioBlockStore (alluxio.client.block.AlluxioBlockStore)2 OutStreamOptions (alluxio.client.file.options.OutStreamOptions)2 FileWriteLocationPolicy (alluxio.client.file.policy.FileWriteLocationPolicy)2 BlockLocation (alluxio.wire.BlockLocation)2 FileInfo (alluxio.wire.FileInfo)2 HashMap (java.util.HashMap)2 Before (org.junit.Before)2 BlockMasterClient (alluxio.client.block.BlockMasterClient)1 BufferedBlockOutStream (alluxio.client.block.BufferedBlockOutStream)1 TestBufferedBlockOutStream (alluxio.client.block.TestBufferedBlockOutStream)1