Examples with WorkerNetAddress alluxio.wire.WorkerNetAddress used on opensource projects

Example 66 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class LocalFirstAvoidEvictionPolicyTest method getLocalWhenNoneHasSpace.

/**
   * Tests that local host is picked if none of the workers has enough availability.
   */
@Test
public void getLocalWhenNoneHasSpace() {
    String localhostName = NetworkAddressUtils.getLocalHostName();
    LocalFirstAvoidEvictionPolicy policy = new LocalFirstAvoidEvictionPolicy();
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, Constants.MB));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName).setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, Constants.MB));
    Assert.assertEquals(localhostName, policy.getWorkerForNextBlock(workerInfoList, Constants.GB).getHost());
}

Example 67 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class LocalFirstPolicyTest method getOthersWhenNotEnoughSpaceOnLocal.

/**
   * Tests that another worker is picked in case the local host does not have enough space.
   */
@Test
public void getOthersWhenNotEnoughSpaceOnLocal() {
    String localhostName = NetworkAddressUtils.getLocalHostName();
    LocalFirstPolicy policy = new LocalFirstPolicy();
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost(localhostName).setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.MB, Constants.MB));
    Assert.assertEquals("worker1", policy.getWorkerForNextBlock(workerInfoList, Constants.GB).getHost());
}

Example 68 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class MostAvailableFirstPolicyTest method getMostAvailableWorker.

/**
   * Tests that the worker with the most available space is chosen.
   */
@Test
public void getMostAvailableWorker() {
    List<BlockWorkerInfo> workerInfoList = new ArrayList<>();
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker1").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker2").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 2 * (long) Constants.GB, 0));
    workerInfoList.add(new BlockWorkerInfo(new WorkerNetAddress().setHost("worker3").setRpcPort(PORT).setDataPort(PORT).setWebPort(PORT), 3 * (long) Constants.GB, 0));
    MostAvailableFirstPolicy policy = new MostAvailableFirstPolicy();
    Assert.assertEquals("worker3", policy.getWorkerForNextBlock(workerInfoList, Constants.MB).getHost());
}

Example 69 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class ReplicationCheckerTest method heartbeatFileUnderReplicatedAndLost.

@Test
public void heartbeatFileUnderReplicatedAndLost() throws Exception {
    mFileContext.getOptions().setReplicationMin(2);
    long blockId = createBlockHelper(TEST_FILE_1, mFileContext, "");
    // Create a worker.
    long workerId = mBlockMaster.getWorkerId(new WorkerNetAddress().setHost("localhost").setRpcPort(80).setDataPort(81).setWebPort(82));
    mBlockMaster.workerRegister(workerId, Collections.singletonList(Constants.MEDIUM_MEM), ImmutableMap.of(Constants.MEDIUM_MEM, 100L), ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
    mBlockMaster.commitBlock(workerId, 50L, Constants.MEDIUM_MEM, Constants.MEDIUM_MEM, blockId, 20L);
    // Indicate that blockId is removed on the worker.
    mBlockMaster.workerHeartbeat(workerId, null, ImmutableMap.of(Constants.MEDIUM_MEM, 0L), ImmutableList.of(blockId), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, NO_METRICS);
    mReplicationChecker.heartbeat();
    Assert.assertEquals(EMPTY, mMockReplicationHandler.getEvictRequests());
    Assert.assertEquals(EMPTY, mMockReplicationHandler.getReplicateRequests());
}

Example 70 with WorkerNetAddress

use of alluxio.wire.WorkerNetAddress in project alluxio by Alluxio.

the class LoadCommand method runLoadTask.

private void runLoadTask(AlluxioURI filePath, URIStatus status, boolean local) throws IOException {
    AlluxioConfiguration conf = mFsContext.getPathConf(filePath);
    OpenFilePOptions options = FileSystemOptions.openFileDefaults(conf);
    BlockLocationPolicy policy = Preconditions.checkNotNull(BlockLocationPolicy.Factory.create(conf.getString(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY), conf), "UFS read location policy Required when loading files");
    WorkerNetAddress dataSource;
    List<Long> blockIds = status.getBlockIds();
    for (long blockId : blockIds) {
        if (local) {
            dataSource = mFsContext.getNodeLocalWorker();
        } else {
            // send request to data source
            AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext);
            Pair<WorkerNetAddress, BlockInStream.BlockInStreamSource> dataSourceAndType = blockStore.getDataSourceAndType(status.getBlockInfo(blockId), status, policy, ImmutableMap.of());
            dataSource = dataSourceAndType.getFirst();
        }
        Protocol.OpenUfsBlockOptions openUfsBlockOptions = new InStreamOptions(status, options, conf).getOpenUfsBlockOptions(blockId);
        cacheBlock(blockId, dataSource, status, openUfsBlockOptions);
    }
}