Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 1 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class WebInterfaceWorkersServlet method generateOrderedNodeInfos.

/**
   * Order the nodes by hostName and generate {@link NodeInfo} list for UI display.
   *
   * @param workerInfos the list of {@link WorkerInfo} objects
   * @return the list of {@link NodeInfo} objects
   */
private NodeInfo[] generateOrderedNodeInfos(Collection<WorkerInfo> workerInfos) {
    NodeInfo[] ret = new NodeInfo[workerInfos.size()];
    int index = 0;
    for (WorkerInfo workerInfo : workerInfos) {
        ret[index++] = new NodeInfo(workerInfo);
    }
    Arrays.sort(ret);
    return ret;
}

Example 2 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class AlluxioMasterRestServiceHandler method getWebUIWorkers.

/**
 * Gets Web UI workers page data.
 *
 * @return the response object
 */
@GET
@Path(WEBUI_WORKERS)
public Response getWebUIWorkers() {
    return RestUtils.call(() -> {
        MasterWebUIWorkers response = new MasterWebUIWorkers();
        response.setDebug(ServerConfiguration.getBoolean(PropertyKey.DEBUG));
        List<WorkerInfo> workerInfos = mBlockMaster.getWorkerInfoList();
        NodeInfo[] normalNodeInfos = WebUtils.generateOrderedNodeInfos(workerInfos);
        response.setNormalNodeInfos(normalNodeInfos);
        List<WorkerInfo> lostWorkerInfos = mBlockMaster.getLostWorkersInfoList();
        NodeInfo[] failedNodeInfos = WebUtils.generateOrderedNodeInfos(lostWorkerInfos);
        response.setFailedNodeInfos(failedNodeInfos);
        return response;
    }, ServerConfiguration.global());
}

Example 3 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithDifferentWorkerHeartbeatDifferentBlock.

@Test
public void concurrentCommitWithDifferentWorkerHeartbeatDifferentBlock() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    long worker2 = registerEmptyWorker(NET_ADDRESS_2);
    // Register with block 2 on both workers
    mBlockMaster.commitBlock(worker1, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
    mBlockMaster.commitBlock(worker2, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
    CountDownLatch w1Latch = new CountDownLatch(1);
    mBlockMaster.setLatch(w1Latch);
    concurrentWriterWithWriter(w1Latch, // W1
    () -> {
        // worker 1 has block 1 and block 2 now
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH + BLOCK2_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // W2
    () -> {
        // A different block is removed on the same worker
        // This should contend on the worker metadata
        Command cmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // 0 used because the block is already removed
        MEM_USAGE_EMPTY, // list of removed blockIds
        ImmutableList.of(BLOCK2_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
        // The block has been removed, nothing from command
        assertEquals(EMPTY_CMD, cmd);
        return null;
    }, // Verifier
    () -> {
        // After heartbeat, verify the worker info
        List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
        assertEquals(2, workerInfoList.size());
        WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
        assertEquals(BLOCK1_LENGTH + BLOCK2_LENGTH, worker1Info.getUsedBytes());
        WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
        assertEquals(0L, worker2Info.getUsedBytes());
        // Block 1 should exist on master 1
        verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList(worker1Info));
        // Block 2 should exist on master 1
        verifyBlockOnWorkers(mBlockMaster, BLOCK2_ID, BLOCK2_LENGTH, Arrays.asList(worker1Info));
        return null;
    });
}

Example 4 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithReaders.

/**
 * RW contention: Concurrent remove operation and readers.
 * Readers should read the state either before or after the removal.
 */
@Test
public void concurrentRemoveWithReaders() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker and block 1 on worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        CountDownLatch readerLatch = new CountDownLatch(1);
        mBlockMaster.setLatch(readerLatch);
        concurrentWriterWithReaders(readerLatch, // Writer
        () -> {
            List<Long> blocksToRemove = new ArrayList<>();
            blocksToRemove.add(BLOCK1_ID);
            mBlockMaster.removeBlocks(blocksToRemove, deleteMetadata);
            return null;
        }, // Reader
        () -> {
            try {
                // Even if the block is removed, the worker usage will not be updated
                // until the next worker heartbeat
                List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
                assertEquals(1, workerInfoList.size());
                WorkerInfo worker = workerInfoList.get(0);
                assertEquals(BLOCK1_LENGTH, worker.getUsedBytes());
                // If the block is removed already, a BlockInfoException will be thrown
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, workerInfoList);
            } catch (BlockInfoException e) {
            // If the block has been removed, this exception is expected
            // There is nothing more to test here
            }
            return null;
        });
    }
}

Example 5 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithSameWorkerHeartbeatSameBlock.

@Test
public void concurrentCommitWithSameWorkerHeartbeatSameBlock() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    CountDownLatch w1Latch = new CountDownLatch(1);
    mBlockMaster.setLatch(w1Latch);
    concurrentWriterWithWriter(w1Latch, // W1
    () -> {
        mBlockMaster.commitBlock(worker1, 49L, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // W2
    () -> {
        // The same block is removed on worker in this heartbeat
        // This should succeed as commit locks the block exclusively and finishes first
        // When the block heartbeat processes the same block, it has been committed
        Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // 0 used because the block removed on this worker
        MEM_USAGE_EMPTY, // list of removed blockIds
        ImmutableList.of(BLOCK1_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
        // The block has been removed, nothing from command
        assertEquals(EMPTY_CMD, cmd);
        return null;
    }, // Verifier
    () -> {
        // After heartbeat, verify the worker info
        List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
        assertEquals(1, workerInfoList.size());
        WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
        assertEquals(0L, worker1Info.getUsedBytes());
        // The block has no locations now because the last location is removed
        verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList());
        return null;
    });
}