Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 46 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithSameWorkerHeartbeatDifferentBlock.

@Test
public void concurrentRemoveWithSameWorkerHeartbeatDifferentBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true)) {
        // Prepare block 1 and 2 on the worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH + BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        AtomicBoolean freeCommandSeen = new AtomicBoolean(false);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // A different block is removed on the same worker
            // This should contend on the worker metadata
            Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // Block 2 is removed but 1 is still on the worker
            ImmutableMap.of("MEM", BLOCK1_LENGTH), // list of removed blockIds
            ImmutableList.of(BLOCK2_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // adds to the to-be-removed list. In this case the return command has nothing.
            if (cmd.equals(FREE_BLOCK1_CMD)) {
                freeCommandSeen.set(true);
            } else {
                assertEquals(EMPTY_CMD, cmd);
            }
            return null;
        }, // Verifier
        () -> {
            // After heartbeat, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(1, workerInfoList.size());
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // All locations of block 1 are freed in metadata
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, workerInfoList);
            }
            verifyBlockOnWorkers(mBlockMaster, BLOCK2_ID, BLOCK2_LENGTH, ImmutableList.of());
            // and updated the to-be-removed list
            if (!freeCommandSeen.get()) {
                Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // Block 2 is removed but 1 is still on the worker
                ImmutableMap.of("MEM", BLOCK1_LENGTH), // list of removed blockIds
                ImmutableList.of(BLOCK2_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
                assertEquals(FREE_BLOCK1_CMD, cmd);
            }
            return null;
        });
    }
}

Example 47 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithRegisterNewWorkerSameBlock.

@Test
public void concurrentRemoveWithRegisterNewWorkerSameBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        // Prepare block on the worker
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        // A new worker as the W2
        long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // The new worker contains the block
            // W1 will remove the block exclusively before worker2 registers with the same block
            // So when worker 2 comes in, the block should be removed already
            // So the block on worker 2 should be ignored
            mBlockMaster.workerRegister(worker2, Arrays.asList("MEM"), MEM_CAPACITY, ImmutableMap.of("MEM", BLOCK1_LENGTH), ImmutableMap.of(newBlockLocationOnWorkerMemTier(worker2), ImmutableList.of(BLOCK1_ID)), NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
            return null;
        }, // Verifier
        () -> {
            // After registration, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(2, workerInfoList.size());
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
            WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
            assertEquals(BLOCK1_LENGTH, worker2Info.getUsedBytes());
            // Verify the block metadata
            if (deleteMetadata) {
                // If the block metadata has been removed, getting that will get an exception
                assertThrows(BlockInfoException.class, () -> {
                    mBlockMaster.getBlockInfo(BLOCK1_ID);
                });
            } else {
                // The master will issue commands to remove blocks on the next heartbeat
                // So now the locations are still there
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, workerInfoList);
            }
            // Verify the heartbeat from worker will get a command to remove the block
            Command worker1HeartbeatCmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // the block has not yet been removed
            ImmutableMap.of("MEM", BLOCK1_LENGTH), // an empty list of removed blockIds
            ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            assertEquals(FREE_BLOCK1_CMD, worker1HeartbeatCmd);
            if (deleteMetadata) {
                // Block on worker 2 will be freed because the block is already removed
                Command worker2HeartbeatCmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // the block has not yet been removed
                ImmutableMap.of("MEM", BLOCK1_LENGTH), // an empty list of removed blockIds
                ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
                // Block on worker 2 will be freed because the block is already removed
                // Unrecognized blocks will be freed
                assertEquals(FREE_BLOCK1_CMD, worker2HeartbeatCmd);
            } else {
                // Two cases can happen:
                // 1. Worker 2 registers before the free operation checks the block locations
                // In this case the block on worker 2 will be freed
                // 2. Worker 2 registers after the free operation is complete
                // In this case the block on worker 2 will not be freed
                Command worker2HeartbeatCmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // the block has not yet been removed
                ImmutableMap.of("MEM", BLOCK1_LENGTH), // an empty list of removed blockIds
                ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
                assertTrue(worker2HeartbeatCmd.equals(FREE_BLOCK1_CMD) || worker2HeartbeatCmd.equals(EMPTY_CMD));
            }
            return null;
        });
    }
}

Example 48 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithReaders.

/**
 * RW contention: Concurrent commit and readers.
 * Signal in commit and the readers inquire the state
 */
@Test
public void concurrentCommitWithReaders() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    // Replace the latch used for preparation
    CountDownLatch readerLatch = new CountDownLatch(1);
    mBlockMaster.setLatch(readerLatch);
    concurrentWriterWithReaders(readerLatch, // Writer
    () -> {
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // Reader
    () -> {
        try {
            // If the block is not committed yet, a BlockInfoException will be thrown
            BlockInfo blockInfo = mBlockMaster.getBlockInfo(BLOCK1_ID);
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            WorkerInfo worker = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker.getUsedBytes());
            BlockLocation blockLocation = new BlockLocation().setTierAlias("MEM").setWorkerAddress(NET_ADDRESS_1).setWorkerId(worker1).setMediumType("MEM");
            BlockInfo expectedBlockInfo = new BlockInfo().setBlockId(BLOCK1_ID).setLength(BLOCK1_LENGTH).setLocations(ImmutableList.of(blockLocation));
            assertEquals(expectedBlockInfo, blockInfo);
            assertEquals(1, workerInfoList.size());
        } catch (BlockInfoException e) {
            // The reader came in before the writer started the commit
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(1, workerInfoList.size());
            WorkerInfo worker = workerInfoList.get(0);
            // We may just see the result before or after the commit
            // But other values should be illegal
            assertTrue(BLOCK1_LENGTH == worker.getUsedBytes() || 100L == worker.getUsedBytes());
        }
        return null;
    });
}

Example 49 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class BlockMasterTest method workerHeartbeatUpdatesMemoryCount.

@Test
public void workerHeartbeatUpdatesMemoryCount() throws Exception {
    // Create a worker.
    long worker = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    Map<String, Long> initialUsedBytesOnTiers = ImmutableMap.of(Constants.MEDIUM_MEM, 50L);
    mBlockMaster.workerRegister(worker, Arrays.asList(Constants.MEDIUM_MEM), ImmutableMap.of(Constants.MEDIUM_MEM, 100L), initialUsedBytesOnTiers, NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
    // Update used bytes with a worker heartbeat.
    Map<String, Long> newUsedBytesOnTiers = ImmutableMap.of(Constants.MEDIUM_MEM, 50L);
    mBlockMaster.workerHeartbeat(worker, null, newUsedBytesOnTiers, NO_BLOCKS, NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, mMetrics);
    WorkerInfo workerInfo = Iterables.getOnlyElement(mBlockMaster.getWorkerInfoList());
    assertEquals(50, workerInfo.getUsedBytes());
}

Example 50 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class MasterWorkerInfoTest method workerInfoGeneration.

/**
 * Tests the {@link MasterWorkerInfo#generateWorkerInfo} method.
 */
@Test
public void workerInfoGeneration() {
    WorkerInfo workerInfo = mInfo.generateWorkerInfo(null, true);
    assertEquals(mInfo.getId(), workerInfo.getId());
    assertEquals(mInfo.getWorkerAddress(), workerInfo.getAddress());
    assertEquals("In Service", workerInfo.getState());
    assertEquals(mInfo.getCapacityBytes(), workerInfo.getCapacityBytes());
    assertEquals(mInfo.getUsedBytes(), workerInfo.getUsedBytes());
    assertEquals(mInfo.getStartTime(), workerInfo.getStartTimeMs());
}