Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 51 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithSameWorkerHeartbeatSameBlock.

@Test
public void concurrentRemoveWithSameWorkerHeartbeatSameBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        // Prepare block in alluxio
        mBlockMaster.commitBlockInUFS(BLOCK1_ID, BLOCK1_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // The same block is removed on worker in this heartbeat
            // This should succeed as commit locks the block exclusively and finishes first
            // When the block heartbeat processes the same block, it has been committed
            Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // 0 used because the block is already removed
            MEM_USAGE_EMPTY, // list of removed blockIds
            ImmutableList.of(BLOCK1_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // The block has been removed, nothing from command
            assertEquals(EMPTY_CMD, cmd);
            return null;
        }, // Verifier
        () -> {
            // After heartbeat, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(1, workerInfoList.size());
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(0L, worker1Info.getUsedBytes());
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // The block has no locations now because the last location is removed
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList());
            }
            return null;
        });
    }
}

Example 52 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithDifferentWorkerHeartbeatDifferentBlock.

@Test
public void concurrentRemoveWithDifferentWorkerHeartbeatDifferentBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        long worker2 = registerEmptyWorker(NET_ADDRESS_2);
        // Worker 1 has block 1
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        // Worker 2 has block 2
        mBlockMaster.commitBlock(worker2, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // A different block is removed on another worker
            Command cmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // 0 used because the block is already removed
            MEM_USAGE_EMPTY, // list of removed blockIds
            ImmutableList.of(BLOCK2_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // Nothing for worker 2 to do because it does not have block 1
            assertEquals(EMPTY_CMD, cmd);
            return null;
        }, // Verifier
        () -> {
            // After heartbeat, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(2, workerInfoList.size());
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
            WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
            assertEquals(0L, worker2Info.getUsedBytes());
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // Block 1 should still exist on worker 1 until the next heartbeat frees it
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList(worker1Info));
            }
            // No copies for block 2
            verifyBlockOnWorkers(mBlockMaster, BLOCK2_ID, BLOCK2_LENGTH, Arrays.asList());
            return null;
        });
    }
}

Example 53 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithRegisterNewWorkerDifferentBlock.

@Test
public void concurrentCommitWithRegisterNewWorkerDifferentBlock() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2);
    CountDownLatch w1Latch = new CountDownLatch(1);
    mBlockMaster.setLatch(w1Latch);
    concurrentWriterWithWriter(w1Latch, // W1
    () -> {
        mBlockMaster.commitBlock(worker1, 49L, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // W2
    () -> {
        // The new worker contains another block
        // The new block on worker 2 is not recognized and will be ignored by master
        // because the block metadata is not in alluxio
        mBlockMaster.workerRegister(worker2, Arrays.asList("MEM"), MEM_CAPACITY, ImmutableMap.of("MEM", BLOCK2_LENGTH), ImmutableMap.of(newBlockLocationOnWorkerMemTier(worker2), ImmutableList.of(BLOCK2_ID)), NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
        return null;
    }, // Verifier
    () -> {
        // After registration, verify the worker info
        List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
        assertEquals(2, workerInfoList.size());
        WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
        assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
        // Although the new block is not recognized, the worker usage will be taken as-is
        // That new block will be marked orphaned on the worker and will be removed later
        // So that later the worker usage will be rectified
        WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
        assertEquals(BLOCK2_LENGTH, worker2Info.getUsedBytes());
        // Verify the block metadata
        verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList(worker1Info));
        verifyBlockNotExisting(mBlockMaster, BLOCK2_ID);
        return null;
    });
}

Example 54 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class LoadDefinitionTest method loadedBySpecifiedHost.

private void loadedBySpecifiedHost(Set<String> workerSet, Set<String> excludedWorkerSet, Set<String> localityIds, Set<String> excludedLocalityIds, Set<Long> workerIds) throws Exception {
    int numBlocks = 10;
    createFileWithNoLocations(TEST_URI, numBlocks);
    LoadConfig config = new LoadConfig(TEST_URI, 1, workerSet, excludedWorkerSet, localityIds, excludedLocalityIds, false);
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, ArrayList<LoadTask>> assignment : assignments) {
        totalBlockLoads += assignment.getSecond().size();
        Assert.assertTrue(workerIds.contains(assignment.getFirst().getId()));
    }
    Assert.assertEquals(numBlocks, totalBlockLoads);
}

Example 55 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class LoadDefinitionTest method replicationSatisfied.

@Test
public void replicationSatisfied() throws Exception {
    int numBlocks = 7;
    int replication = 3;
    createFileWithNoLocations(TEST_URI, numBlocks);
    LoadConfig config = new LoadConfig(TEST_URI, replication, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false);
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, ArrayList<LoadTask>> assignment : assignments) {
        totalBlockLoads += assignment.getSecond().size();
    }
    Assert.assertEquals(numBlocks * replication, totalBlockLoads);
}