Examples with Command alluxio.grpc.Command used on opensource projects

Example 26 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithSameWorkerHeartbeatSameBlock.

@Test
public void concurrentRemoveWithSameWorkerHeartbeatSameBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        // Prepare block in alluxio
        mBlockMaster.commitBlockInUFS(BLOCK1_ID, BLOCK1_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // The same block is removed on worker in this heartbeat
            // This should succeed as commit locks the block exclusively and finishes first
            // When the block heartbeat processes the same block, it has been committed
            Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // 0 used because the block is already removed
            MEM_USAGE_EMPTY, // list of removed blockIds
            ImmutableList.of(BLOCK1_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // The block has been removed, nothing from command
            assertEquals(EMPTY_CMD, cmd);
            return null;
        }, // Verifier
        () -> {
            // After heartbeat, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(1, workerInfoList.size());
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(0L, worker1Info.getUsedBytes());
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // The block has no locations now because the last location is removed
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList());
            }
            return null;
        });
    }
}

Example 27 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithDifferentWorkerHeartbeatDifferentBlock.

@Test
public void concurrentRemoveWithDifferentWorkerHeartbeatDifferentBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        long worker2 = registerEmptyWorker(NET_ADDRESS_2);
        // Worker 1 has block 1
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        // Worker 2 has block 2
        mBlockMaster.commitBlock(worker2, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // A different block is removed on another worker
            Command cmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // 0 used because the block is already removed
            MEM_USAGE_EMPTY, // list of removed blockIds
            ImmutableList.of(BLOCK2_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // Nothing for worker 2 to do because it does not have block 1
            assertEquals(EMPTY_CMD, cmd);
            return null;
        }, // Verifier
        () -> {
            // After heartbeat, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(2, workerInfoList.size());
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
            WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
            assertEquals(0L, worker2Info.getUsedBytes());
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // Block 1 should still exist on worker 1 until the next heartbeat frees it
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList(worker1Info));
            }
            // No copies for block 2
            verifyBlockOnWorkers(mBlockMaster, BLOCK2_ID, BLOCK2_LENGTH, Arrays.asList());
            return null;
        });
    }
}