Examples with Command alluxio.grpc.Command used on opensource projects

Example 1 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithDifferentWorkerHeartbeatDifferentBlock.

@Test
public void concurrentCommitWithDifferentWorkerHeartbeatDifferentBlock() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    long worker2 = registerEmptyWorker(NET_ADDRESS_2);
    // Register with block 2 on both workers
    mBlockMaster.commitBlock(worker1, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
    mBlockMaster.commitBlock(worker2, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
    CountDownLatch w1Latch = new CountDownLatch(1);
    mBlockMaster.setLatch(w1Latch);
    concurrentWriterWithWriter(w1Latch, // W1
    () -> {
        // worker 1 has block 1 and block 2 now
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH + BLOCK2_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // W2
    () -> {
        // A different block is removed on the same worker
        // This should contend on the worker metadata
        Command cmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // 0 used because the block is already removed
        MEM_USAGE_EMPTY, // list of removed blockIds
        ImmutableList.of(BLOCK2_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
        // The block has been removed, nothing from command
        assertEquals(EMPTY_CMD, cmd);
        return null;
    }, // Verifier
    () -> {
        // After heartbeat, verify the worker info
        List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
        assertEquals(2, workerInfoList.size());
        WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
        assertEquals(BLOCK1_LENGTH + BLOCK2_LENGTH, worker1Info.getUsedBytes());
        WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
        assertEquals(0L, worker2Info.getUsedBytes());
        // Block 1 should exist on master 1
        verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList(worker1Info));
        // Block 2 should exist on master 1
        verifyBlockOnWorkers(mBlockMaster, BLOCK2_ID, BLOCK2_LENGTH, Arrays.asList(worker1Info));
        return null;
    });
}

Example 2 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithSameWorkerHeartbeatSameBlock.

@Test
public void concurrentCommitWithSameWorkerHeartbeatSameBlock() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    CountDownLatch w1Latch = new CountDownLatch(1);
    mBlockMaster.setLatch(w1Latch);
    concurrentWriterWithWriter(w1Latch, // W1
    () -> {
        mBlockMaster.commitBlock(worker1, 49L, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // W2
    () -> {
        // The same block is removed on worker in this heartbeat
        // This should succeed as commit locks the block exclusively and finishes first
        // When the block heartbeat processes the same block, it has been committed
        Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // 0 used because the block removed on this worker
        MEM_USAGE_EMPTY, // list of removed blockIds
        ImmutableList.of(BLOCK1_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
        // The block has been removed, nothing from command
        assertEquals(EMPTY_CMD, cmd);
        return null;
    }, // Verifier
    () -> {
        // After heartbeat, verify the worker info
        List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
        assertEquals(1, workerInfoList.size());
        WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
        assertEquals(0L, worker1Info.getUsedBytes());
        // The block has no locations now because the last location is removed
        verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList());
        return null;
    });
}

Example 3 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithDifferentWorkerHeartbeatSameBlock.

@Test
public void concurrentCommitWithDifferentWorkerHeartbeatSameBlock() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    long worker2 = registerEmptyWorker(NET_ADDRESS_2);
    // The block is on worker 2
    mBlockMaster.commitBlock(worker2, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
    CountDownLatch w1Latch = new CountDownLatch(1);
    mBlockMaster.setLatch(w1Latch);
    concurrentWriterWithWriter(w1Latch, // W1
    () -> {
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // W2
    () -> {
        // The same block is removed on another worker
        // This should succeed as commit locks the block exclusively and finishes first
        // When the block heartbeat processes the same block, it has been committed
        Command cmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // 0 used because the block is already removed
        MEM_USAGE_EMPTY, // list of removed blockIds
        ImmutableList.of(BLOCK1_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
        // The block has been removed, nothing from command
        assertEquals(EMPTY_CMD, cmd);
        return null;
    }, // Verifier
    () -> {
        // After heartbeat, verify the worker info
        List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
        assertEquals(2, workerInfoList.size());
        WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
        assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
        WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
        assertEquals(0L, worker2Info.getUsedBytes());
        // The block has 1 location on worker 1
        verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, Arrays.asList(worker1Info));
        return null;
    });
}

Example 4 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithDifferentWorkerHeartbeatSameBlock.

@Test
public void concurrentRemoveWithDifferentWorkerHeartbeatSameBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        long worker2 = registerEmptyWorker(NET_ADDRESS_2);
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        mBlockMaster.commitBlock(worker2, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // The same block is removed on another worker
            // This should succeed as commit locks the block exclusively and finishes first
            // When the block heartbeat processes the same block, it has been committed
            Command cmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // 0 used because the block is already removed
            MEM_USAGE_EMPTY, // list of removed blockIds
            ImmutableList.of(BLOCK1_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // The block has been removed, nothing from command
            assertEquals(EMPTY_CMD, cmd);
            return null;
        }, // Verifier
        () -> {
            // After heartbeat, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(2, workerInfoList.size());
            // The block is still on worker 1, will be removed on the next heartbeat
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
            WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
            assertEquals(0L, worker2Info.getUsedBytes());
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // The location is still on worker 1, until it is removed after the next heartbeat
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, ImmutableList.of(worker1Info));
            }
            // On the heartbeat worker 1 block will be removed
            Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // Block 1 is still on worker 1
            ImmutableMap.of("MEM", BLOCK1_LENGTH), // list of removed blockIds
            ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            assertEquals(FREE_BLOCK1_CMD, cmd);
            return null;
        });
    }
}

Example 5 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method freeDir.

/**
 * Tests the {@link FileSystemMaster#free} method with a directory.
 */
@Test
public void freeDir() throws Exception {
    mNestedFileContext.setWriteType(WriteType.CACHE_THROUGH);
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size());
    // free the dir
    mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true).setRecursive(true)));
    // Update the heartbeat of removedBlockId received from worker 1.
    Command heartbeat3 = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat3);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}