Examples with Command alluxio.grpc.Command used on opensource projects

Example 6 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithSameWorkerHeartbeatDifferentBlock.

@Test
public void concurrentCommitWithSameWorkerHeartbeatDifferentBlock() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    // Block 2 is on worker 1
    mBlockMaster.commitBlock(worker1, BLOCK2_LENGTH, "MEM", "MEM", BLOCK2_ID, BLOCK2_LENGTH);
    CountDownLatch w1Latch = new CountDownLatch(1);
    mBlockMaster.setLatch(w1Latch);
    concurrentWriterWithWriter(w1Latch, // W1
    () -> {
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // W2
    () -> {
        // A different block is removed on the same worker
        // This should contend on the worker metadata
        Command cmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // 0 used because the block is already removed
        MEM_USAGE_EMPTY, // list of removed blockIds
        ImmutableList.of(BLOCK2_ID), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
        // The block has been removed, nothing from command
        assertEquals(EMPTY_CMD, cmd);
        return null;
    }, // Verifier
    () -> {
        // After heartbeat, verify the worker info
        List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
        assertEquals(1, workerInfoList.size());
        WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
        assertEquals(0L, worker1Info.getUsedBytes());
        verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, workerInfoList);
        return null;
    });
}

Example 7 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithRegisterNewWorkerDifferentBlock.

@Test
public void concurrentRemoveWithRegisterNewWorkerDifferentBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        // Prepare block on the worker
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        // Prepare block 2 so it is recognized at worker register
        mBlockMaster.commitBlockInUFS(BLOCK2_ID, BLOCK2_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        // A new worker as the W2
        long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // The new worker contains the block
            // W1 will remove the block exclusively before worker2 registers with the same block
            // So when worker 2 comes in, the block should be removed already
            // So the block on worker 2 should be ignored
            mBlockMaster.workerRegister(worker2, Arrays.asList("MEM"), MEM_CAPACITY, ImmutableMap.of("MEM", BLOCK2_LENGTH), ImmutableMap.of(newBlockLocationOnWorkerMemTier(worker2), ImmutableList.of(BLOCK2_ID)), NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
            return null;
        }, // Verifier
        () -> {
            // After registration, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(2, workerInfoList.size());
            // Block 1 has not been removed from the workers yet
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
            WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
            assertEquals(BLOCK2_LENGTH, worker2Info.getUsedBytes());
            // Verify the block metadata
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // The master will issue commands to remove blocks on the next heartbeat
                // So now the locations are still there
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, ImmutableList.of(worker1Info));
            }
            // Block 2 is unaffected
            verifyBlockOnWorkers(mBlockMaster, BLOCK2_ID, BLOCK2_LENGTH, ImmutableList.of(worker2Info));
            // Regardless of whether the metadata is removed, the existing block will be freed
            Command worker1HeartbeatCmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // the block has not yet been removed
            ImmutableMap.of("MEM", BLOCK1_LENGTH), // an empty list of removed blockIds
            ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            assertEquals(FREE_BLOCK1_CMD, worker1HeartbeatCmd);
            Command worker2HeartbeatCmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // the block has not yet been removed
            ImmutableMap.of("MEM", BLOCK1_LENGTH), // an empty list of removed blockIds
            ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // Blocks on worker 2 are unaffected
            assertEquals(EMPTY_CMD, worker2HeartbeatCmd);
            return null;
        });
    }
}

Example 8 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class BlockMasterTest method unknownWorkerHeartbeatTriggersRegisterRequest.

@Test
public void unknownWorkerHeartbeatTriggersRegisterRequest() {
    Command heartBeat = mBlockMaster.workerHeartbeat(0, null, null, null, null, null, mMetrics);
    assertEquals(Command.newBuilder().setCommandType(CommandType.Register).build(), heartBeat);
}

Example 9 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method ttlDirectoryFree.

/**
 * Tests that file information is still present after it has been freed after the parent
 * directory's TTL has been set to 0.
 */
@Test
public void ttlDirectoryFree() throws Exception {
    CreateDirectoryContext directoryContext = CreateDirectoryContext.mergeFrom(CreateDirectoryPOptions.newBuilder().setRecursive(true));
    mFileSystemMaster.createDirectory(NESTED_URI, directoryContext);
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size());
    // Set ttl & operation.
    mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))));
    Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.<String, StorageList>of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}

Example 10 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method deleteFile.

/**
 * Tests the {@link FileSystemMaster#delete(AlluxioURI, DeleteContext)} method.
 */
@Test
public void deleteFile() throws Exception {
    // cannot delete root
    try {
        mFileSystemMaster.delete(ROOT_URI, DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true)));
        fail("Should not have been able to delete the root");
    } catch (InvalidPathException e) {
        assertEquals(ExceptionMessage.DELETE_ROOT_DIRECTORY.getMessage(), e.getMessage());
    }
    // delete the file
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    mFileSystemMaster.delete(NESTED_FILE_URI, DeleteContext.defaults());
    try {
        mBlockMaster.getBlockInfo(blockId);
        fail("Expected blockInfo to fail");
    } catch (BlockInfoException e) {
    // expected
    }
    // Update the heartbeat of removedBlockId received from worker 1.
    Command heartbeat1 = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat1);
    assertFalse(mBlockMaster.isBlockLost(blockId));
    // verify the file is deleted
    assertEquals(IdUtils.INVALID_FILE_ID, mFileSystemMaster.getFileId(NESTED_FILE_URI));
    AlluxioURI ufsMount = new AlluxioURI(mTestFolder.newFolder().getAbsolutePath());
    mFileSystemMaster.createDirectory(new AlluxioURI("/mnt/"), CreateDirectoryContext.defaults());
    // Create ufs file.
    Files.createDirectory(Paths.get(ufsMount.join("dir1").getPath()));
    Files.createFile(Paths.get(ufsMount.join("dir1").join("file1").getPath()));
    mFileSystemMaster.mount(new AlluxioURI("/mnt/local"), ufsMount, MountContext.defaults());
    AlluxioURI uri = new AlluxioURI("/mnt/local/dir1");
    mFileSystemMaster.listStatus(uri, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.ALWAYS)));
    mFileSystemMaster.delete(new AlluxioURI("/mnt/local/dir1/file1"), DeleteContext.mergeFrom(DeletePOptions.newBuilder().setAlluxioOnly(true)));
    // ufs file still exists
    assertTrue(Files.exists(Paths.get(ufsMount.join("dir1").join("file1").getPath())));
    // verify the file is deleted
    mThrown.expect(FileDoesNotExistException.class);
    mFileSystemMaster.getFileInfo(new AlluxioURI("/mnt/local/dir1/file1"), GetStatusContext.mergeFrom(GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER)));
}