Examples with Command alluxio.grpc.Command used on opensource projects

Example 11 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method ttlFileFree.

/**
 * Tests that file information is still present after it has been freed after the TTL has been set
 * to 0.
 */
@Test
public void ttlFileFree() throws Exception {
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size());
    // Set ttl & operation.
    mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setCommonOptions(FileSystemOptions.commonDefaults(ServerConfiguration.global()).toBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))));
    Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}

Example 12 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class BlockMasterClient method heartbeat.

/**
 * The method the worker should periodically execute to heartbeat back to the master.
 *
 * @param workerId the worker id
 * @param capacityBytesOnTiers a mapping from storage tier alias to capacity bytes
 * @param usedBytesOnTiers a mapping from storage tier alias to used bytes
 * @param removedBlocks a list of block removed from this worker
 * @param addedBlocks a mapping from storage tier alias to added blocks
 * @param lostStorage a mapping from storage tier alias to a list of lost storage paths
 * @param metrics a list of worker metrics
 * @return an optional command for the worker to execute
 */
public synchronized Command heartbeat(final long workerId, final Map<String, Long> capacityBytesOnTiers, final Map<String, Long> usedBytesOnTiers, final List<Long> removedBlocks, final Map<BlockStoreLocation, List<Long>> addedBlocks, final Map<String, List<String>> lostStorage, final List<Metric> metrics) throws IOException {
    final BlockHeartbeatPOptions options = BlockHeartbeatPOptions.newBuilder().addAllMetrics(metrics).putAllCapacityBytesOnTiers(capacityBytesOnTiers).build();
    final List<LocationBlockIdListEntry> entryList = convertBlockListMapToProto(addedBlocks);
    final Map<String, StorageList> lostStorageMap = lostStorage.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> StorageList.newBuilder().addAllStorage(e.getValue()).build()));
    final BlockHeartbeatPRequest request = BlockHeartbeatPRequest.newBuilder().setWorkerId(workerId).putAllUsedBytesOnTiers(usedBytesOnTiers).addAllRemovedBlockIds(removedBlocks).addAllAddedBlocks(entryList).setOptions(options).putAllLostStorage(lostStorageMap).build();
    return retryRPC(() -> mClient.withDeadlineAfter(mContext.getClusterConf().getMs(PropertyKey.WORKER_MASTER_PERIODICAL_RPC_TIMEOUT), TimeUnit.MILLISECONDS).blockHeartbeat(request).getCommand(), LOG, "Heartbeat", "workerId=%d", workerId);
}

Example 13 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method reregisterWithFree.

@Test
public void reregisterWithFree() throws Exception {
    // Register the worker so the worker is marked active in master
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(workerId);
    prepareBlocksOnMaster(requestChunks);
    Queue<Throwable> errorQueue = new ConcurrentLinkedQueue<>();
    sendStreamToMaster(requestChunks, RegisterStreamTestUtils.getErrorCapturingResponseObserver(errorQueue));
    assertEquals(0, errorQueue.size());
    assertEquals(1, mBlockMaster.getWorkerCount());
    // Find a block to free
    long blockToRemove = RegisterStreamTestUtils.findFirstBlock(requestChunks);
    // Register again
    CountDownLatch latch = new CountDownLatch(1);
    Queue<Throwable> newErrorQueue = new ConcurrentLinkedQueue<>();
    mExecutorService.submit(() -> {
        sendStreamToMasterAndSignal(requestChunks, RegisterStreamTestUtils.getErrorCapturingResponseObserver(newErrorQueue), latch);
    });
    // During the register stream, trigger a delete on worker
    latch.await();
    mBlockMaster.removeBlocks(ImmutableList.of(blockToRemove), false);
    BlockInfo info = mBlockMaster.getBlockInfo(blockToRemove);
    MasterWorkerInfo worker = mBlockMaster.getWorker(workerId);
    assertEquals(0, newErrorQueue.size());
    assertEquals(1, mBlockMaster.getWorkerCount());
    // The block still exists on the worker but a command will be issued to remove it
    assertEquals(TIER_BLOCK_TOTAL, worker.getBlockCount());
    Command command = sendHeartbeatToMaster(workerId);
    assertEquals(Command.newBuilder().setCommandType(CommandType.Free).addData(blockToRemove).build(), command);
}

Example 14 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method ttlDirectoryFreeReplay.

/**
 * Tests that TTL free of a directory is not forgotten across restarts.
 */
@Test
public void ttlDirectoryFreeReplay() throws Exception {
    CreateDirectoryContext directoryContext = CreateDirectoryContext.mergeFrom(CreateDirectoryPOptions.newBuilder().setRecursive(true));
    mFileSystemMaster.createDirectory(NESTED_URI, directoryContext);
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size());
    // Set ttl & operation.
    mFileSystemMaster.setAttribute(NESTED_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setCommonOptions(FileSystemOptions.commonDefaults(ServerConfiguration.global()).toBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))));
    // Simulate restart.
    stopServices();
    startServices();
    Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}

Example 15 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method freeDirWithPinnedFileAndForced.

/**
 * Tests the {@link FileSystemMaster#free} method with a directory with a file pinned when
 * forced flag is true.
 */
@Test
public void freeDirWithPinnedFileAndForced() throws Exception {
    mNestedFileContext.setWriteType(WriteType.CACHE_THROUGH);
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setPinned(true)));
    // free the parent dir of a pinned file with "forced"
    mFileSystemMaster.free(NESTED_FILE_URI.getParent(), FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true).setRecursive(true)));
    // Update the heartbeat of removedBlockId received from worker 1.
    Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}