Examples with Command alluxio.grpc.Command used on opensource projects

Example 16 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method freePinnedFileWithForce.

/**
 * Tests {@link FileSystemMaster#free} on pinned file when forced flag is true.
 */
@Test
public void freePinnedFileWithForce() throws Exception {
    mNestedFileContext.setWriteType(WriteType.CACHE_THROUGH);
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setPinned(true)));
    assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size());
    // free the file
    mFileSystemMaster.free(NESTED_FILE_URI, FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(true)));
    // Update the heartbeat of removedBlockId received from worker 1.
    Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}

Example 17 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method free.

/**
 * Tests {@link FileSystemMaster#free} on persisted file.
 */
@Test
public void free() throws Exception {
    mNestedFileContext.setWriteType(WriteType.CACHE_THROUGH);
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size());
    // free the file
    mFileSystemMaster.free(NESTED_FILE_URI, FreeContext.mergeFrom(FreePOptions.newBuilder().setForced(false).setRecursive(false)));
    // Update the heartbeat of removedBlockId received from worker 1.
    Command heartbeat2 = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat2);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}

Example 18 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class FileSystemMasterTest method ttlFileFreeReplay.

/**
 * Tests that TTL free of a file is not forgotten across restarts.
 */
@Test
public void ttlFileFreeReplay() throws Exception {
    long blockId = createFileWithSingleBlock(NESTED_FILE_URI);
    assertEquals(1, mBlockMaster.getBlockInfo(blockId).getLocations().size());
    // Set ttl & operation.
    mFileSystemMaster.setAttribute(NESTED_FILE_URI, SetAttributeContext.mergeFrom(SetAttributePOptions.newBuilder().setCommonOptions(FileSystemOptions.commonDefaults(ServerConfiguration.global()).toBuilder().setTtl(0).setTtlAction(alluxio.grpc.TtlAction.FREE))));
    // Simulate restart.
    stopServices();
    startServices();
    Command heartbeat = mBlockMaster.workerHeartbeat(mWorkerId1, null, ImmutableMap.of(Constants.MEDIUM_MEM, (long) Constants.KB), ImmutableList.of(blockId), ImmutableMap.of(), ImmutableMap.<String, StorageList>of(), mMetrics);
    // Verify the muted Free command on worker1.
    assertEquals(Command.newBuilder().setCommandType(CommandType.Nothing).build(), heartbeat);
    assertEquals(0, mBlockMaster.getBlockInfo(blockId).getLocations().size());
}

Example 19 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class WorkerHeartbeatBench method simulateBlockHeartbeat.

private RpcTaskResult simulateBlockHeartbeat(alluxio.worker.block.BlockMasterClient client, long workerId, Instant endTime) {
    RpcTaskResult result = new RpcTaskResult();
    // Keep sending heartbeats until the expected end time
    long i = 0;
    while (Instant.now().isBefore(endTime)) {
        Instant s = Instant.now();
        try {
            Command cmd = client.heartbeat(workerId, CAPACITY_MEM, USED_MEM_EMPTY, EMPTY_REMOVED_BLOCKS, // So an empty map will be used here
            ImmutableMap.of(), LOST_STORAGE, EMPTY_METRICS);
            LOG.debug("Received command from heartbeat {}", cmd);
            Instant e = Instant.now();
            Duration d = Duration.between(s, e);
            RpcTaskResult.Point p = new RpcTaskResult.Point(d.toMillis());
            LOG.debug("Iter {} took {}ms", i, p.mDurationMs);
            result.addPoint(p);
        } catch (Exception e) {
            LOG.error("Failed to run blockHeartbeat {}", i, e);
            result.addError(e.getMessage());
        // Keep trying even when an exception is met
        }
    }
    return result;
}

Example 20 with Command

use of alluxio.grpc.Command in project alluxio by Alluxio.

the class DefaultBlockMaster method workerHeartbeat.

@Override
public Command workerHeartbeat(long workerId, Map<String, Long> capacityBytesOnTiers, Map<String, Long> usedBytesOnTiers, List<Long> removedBlockIds, Map<BlockLocation, List<Long>> addedBlocks, Map<String, StorageList> lostStorage, List<Metric> metrics) {
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    if (worker == null) {
        LOG.warn("Could not find worker id: {} for heartbeat.", workerId);
        return Command.newBuilder().setCommandType(CommandType.Register).build();
    }
    // Update the TS before the heartbeat so even if the worker heartbeat processing
    // is time-consuming or triggers GC, the worker does not get marked as lost
    // by the LostWorkerDetectionHeartbeatExecutor
    worker.updateLastUpdatedTimeMs();
    // The address is final, no need for locking
    processWorkerMetrics(worker.getWorkerAddress().getHost(), metrics);
    Command workerCommand = null;
    try (LockResource r = worker.lockWorkerMeta(EnumSet.of(WorkerMetaLockSection.USAGE, WorkerMetaLockSection.BLOCKS), false)) {
        worker.addLostStorage(lostStorage);
        if (capacityBytesOnTiers != null) {
            worker.updateCapacityBytes(capacityBytesOnTiers);
        }
        worker.updateUsedBytes(usedBytesOnTiers);
        // Technically, 'worker' should be confirmed to still be in the data structure. Lost worker
        // detection can remove it. However, we are intentionally ignoring this race, since the worker
        // will just re-register regardless.
        processWorkerRemovedBlocks(worker, removedBlockIds, false);
        processWorkerAddedBlocks(worker, addedBlocks);
        Set<Long> toRemoveBlocks = worker.getToRemoveBlocks();
        if (toRemoveBlocks.isEmpty()) {
            workerCommand = Command.newBuilder().setCommandType(CommandType.Nothing).build();
        } else {
            workerCommand = Command.newBuilder().setCommandType(CommandType.Free).addAllData(toRemoveBlocks).build();
        }
    }
    // Update the TS again
    worker.updateLastUpdatedTimeMs();
    // Should not reach here
    Preconditions.checkNotNull(workerCommand, "Worker heartbeat response command is null!");
    return workerCommand;
}