Examples with MasterWorkerInfo alluxio.master.block.meta.MasterWorkerInfo used on opensource projects

Example 11 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method registerEmptyWorkerStream.

/**
 * Tests below cover the most normal cases.
 */
@Test
public void registerEmptyWorkerStream() throws Exception {
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForEmptyWorker(workerId);
    Queue<Throwable> errorQueue = new ConcurrentLinkedQueue<>();
    sendStreamToMaster(requestChunks, RegisterStreamTestUtils.getErrorCapturingResponseObserver(errorQueue));
    // Verify the worker is registered
    assertEquals(0, errorQueue.size());
    assertEquals(1, mBlockMaster.getWorkerCount());
    MasterWorkerInfo worker = mBlockMaster.getWorker(workerId);
    assertEquals(0, worker.getBlockCount());
    assertEquals(0, worker.getToRemoveBlockCount());
    assertEquals(MEM_CAPACITY_BYTES, worker.getCapacityBytes());
    assertEquals(MEM_CAPACITY_BYTES, worker.getAvailableBytes());
    assertEquals(0L, worker.getUsedBytes());
    // Verify the worker is readable and writable
    verifyWorkerWritable(workerId);
}

Example 12 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMasterRegisterStreamIntegrationTest method registerWorkerStream.

@Test
public void registerWorkerStream() throws Exception {
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    List<RegisterWorkerPRequest> requestChunks = RegisterStreamTestUtils.generateRegisterStreamForWorker(workerId);
    prepareBlocksOnMaster(requestChunks);
    Queue<Throwable> errorQueue = new ConcurrentLinkedQueue<>();
    sendStreamToMaster(requestChunks, RegisterStreamTestUtils.getErrorCapturingResponseObserver(errorQueue));
    // Verify the worker is registered
    assertEquals(0, errorQueue.size());
    MasterWorkerInfo worker = mBlockMaster.getWorker(workerId);
    assertEquals(TIER_BLOCK_TOTAL, worker.getBlockCount());
    assertEquals(0, worker.getToRemoveBlockCount());
    assertEquals(1, mBlockMaster.getWorkerCount());
    // Verify the worker is readable and writable
    verifyWorkerWritable(workerId);
}

Example 13 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method getTotalBytesOnTiers.

/**
   * @return the total bytes on each storage tier
   */
public Map<String, Long> getTotalBytesOnTiers() {
    Map<String, Long> ret = new HashMap<>();
    for (MasterWorkerInfo worker : mWorkers) {
        synchronized (worker) {
            for (Map.Entry<String, Long> entry : worker.getTotalBytesOnTiers().entrySet()) {
                Long total = ret.get(entry.getKey());
                ret.put(entry.getKey(), (total == null ? 0L : total) + entry.getValue());
            }
        }
    }
    return ret;
}

Example 14 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method workerHeartbeat.

/**
   * Updates metadata when a worker periodically heartbeats with the master.
   *
   * @param workerId the worker id
   * @param usedBytesOnTiers a mapping from tier alias to the used bytes
   * @param removedBlockIds a list of block ids removed from this worker
   * @param addedBlocksOnTiers a mapping from tier alias to the added blocks
   * @return an optional command for the worker to execute
   */
public Command workerHeartbeat(long workerId, Map<String, Long> usedBytesOnTiers, List<Long> removedBlockIds, Map<String, List<Long>> addedBlocksOnTiers) {
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    if (worker == null) {
        LOG.warn("Could not find worker id: {} for heartbeat.", workerId);
        return new Command(CommandType.Register, new ArrayList<Long>());
    }
    synchronized (worker) {
        // Technically, 'worker' should be confirmed to still be in the data structure. Lost worker
        // detection can remove it. However, we are intentionally ignoring this race, since the worker
        // will just re-register regardless.
        processWorkerRemovedBlocks(worker, removedBlockIds);
        processWorkerAddedBlocks(worker, addedBlocksOnTiers);
        worker.updateUsedBytes(usedBytesOnTiers);
        worker.updateLastUpdatedTimeMs();
        List<Long> toRemoveBlocks = worker.getToRemoveBlocks();
        if (toRemoveBlocks.isEmpty()) {
            return new Command(CommandType.Nothing, new ArrayList<Long>());
        }
        return new Command(CommandType.Free, toRemoveBlocks);
    }
}

Example 15 with MasterWorkerInfo

use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.

the class BlockMaster method commitBlock.

/**
   * Marks a block as committed on a specific worker.
   *
   * @param workerId the worker id committing the block
   * @param usedBytesOnTier the updated used bytes on the tier of the worker
   * @param tierAlias the alias of the storage tier where the worker is committing the block to
   * @param blockId the committing block id
   * @param length the length of the block
   * @throws NoWorkerException if the workerId is not active
   */
// TODO(binfan): check the logic is correct or not when commitBlock is a retry
public void commitBlock(long workerId, long usedBytesOnTier, String tierAlias, long blockId, long length) throws NoWorkerException {
    LOG.debug("Commit block from workerId: {}, usedBytesOnTier: {}, blockId: {}, length: {}", workerId, usedBytesOnTier, blockId, length);
    MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
    // TODO(peis): Check lost workers as well.
    if (worker == null) {
        throw new NoWorkerException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
    }
    // Lock the worker metadata first.
    try (JournalContext journalContext = createJournalContext()) {
        synchronized (worker) {
            // Loop until block metadata is successfully locked.
            while (true) {
                boolean newBlock = false;
                MasterBlockInfo block = mBlocks.get(blockId);
                if (block == null) {
                    // The block metadata doesn't exist yet.
                    block = new MasterBlockInfo(blockId, length);
                    newBlock = true;
                }
                // Lock the block metadata.
                synchronized (block) {
                    boolean writeJournal = false;
                    if (newBlock) {
                        if (mBlocks.putIfAbsent(blockId, block) != null) {
                            // Another thread already inserted the metadata for this block, so start loop over.
                            continue;
                        }
                        // Successfully added the new block metadata. Append a journal entry for the new
                        // metadata.
                        writeJournal = true;
                    } else if (block.getLength() != length && block.getLength() == Constants.UNKNOWN_SIZE) {
                        // The block size was previously unknown. Update the block size with the committed
                        // size, and append a journal entry.
                        block.updateLength(length);
                        writeJournal = true;
                    }
                    if (writeJournal) {
                        BlockInfoEntry blockInfo = BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build();
                        appendJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build(), journalContext);
                    }
                    // At this point, both the worker and the block metadata are locked.
                    // Update the block metadata with the new worker location.
                    block.addWorker(workerId, tierAlias);
                    // This worker has this block, so it is no longer lost.
                    mLostBlocks.remove(blockId);
                    // Update the worker information for this new block.
                    // TODO(binfan): when retry commitBlock on master is expected, make sure metrics are not
                    // double counted.
                    worker.addBlock(blockId);
                    worker.updateUsedBytes(tierAlias, usedBytesOnTier);
                    worker.updateLastUpdatedTimeMs();
                }
                break;
            }
        }
    }
}