use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.
the class BlockMaster method commitBlock.
/**
* Marks a block as committed on a specific worker.
*
* @param workerId the worker id committing the block
* @param usedBytesOnTier the updated used bytes on the tier of the worker
* @param tierAlias the alias of the storage tier where the worker is committing the block to
* @param blockId the committing block id
* @param length the length of the block
* @throws NoWorkerException if the workerId is not active
*/
// TODO(binfan): check the logic is correct or not when commitBlock is a retry
public void commitBlock(long workerId, long usedBytesOnTier, String tierAlias, long blockId, long length) throws NoWorkerException {
LOG.debug("Commit block from workerId: {}, usedBytesOnTier: {}, blockId: {}, length: {}", workerId, usedBytesOnTier, blockId, length);
MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
// TODO(peis): Check lost workers as well.
if (worker == null) {
throw new NoWorkerException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
}
// Lock the worker metadata first.
try (JournalContext journalContext = createJournalContext()) {
synchronized (worker) {
// Loop until block metadata is successfully locked.
while (true) {
boolean newBlock = false;
MasterBlockInfo block = mBlocks.get(blockId);
if (block == null) {
// The block metadata doesn't exist yet.
block = new MasterBlockInfo(blockId, length);
newBlock = true;
}
// Lock the block metadata.
synchronized (block) {
boolean writeJournal = false;
if (newBlock) {
if (mBlocks.putIfAbsent(blockId, block) != null) {
// Another thread already inserted the metadata for this block, so start loop over.
continue;
}
// Successfully added the new block metadata. Append a journal entry for the new
// metadata.
writeJournal = true;
} else if (block.getLength() != length && block.getLength() == Constants.UNKNOWN_SIZE) {
// The block size was previously unknown. Update the block size with the committed
// size, and append a journal entry.
block.updateLength(length);
writeJournal = true;
}
if (writeJournal) {
BlockInfoEntry blockInfo = BlockInfoEntry.newBuilder().setBlockId(blockId).setLength(length).build();
appendJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build(), journalContext);
}
// At this point, both the worker and the block metadata are locked.
// Update the block metadata with the new worker location.
block.addWorker(workerId, tierAlias);
// This worker has this block, so it is no longer lost.
mLostBlocks.remove(blockId);
// Update the worker information for this new block.
// TODO(binfan): when retry commitBlock on master is expected, make sure metrics are not
// double counted.
worker.addBlock(blockId);
worker.updateUsedBytes(tierAlias, usedBytesOnTier);
worker.updateLastUpdatedTimeMs();
}
break;
}
}
}
}
use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.
the class BlockMaster method generateBlockInfo.
/**
* Creates a {@link BlockInfo} form a given {@link MasterBlockInfo}, by populating worker
* locations.
*
* @param masterBlockInfo the {@link MasterBlockInfo}
* @return a {@link BlockInfo} from a {@link MasterBlockInfo}. Populates worker locations
*/
@GuardedBy("masterBlockInfo")
private BlockInfo generateBlockInfo(MasterBlockInfo masterBlockInfo) {
// "Join" to get all the addresses of the workers.
List<BlockLocation> locations = new ArrayList<>();
List<MasterBlockLocation> blockLocations = masterBlockInfo.getBlockLocations();
// Sort the block locations by their alias ordinal in the master storage tier mapping
Collections.sort(blockLocations, new Comparator<MasterBlockLocation>() {
@Override
public int compare(MasterBlockLocation o1, MasterBlockLocation o2) {
return mGlobalStorageTierAssoc.getOrdinal(o1.getTierAlias()) - mGlobalStorageTierAssoc.getOrdinal(o2.getTierAlias());
}
});
for (MasterBlockLocation masterBlockLocation : blockLocations) {
MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, masterBlockLocation.getWorkerId());
if (workerInfo != null) {
// worker metadata is intentionally not locked here because:
// - it would be an incorrect order (correct order is lock worker first, then block)
// - only uses getters of final variables
locations.add(new BlockLocation().setWorkerId(masterBlockLocation.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(masterBlockLocation.getTierAlias()));
}
}
return new BlockInfo().setBlockId(masterBlockInfo.getBlockId()).setLength(masterBlockInfo.getLength()).setLocations(locations);
}
use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.
the class BlockMaster method removeBlocks.
/**
* Removes blocks from workers.
*
* @param blockIds a list of block ids to remove from Alluxio space
* @param delete whether to delete blocks' metadata in Master
*/
public void removeBlocks(List<Long> blockIds, boolean delete) {
for (long blockId : blockIds) {
MasterBlockInfo block = mBlocks.get(blockId);
if (block == null) {
continue;
}
HashSet<Long> workerIds = new HashSet<>();
synchronized (block) {
// Technically, 'block' should be confirmed to still be in the data structure. A
// concurrent removeBlock call can remove it. However, we are intentionally ignoring this
// race, since deleting the same block again is a noop.
workerIds.addAll(block.getWorkers());
// processWorkerRemovedBlocks
if (delete) {
// Make sure blockId is removed from mLostBlocks when the block metadata is deleted.
// Otherwise blockId in mLostBlock can be dangling index if the metadata is gone.
mLostBlocks.remove(blockId);
mBlocks.remove(blockId);
}
}
// metadata, since it is essentially an asynchronous signal to the worker to remove the block.
for (long workerId : workerIds) {
MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
if (worker != null) {
synchronized (worker) {
worker.updateToRemovedBlock(true, blockId);
}
}
}
}
}
use of alluxio.master.block.meta.MasterWorkerInfo in project alluxio by Alluxio.
the class BlockMaster method workerRegister.
/**
* Updates metadata when a worker registers with the master.
*
* @param workerId the worker id of the worker registering
* @param storageTiers a list of storage tier aliases in order of their position in the worker's
* hierarchy
* @param totalBytesOnTiers a mapping from storage tier alias to total bytes
* @param usedBytesOnTiers a mapping from storage tier alias to the used byes
* @param currentBlocksOnTiers a mapping from storage tier alias to a list of blocks
* @throws NoWorkerException if workerId cannot be found
*/
public void workerRegister(long workerId, List<String> storageTiers, Map<String, Long> totalBytesOnTiers, Map<String, Long> usedBytesOnTiers, Map<String, List<Long>> currentBlocksOnTiers) throws NoWorkerException {
MasterWorkerInfo worker = mWorkers.getFirstByField(ID_INDEX, workerId);
if (worker == null) {
throw new NoWorkerException(ExceptionMessage.NO_WORKER_FOUND.getMessage(workerId));
}
// Gather all blocks on this worker.
HashSet<Long> blocks = new HashSet<>();
for (List<Long> blockIds : currentBlocksOnTiers.values()) {
blocks.addAll(blockIds);
}
synchronized (worker) {
worker.updateLastUpdatedTimeMs();
// Detect any lost blocks on this worker.
Set<Long> removedBlocks = worker.register(mGlobalStorageTierAssoc, storageTiers, totalBytesOnTiers, usedBytesOnTiers, blocks);
processWorkerRemovedBlocks(worker, removedBlocks);
processWorkerAddedBlocks(worker, currentBlocksOnTiers);
}
LOG.info("registerWorker(): {}", worker);
}
Aggregations