Examples with BlockLocation alluxio.wire.BlockLocation used on opensource projects

Example 26 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class JobUtils method getWorkerWithMostBlocks.

/**
 * Returns whichever specified worker stores the most blocks from the block info list.
 *
 * @param workers a list of workers to consider
 * @param fileBlockInfos a list of file block information
 * @return a worker address storing the most blocks from the list
 */
public static BlockWorkerInfo getWorkerWithMostBlocks(List<BlockWorkerInfo> workers, List<FileBlockInfo> fileBlockInfos) {
    // Index workers by their addresses.
    IndexedSet<BlockWorkerInfo> addressIndexedWorkers = new IndexedSet<>(WORKER_ADDRESS_INDEX);
    addressIndexedWorkers.addAll(workers);
    // Use ConcurrentMap for putIfAbsent. A regular Map works in Java 8.
    ConcurrentMap<BlockWorkerInfo, Integer> blocksPerWorker = Maps.newConcurrentMap();
    int maxBlocks = 0;
    BlockWorkerInfo mostBlocksWorker = null;
    for (FileBlockInfo fileBlockInfo : fileBlockInfos) {
        for (BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
            BlockWorkerInfo worker = addressIndexedWorkers.getFirstByField(WORKER_ADDRESS_INDEX, location.getWorkerAddress());
            if (worker == null) {
                // We can only choose workers in the workers list.
                continue;
            }
            blocksPerWorker.putIfAbsent(worker, 0);
            int newBlockCount = blocksPerWorker.get(worker) + 1;
            blocksPerWorker.put(worker, newBlockCount);
            if (newBlockCount > maxBlocks) {
                maxBlocks = newBlockCount;
                mostBlocksWorker = worker;
            }
        }
    }
    return mostBlocksWorker;
}

Example 27 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class BlockMaster method generateBlockInfo.

/**
   * Creates a {@link BlockInfo} form a given {@link MasterBlockInfo}, by populating worker
   * locations.
   *
   * @param masterBlockInfo the {@link MasterBlockInfo}
   * @return a {@link BlockInfo} from a {@link MasterBlockInfo}. Populates worker locations
   */
@GuardedBy("masterBlockInfo")
private BlockInfo generateBlockInfo(MasterBlockInfo masterBlockInfo) {
    // "Join" to get all the addresses of the workers.
    List<BlockLocation> locations = new ArrayList<>();
    List<MasterBlockLocation> blockLocations = masterBlockInfo.getBlockLocations();
    // Sort the block locations by their alias ordinal in the master storage tier mapping
    Collections.sort(blockLocations, new Comparator<MasterBlockLocation>() {

        @Override
        public int compare(MasterBlockLocation o1, MasterBlockLocation o2) {
            return mGlobalStorageTierAssoc.getOrdinal(o1.getTierAlias()) - mGlobalStorageTierAssoc.getOrdinal(o2.getTierAlias());
        }
    });
    for (MasterBlockLocation masterBlockLocation : blockLocations) {
        MasterWorkerInfo workerInfo = mWorkers.getFirstByField(ID_INDEX, masterBlockLocation.getWorkerId());
        if (workerInfo != null) {
            // worker metadata is intentionally not locked here because:
            // - it would be an incorrect order (correct order is lock worker first, then block)
            // - only uses getters of final variables
            locations.add(new BlockLocation().setWorkerId(masterBlockLocation.getWorkerId()).setWorkerAddress(workerInfo.getWorkerAddress()).setTierAlias(masterBlockLocation.getTierAlias()));
        }
    }
    return new BlockInfo().setBlockId(masterBlockInfo.getBlockId()).setLength(masterBlockInfo.getLength()).setLocations(locations);
}

Example 28 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method scheduleAsyncPersist.

@Test
public void scheduleAsyncPersist() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    assertEquals(1, persistFiles.size());
    assertEquals(Lists.newArrayList(blockId), persistFiles.get(0).getBlockIds());
}

Example 29 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class DefaultAsyncPersistHandlerTest method persistenceFileAfterDeletion.

/**
 * Tests persistence after deletion of files.
 */
@Test
public void persistenceFileAfterDeletion() throws Exception {
    DefaultAsyncPersistHandler handler = new DefaultAsyncPersistHandler(new FileSystemMasterView(mFileSystemMaster));
    AlluxioURI path = new AlluxioURI("/test");
    long blockId = 0;
    long workerId = 1;
    long fileId = 2;
    List<FileBlockInfo> blockInfoList = new ArrayList<>();
    BlockLocation location = new BlockLocation().setWorkerId(workerId);
    blockInfoList.add(new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(blockId).setLocations(Lists.newArrayList(location))));
    when(mFileSystemMaster.getFileBlockInfoList(path)).thenReturn(blockInfoList);
    when(mFileSystemMaster.getFileId(path)).thenReturn(fileId);
    when(mFileSystemMaster.getPath(fileId)).thenReturn(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenReturn(new FileInfo().setLength(1).setCompleted(true));
    handler.scheduleAsyncPersistence(path);
    when(mFileSystemMaster.getFileInfo(fileId)).thenThrow(new FileDoesNotExistException("no file"));
    List<PersistFile> persistFiles = handler.pollFilesToPersist(workerId);
    assertEquals(0, persistFiles.size());
}

Example 30 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class EvictDefinitionTest method selectExecutorsNoJobWorkerHasBlock.

@Test
public void selectExecutorsNoJobWorkerHasBlock() throws Exception {
    Set<Pair<WorkerInfo, SerializableVoid>> result = selectExecutorsTestHelper(Lists.newArrayList(new BlockLocation().setWorkerAddress(ADDRESS_1)), 1, Lists.newArrayList(WORKER_INFO_2, WORKER_INFO_3));
    // Expect none as no worker that is available has this copy
    Assert.assertEquals(EMPTY, result);
}