Examples with BlockLocation alluxio.wire.BlockLocation used on opensource projects

Example 36 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class EvictDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, SerializableVoid>> selectExecutors(EvictConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available");
    long blockId = config.getBlockId();
    int numReplicas = config.getReplicas();
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context.getFsContext());
    BlockInfo blockInfo = blockStore.getInfo(blockId);
    Set<String> hosts = new HashSet<>();
    for (BlockLocation blockLocation : blockInfo.getLocations()) {
        hosts.add(blockLocation.getWorkerAddress().getHost());
    }
    Set<Pair<WorkerInfo, SerializableVoid>> result = Sets.newHashSet();
    Collections.shuffle(jobWorkerInfoList);
    for (WorkerInfo workerInfo : jobWorkerInfoList) {
        // Select job workers that have this block locally to evict
        if (hosts.contains(workerInfo.getAddress().getHost())) {
            result.add(new Pair<>(workerInfo, null));
            if (result.size() >= numReplicas) {
                break;
            }
        }
    }
    return result;
}

Example 37 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class ReplicateDefinition method selectExecutors.

@Override
public Set<Pair<WorkerInfo, SerializableVoid>> selectExecutors(ReplicateConfig config, List<WorkerInfo> jobWorkerInfoList, SelectExecutorsContext context) throws Exception {
    Preconditions.checkArgument(!jobWorkerInfoList.isEmpty(), "No worker is available");
    long blockId = config.getBlockId();
    int numReplicas = config.getReplicas();
    Preconditions.checkArgument(numReplicas > 0);
    AlluxioBlockStore blockStore = AlluxioBlockStore.create(context.getFsContext());
    BlockInfo blockInfo = blockStore.getInfo(blockId);
    Set<String> hosts = new HashSet<>();
    for (BlockLocation blockLocation : blockInfo.getLocations()) {
        hosts.add(blockLocation.getWorkerAddress().getHost());
    }
    Set<Pair<WorkerInfo, SerializableVoid>> result = Sets.newHashSet();
    Collections.shuffle(jobWorkerInfoList);
    for (WorkerInfo workerInfo : jobWorkerInfoList) {
        // Select job workers that don't have this block locally to replicate
        if (!hosts.contains(workerInfo.getAddress().getHost())) {
            result.add(new Pair<>(workerInfo, null));
            if (result.size() >= numReplicas) {
                break;
            }
        }
    }
    return result;
}

Example 38 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamRemote.

@Test
public void getInStreamRemote() throws Exception {
    WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1");
    WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2");
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote1), new BlockLocation().setWorkerAddress(remote2)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    // We should sometimes get remote1 and sometimes get remote2.
    Set<WorkerNetAddress> results = new HashSet<>();
    for (int i = 0; i < 40; i++) {
        results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)).getAddress());
    }
    assertEquals(Sets.newHashSet(remote1, remote2), results);
}

Example 39 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentCommitWithReaders.

/**
 * RW contention: Concurrent commit and readers.
 * Signal in commit and the readers inquire the state
 */
@Test
public void concurrentCommitWithReaders() throws Exception {
    // Prepare worker
    long worker1 = registerEmptyWorker(NET_ADDRESS_1);
    // Replace the latch used for preparation
    CountDownLatch readerLatch = new CountDownLatch(1);
    mBlockMaster.setLatch(readerLatch);
    concurrentWriterWithReaders(readerLatch, // Writer
    () -> {
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        return null;
    }, // Reader
    () -> {
        try {
            // If the block is not committed yet, a BlockInfoException will be thrown
            BlockInfo blockInfo = mBlockMaster.getBlockInfo(BLOCK1_ID);
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            WorkerInfo worker = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker.getUsedBytes());
            BlockLocation blockLocation = new BlockLocation().setTierAlias("MEM").setWorkerAddress(NET_ADDRESS_1).setWorkerId(worker1).setMediumType("MEM");
            BlockInfo expectedBlockInfo = new BlockInfo().setBlockId(BLOCK1_ID).setLength(BLOCK1_LENGTH).setLocations(ImmutableList.of(blockLocation));
            assertEquals(expectedBlockInfo, blockInfo);
            assertEquals(1, workerInfoList.size());
        } catch (BlockInfoException e) {
            // The reader came in before the writer started the commit
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(1, workerInfoList.size());
            WorkerInfo worker = workerInfoList.get(0);
            // We may just see the result before or after the commit
            // But other values should be illegal
            assertTrue(BLOCK1_LENGTH == worker.getUsedBytes() || 100L == worker.getUsedBytes());
        }
        return null;
    });
}

Example 40 with BlockLocation

use of alluxio.wire.BlockLocation in project alluxio by Alluxio.

the class BlockMasterTest method getBlockInfo.

@Test
public void getBlockInfo() throws Exception {
    // Create a worker with a block.
    long worker1 = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    long blockId = 1L;
    long blockLength = 20L;
    mBlockMaster.workerRegister(worker1, Arrays.asList(Constants.MEDIUM_MEM), ImmutableMap.of(Constants.MEDIUM_MEM, 100L), ImmutableMap.of(Constants.MEDIUM_MEM, 0L), NO_BLOCKS_ON_LOCATION, NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
    mBlockMaster.commitBlock(worker1, 50L, Constants.MEDIUM_MEM, Constants.MEDIUM_MEM, blockId, blockLength);
    BlockLocation blockLocation = new BlockLocation().setTierAlias(Constants.MEDIUM_MEM).setWorkerAddress(NET_ADDRESS_1).setWorkerId(worker1).setMediumType(Constants.MEDIUM_MEM);
    BlockInfo expectedBlockInfo = new BlockInfo().setBlockId(1L).setLength(20L).setLocations(ImmutableList.of(blockLocation));
    assertEquals(expectedBlockInfo, mBlockMaster.getBlockInfo(blockId));
}