Examples with WorkerInfo alluxio.wire.WorkerInfo used on opensource projects

Example 11 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class ConcurrentBlockMasterTest method concurrentRemoveWithRegisterNewWorkerDifferentBlock.

@Test
public void concurrentRemoveWithRegisterNewWorkerDifferentBlock() throws Exception {
    for (boolean deleteMetadata : ImmutableList.of(true, false)) {
        // Prepare worker
        long worker1 = registerEmptyWorker(NET_ADDRESS_1);
        // Prepare block on the worker
        mBlockMaster.commitBlock(worker1, BLOCK1_LENGTH, "MEM", "MEM", BLOCK1_ID, BLOCK1_LENGTH);
        // Prepare block 2 so it is recognized at worker register
        mBlockMaster.commitBlockInUFS(BLOCK2_ID, BLOCK2_LENGTH);
        CountDownLatch w1Latch = new CountDownLatch(1);
        mBlockMaster.setLatch(w1Latch);
        // A new worker as the W2
        long worker2 = mBlockMaster.getWorkerId(NET_ADDRESS_2);
        concurrentWriterWithWriter(w1Latch, // W1
        () -> {
            mBlockMaster.removeBlocks(ImmutableList.of(BLOCK1_ID), deleteMetadata);
            return null;
        }, // W2
        () -> {
            // The new worker contains the block
            // W1 will remove the block exclusively before worker2 registers with the same block
            // So when worker 2 comes in, the block should be removed already
            // So the block on worker 2 should be ignored
            mBlockMaster.workerRegister(worker2, Arrays.asList("MEM"), MEM_CAPACITY, ImmutableMap.of("MEM", BLOCK2_LENGTH), ImmutableMap.of(newBlockLocationOnWorkerMemTier(worker2), ImmutableList.of(BLOCK2_ID)), NO_LOST_STORAGE, RegisterWorkerPOptions.getDefaultInstance());
            return null;
        }, // Verifier
        () -> {
            // After registration, verify the worker info
            List<WorkerInfo> workerInfoList = mBlockMaster.getWorkerReport(GetWorkerReportOptions.defaults());
            assertEquals(2, workerInfoList.size());
            // Block 1 has not been removed from the workers yet
            WorkerInfo worker1Info = findWorkerInfo(workerInfoList, worker1);
            assertEquals(BLOCK1_LENGTH, worker1Info.getUsedBytes());
            WorkerInfo worker2Info = findWorkerInfo(workerInfoList, worker2);
            assertEquals(BLOCK2_LENGTH, worker2Info.getUsedBytes());
            // Verify the block metadata
            if (deleteMetadata) {
                verifyBlockNotExisting(mBlockMaster, BLOCK1_ID);
            } else {
                // The master will issue commands to remove blocks on the next heartbeat
                // So now the locations are still there
                verifyBlockOnWorkers(mBlockMaster, BLOCK1_ID, BLOCK1_LENGTH, ImmutableList.of(worker1Info));
            }
            // Block 2 is unaffected
            verifyBlockOnWorkers(mBlockMaster, BLOCK2_ID, BLOCK2_LENGTH, ImmutableList.of(worker2Info));
            // Regardless of whether the metadata is removed, the existing block will be freed
            Command worker1HeartbeatCmd = mBlockMaster.workerHeartbeat(worker1, MEM_CAPACITY, // the block has not yet been removed
            ImmutableMap.of("MEM", BLOCK1_LENGTH), // an empty list of removed blockIds
            ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            assertEquals(FREE_BLOCK1_CMD, worker1HeartbeatCmd);
            Command worker2HeartbeatCmd = mBlockMaster.workerHeartbeat(worker2, MEM_CAPACITY, // the block has not yet been removed
            ImmutableMap.of("MEM", BLOCK1_LENGTH), // an empty list of removed blockIds
            ImmutableList.of(), ImmutableMap.of(), NO_LOST_STORAGE, ImmutableList.of());
            // Blocks on worker 2 are unaffected
            assertEquals(EMPTY_CMD, worker2HeartbeatCmd);
            return null;
        });
    }
}

Example 12 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class PlanCoordinatorTest method before.

@Before
public void before() throws Exception {
    mCommandManager = new CommandManager();
    // Create mock JobServerContext
    FileSystem fs = mock(FileSystem.class);
    FileSystemContext fsCtx = PowerMockito.mock(FileSystemContext.class);
    UfsManager ufsManager = Mockito.mock(UfsManager.class);
    mJobServerContext = new JobServerContext(fs, fsCtx, ufsManager);
    // Create mock job info.
    mJobconfig = Mockito.mock(JobConfig.class, Mockito.withSettings().serializable());
    Mockito.when(mJobconfig.getName()).thenReturn("mock");
    mJobId = 1;
    // Create mock job definition.
    @SuppressWarnings("unchecked") PlanDefinition<JobConfig, Serializable, Serializable> mockPlanDefinition = Mockito.mock(PlanDefinition.class);
    PlanDefinitionRegistry singleton = PowerMockito.mock(PlanDefinitionRegistry.class);
    Whitebox.setInternalState(PlanDefinitionRegistry.class, "INSTANCE", singleton);
    Mockito.when(singleton.getJobDefinition(mJobconfig)).thenReturn(mockPlanDefinition);
    mPlanDefinition = mockPlanDefinition;
    // Create test worker.
    mWorkerInfo = new WorkerInfo();
    mWorkerInfo.setId(0);
    mWorkerInfoList = Lists.newArrayList(mWorkerInfo);
}

Example 13 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchPersist.

@Test
public void batchPersist() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    ObjectMapper oMapper = new ObjectMapper();
    Map<String, String> map = oMapper.convertValue(config, Map.class);
    configs.add(map);
    BatchedJobConfig batchedJobConfig = new BatchedJobConfig("Persist", configs);
    WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10);
    WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress);
    long blockId = 1;
    BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
    FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
    BlockLocation location = new BlockLocation();
    location.setWorkerAddress(workerNetAddress);
    blockInfo.setLocations(Lists.newArrayList(location));
    FileInfo testFileInfo = new FileInfo();
    testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
    Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> result = new BatchedJobDefinition().selectExecutors(batchedJobConfig, Lists.newArrayList(workerInfo), new SelectExecutorsContext(1, mJobServerContext));
    System.out.println(result);
    Assert.assertNull(result.iterator().next().getSecond().getJobTaskArgs());
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(workerInfo, result.iterator().next().getFirst());
}

Example 14 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class PersistDefinitionTest method selectExecutorsTest.

@Test
public void selectExecutorsTest() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
    WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10);
    WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress);
    long blockId = 1;
    BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
    FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
    BlockLocation location = new BlockLocation();
    location.setWorkerAddress(workerNetAddress);
    blockInfo.setLocations(Lists.newArrayList(location));
    FileInfo testFileInfo = new FileInfo();
    testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
    Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
    Set<Pair<WorkerInfo, SerializableVoid>> result = new PersistDefinition().selectExecutors(config, Lists.newArrayList(workerInfo), new SelectExecutorsContext(1, mJobServerContext));
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(workerInfo, result.iterator().next().getFirst());
}

Example 15 with WorkerInfo

use of alluxio.wire.WorkerInfo in project alluxio by Alluxio.

the class PlanCoordinator method start.

private synchronized void start() throws JobDoesNotExistException {
    // get the job definition
    LOG.info("Starting job Id={} Config={}", mPlanInfo.getId(), mPlanInfo.getJobConfig());
    PlanDefinition<JobConfig, ?, ?> definition;
    try {
        definition = PlanDefinitionRegistry.INSTANCE.getJobDefinition(mPlanInfo.getJobConfig());
    } catch (JobDoesNotExistException e) {
        LOG.info("Exception when getting jobDefinition from jobConfig: ", e);
        mPlanInfo.setErrorType(ErrorUtils.getErrorType(e));
        mPlanInfo.setErrorMessage(e.getMessage());
        DistributedCmdMetrics.incrementForAllConfigsFailStatus(mPlanInfo.getJobConfig());
        mPlanInfo.setStatus(Status.FAILED);
        throw e;
    }
    SelectExecutorsContext context = new SelectExecutorsContext(mPlanInfo.getId(), mJobServerContext);
    Set<? extends Pair<WorkerInfo, ?>> taskAddressToArgs;
    ArrayList<WorkerInfo> workersInfoListCopy = Lists.newArrayList(mWorkersInfoList);
    Collections.shuffle(workersInfoListCopy);
    try {
        taskAddressToArgs = definition.selectExecutors(mPlanInfo.getJobConfig(), workersInfoListCopy, context);
    } catch (Exception e) {
        LOG.warn("Failed to select executor. {})", e.toString());
        LOG.info("Exception: ", e);
        setJobAsFailed(ErrorUtils.getErrorType(e), e.getMessage());
        return;
    }
    if (taskAddressToArgs.isEmpty()) {
        LOG.warn("No executor was selected.");
        updateStatus();
    }
    for (Pair<WorkerInfo, ?> pair : taskAddressToArgs) {
        LOG.debug("Selected executor {} with parameters {}.", pair.getFirst(), pair.getSecond());
        int taskId = mTaskIdToWorkerInfo.size();
        // create task
        mPlanInfo.addTask(taskId, pair.getFirst(), pair.getSecond());
        // submit commands
        JobConfig config;
        if (mPlanInfo.getJobConfig() instanceof BatchedJobConfig) {
            BatchedJobConfig planConfig = (BatchedJobConfig) mPlanInfo.getJobConfig();
            config = new BatchedJobConfig(planConfig.getJobType(), new HashSet<>());
        } else {
            config = mPlanInfo.getJobConfig();
        }
        mCommandManager.submitRunTaskCommand(mPlanInfo.getId(), taskId, config, pair.getSecond(), pair.getFirst().getId());
        mTaskIdToWorkerInfo.put((long) taskId, pair.getFirst());
        mWorkerIdToTaskIds.putIfAbsent(pair.getFirst().getId(), Lists.newArrayList());
        mWorkerIdToTaskIds.get(pair.getFirst().getId()).add((long) taskId);
    }
}