Examples with SelectExecutorsContext alluxio.job.SelectExecutorsContext used on opensource projects

Example 1 with SelectExecutorsContext

use of alluxio.job.SelectExecutorsContext in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchPersist.

@Test
public void batchPersist() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    ObjectMapper oMapper = new ObjectMapper();
    Map<String, String> map = oMapper.convertValue(config, Map.class);
    configs.add(map);
    BatchedJobConfig batchedJobConfig = new BatchedJobConfig("Persist", configs);
    WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10);
    WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress);
    long blockId = 1;
    BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
    FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
    BlockLocation location = new BlockLocation();
    location.setWorkerAddress(workerNetAddress);
    blockInfo.setLocations(Lists.newArrayList(location));
    FileInfo testFileInfo = new FileInfo();
    testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
    Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> result = new BatchedJobDefinition().selectExecutors(batchedJobConfig, Lists.newArrayList(workerInfo), new SelectExecutorsContext(1, mJobServerContext));
    System.out.println(result);
    Assert.assertNull(result.iterator().next().getSecond().getJobTaskArgs());
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(workerInfo, result.iterator().next().getFirst());
}

Example 2 with SelectExecutorsContext

use of alluxio.job.SelectExecutorsContext in project alluxio by Alluxio.

the class LoadDefinitionTest method skipJobWorkersWithoutLocalBlockWorkers.

@Test
public void skipJobWorkersWithoutLocalBlockWorkers() throws Exception {
    List<BlockWorkerInfo> blockWorkers = Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress().setHost("host0"), 0, 0));
    Mockito.when(mMockFsContext.getCachedWorkers()).thenReturn(blockWorkers);
    createFileWithNoLocations(TEST_URI, 10);
    LoadConfig config = new LoadConfig(TEST_URI, 1, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false);
    Set<Pair<WorkerInfo, ArrayList<LoadTask>>> assignments = new LoadDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    Assert.assertEquals(10, assignments.size());
    Assert.assertEquals(1, assignments.iterator().next().getSecond().size());
}

Example 3 with SelectExecutorsContext

use of alluxio.job.SelectExecutorsContext in project alluxio by Alluxio.

the class PersistDefinitionTest method selectExecutorsTest.

@Test
public void selectExecutorsTest() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
    WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10);
    WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress);
    long blockId = 1;
    BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
    FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
    BlockLocation location = new BlockLocation();
    location.setWorkerAddress(workerNetAddress);
    blockInfo.setLocations(Lists.newArrayList(location));
    FileInfo testFileInfo = new FileInfo();
    testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
    Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
    Set<Pair<WorkerInfo, SerializableVoid>> result = new PersistDefinition().selectExecutors(config, Lists.newArrayList(workerInfo), new SelectExecutorsContext(1, mJobServerContext));
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(workerInfo, result.iterator().next().getFirst());
}

Example 4 with SelectExecutorsContext

use of alluxio.job.SelectExecutorsContext in project alluxio by Alluxio.

the class EvictDefinitionTest method selectExecutorsTestHelper.

/**
 * Helper function to select executors.
 *
 * @param blockLocations where the block is stored currently
 * @param replicas how many replicas to evict
 * @param workerInfoList a list of currently available job workers
 * @return the selection result
 */
private Set<Pair<WorkerInfo, SerializableVoid>> selectExecutorsTestHelper(List<BlockLocation> blockLocations, int replicas, List<WorkerInfo> workerInfoList) throws Exception {
    BlockInfo blockInfo = new BlockInfo().setBlockId(TEST_BLOCK_ID);
    blockInfo.setLocations(blockLocations);
    when(mMockBlockStore.getInfo(TEST_BLOCK_ID)).thenReturn(blockInfo);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mMockFileSystemContext)).thenReturn(mMockBlockStore);
    EvictConfig config = new EvictConfig("", TEST_BLOCK_ID, replicas);
    EvictDefinition definition = new EvictDefinition();
    return definition.selectExecutors(config, workerInfoList, new SelectExecutorsContext(1, mJobServerContext));
}

Example 5 with SelectExecutorsContext

use of alluxio.job.SelectExecutorsContext in project alluxio by Alluxio.

the class PlanCoordinator method start.

private synchronized void start() throws JobDoesNotExistException {
    // get the job definition
    LOG.info("Starting job Id={} Config={}", mPlanInfo.getId(), mPlanInfo.getJobConfig());
    PlanDefinition<JobConfig, ?, ?> definition;
    try {
        definition = PlanDefinitionRegistry.INSTANCE.getJobDefinition(mPlanInfo.getJobConfig());
    } catch (JobDoesNotExistException e) {
        LOG.info("Exception when getting jobDefinition from jobConfig: ", e);
        mPlanInfo.setErrorType(ErrorUtils.getErrorType(e));
        mPlanInfo.setErrorMessage(e.getMessage());
        DistributedCmdMetrics.incrementForAllConfigsFailStatus(mPlanInfo.getJobConfig());
        mPlanInfo.setStatus(Status.FAILED);
        throw e;
    }
    SelectExecutorsContext context = new SelectExecutorsContext(mPlanInfo.getId(), mJobServerContext);
    Set<? extends Pair<WorkerInfo, ?>> taskAddressToArgs;
    ArrayList<WorkerInfo> workersInfoListCopy = Lists.newArrayList(mWorkersInfoList);
    Collections.shuffle(workersInfoListCopy);
    try {
        taskAddressToArgs = definition.selectExecutors(mPlanInfo.getJobConfig(), workersInfoListCopy, context);
    } catch (Exception e) {
        LOG.warn("Failed to select executor. {})", e.toString());
        LOG.info("Exception: ", e);
        setJobAsFailed(ErrorUtils.getErrorType(e), e.getMessage());
        return;
    }
    if (taskAddressToArgs.isEmpty()) {
        LOG.warn("No executor was selected.");
        updateStatus();
    }
    for (Pair<WorkerInfo, ?> pair : taskAddressToArgs) {
        LOG.debug("Selected executor {} with parameters {}.", pair.getFirst(), pair.getSecond());
        int taskId = mTaskIdToWorkerInfo.size();
        // create task
        mPlanInfo.addTask(taskId, pair.getFirst(), pair.getSecond());
        // submit commands
        JobConfig config;
        if (mPlanInfo.getJobConfig() instanceof BatchedJobConfig) {
            BatchedJobConfig planConfig = (BatchedJobConfig) mPlanInfo.getJobConfig();
            config = new BatchedJobConfig(planConfig.getJobType(), new HashSet<>());
        } else {
            config = mPlanInfo.getJobConfig();
        }
        mCommandManager.submitRunTaskCommand(mPlanInfo.getId(), taskId, config, pair.getSecond(), pair.getFirst().getId());
        mTaskIdToWorkerInfo.put((long) taskId, pair.getFirst());
        mWorkerIdToTaskIds.putIfAbsent(pair.getFirst().getId(), Lists.newArrayList());
        mWorkerIdToTaskIds.get(pair.getFirst().getId()).add((long) taskId);
    }
}