Examples with LoadTask alluxio.job.plan.load.LoadDefinition.LoadTask used on opensource projects

Example 6 with LoadTask

use of alluxio.job.plan.load.LoadDefinition.LoadTask in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchLoad.

@Test
public void batchLoad() throws Exception {
    int numBlocks = 2;
    int replication = 2;
    int batchSize = 2;
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    for (int i = 0; i < batchSize; i++) {
        createFileWithNoLocations(TEST_URI + i, numBlocks);
        LoadConfig loadConfig = new LoadConfig(TEST_URI + i, replication, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, true);
        ObjectMapper oMapper = new ObjectMapper();
        Map<String, String> map = oMapper.convertValue(loadConfig, Map.class);
        configs.add(map);
    }
    BatchedJobConfig config = new BatchedJobConfig("Load", configs);
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> assignments = new BatchedJobDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask> assignment : assignments) {
        ArrayList<LoadTask> second = (ArrayList<LoadTask>) assignment.getSecond().getJobTaskArgs();
        totalBlockLoads += second.size();
    }
    Assert.assertEquals(numBlocks * replication * batchSize, totalBlockLoads);
}