Examples with BatchedJobDefinition alluxio.job.plan.batch.BatchedJobDefinition used on opensource projects

Example 1 with BatchedJobDefinition

use of alluxio.job.plan.batch.BatchedJobDefinition in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchPersist.

@Test
public void batchPersist() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    ObjectMapper oMapper = new ObjectMapper();
    Map<String, String> map = oMapper.convertValue(config, Map.class);
    configs.add(map);
    BatchedJobConfig batchedJobConfig = new BatchedJobConfig("Persist", configs);
    WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10);
    WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress);
    long blockId = 1;
    BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
    FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
    BlockLocation location = new BlockLocation();
    location.setWorkerAddress(workerNetAddress);
    blockInfo.setLocations(Lists.newArrayList(location));
    FileInfo testFileInfo = new FileInfo();
    testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
    Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> result = new BatchedJobDefinition().selectExecutors(batchedJobConfig, Lists.newArrayList(workerInfo), new SelectExecutorsContext(1, mJobServerContext));
    System.out.println(result);
    Assert.assertNull(result.iterator().next().getSecond().getJobTaskArgs());
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(workerInfo, result.iterator().next().getFirst());
}

Example 2 with BatchedJobDefinition

use of alluxio.job.plan.batch.BatchedJobDefinition in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchLoad.

@Test
public void batchLoad() throws Exception {
    int numBlocks = 2;
    int replication = 2;
    int batchSize = 2;
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    for (int i = 0; i < batchSize; i++) {
        createFileWithNoLocations(TEST_URI + i, numBlocks);
        LoadConfig loadConfig = new LoadConfig(TEST_URI + i, replication, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, true);
        ObjectMapper oMapper = new ObjectMapper();
        Map<String, String> map = oMapper.convertValue(loadConfig, Map.class);
        configs.add(map);
    }
    BatchedJobConfig config = new BatchedJobConfig("Load", configs);
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> assignments = new BatchedJobDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask> assignment : assignments) {
        ArrayList<LoadTask> second = (ArrayList<LoadTask>) assignment.getSecond().getJobTaskArgs();
        totalBlockLoads += second.size();
    }
    Assert.assertEquals(numBlocks * replication * batchSize, totalBlockLoads);
}