Examples with LoadConfig alluxio.job.plan.load.LoadConfig used on opensource projects

Example 1 with LoadConfig

use of alluxio.job.plan.load.LoadConfig in project alluxio by Alluxio.

the class DistributedCommandsStatsTest method testDistributedLoadCancelStats.

@Test
public void testDistributedLoadCancelStats() throws Exception {
    FileSystemTestUtils.createByteFile(sFileSystem, "/testFileNew", WritePType.THROUGH, 10);
    long jobId = sJobMaster.run(new LoadConfig("/testFileNew", 1, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false));
    sJobShell.run("cancel", Long.toString(jobId));
    JobTestUtils.waitForJobStatus(sJobMaster, jobId, Sets.newHashSet(Status.CANCELED), TEST_TIMEOUT);
    sJobShell.run("stat", "-v", Long.toString(jobId));
    String[] output = mOutput.toString().split("\n");
    assertEquals(String.format("ID: %s", jobId), output[0]);
    assertEquals(String.format("Name: Load"), output[1]);
    assertTrue(output[2].contains("Description: LoadConfig"));
    assertTrue(output[2].contains("/testFileNew"));
    assertEquals("Status: CANCELED", output[3]);
    assertEquals("Task 0", output[4]);
    assertTrue(output[5].contains("\tWorker: "));
    assertEquals("\tStatus: CANCELED", output[7]);
    double cancelledCount = MetricsSystem.getMetricValue(MetricKey.MASTER_JOB_DISTRIBUTED_LOAD_CANCEL.getName()).getValue();
    assertEquals(cancelledCount, 1, 0);
}

Example 2 with LoadConfig

use of alluxio.job.plan.load.LoadConfig in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchLoad.

@Test
public void batchLoad() throws Exception {
    int numBlocks = 2;
    int replication = 2;
    int batchSize = 2;
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    for (int i = 0; i < batchSize; i++) {
        createFileWithNoLocations(TEST_URI + i, numBlocks);
        LoadConfig loadConfig = new LoadConfig(TEST_URI + i, replication, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, true);
        ObjectMapper oMapper = new ObjectMapper();
        Map<String, String> map = oMapper.convertValue(loadConfig, Map.class);
        configs.add(map);
    }
    BatchedJobConfig config = new BatchedJobConfig("Load", configs);
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> assignments = new BatchedJobDefinition().selectExecutors(config, JOB_WORKERS, new SelectExecutorsContext(1, mJobServerContext));
    // Check that we are loading the right number of blocks.
    int totalBlockLoads = 0;
    for (Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask> assignment : assignments) {
        ArrayList<LoadTask> second = (ArrayList<LoadTask>) assignment.getSecond().getJobTaskArgs();
        totalBlockLoads += second.size();
    }
    Assert.assertEquals(numBlocks * replication * batchSize, totalBlockLoads);
}

Example 3 with LoadConfig

use of alluxio.job.plan.load.LoadConfig in project alluxio by Alluxio.

the class DistributedCommandsStatsTest method testCompleteStats.

@Test
public void testCompleteStats() throws Exception {
    final int length = 10;
    FileSystemTestUtils.createByteFile(sFileSystem, "/test", WritePType.THROUGH, length);
    long jobId = sJobMaster.run(new LoadConfig("/test", 1, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, Collections.EMPTY_SET, false));
    JobTestUtils.waitForJobStatus(sJobMaster, jobId, Sets.newHashSet(Status.COMPLETED), TEST_TIMEOUT);
    sJobShell.run("stat", "-v", Long.toString(jobId));
    String[] output = mOutput.toString().split("\n");
    assertEquals(String.format("ID: %s", jobId), output[0]);
    assertEquals(String.format("Name: Load"), output[1]);
    assertTrue(output[2].contains("Description: LoadConfig"));
    assertTrue(output[2].contains("/test"));
    assertEquals("Status: COMPLETED", output[3]);
    assertEquals("Task 0", output[4]);
    assertTrue(output[5].contains("\tWorker: "));
    assertEquals("\tStatus: COMPLETED", output[7]);
    double completedCount = MetricsSystem.getMetricValue(MetricKey.MASTER_JOB_DISTRIBUTED_LOAD_SUCCESS.getName()).getValue();
    double fileCount = MetricsSystem.getMetricValue(MetricKey.MASTER_JOB_DISTRIBUTED_LOAD_FILE_COUNT.getName()).getValue();
    double fileSize = MetricsSystem.getMetricValue(MetricKey.MASTER_JOB_DISTRIBUTED_LOAD_FILE_SIZE.getName()).getValue();
    // Metrics for Migrate job type
    double completedMigrateCount = MetricsSystem.getMetricValue(MetricKey.MASTER_MIGRATE_JOB_SUCCESS.getName()).getValue();
    double completedMigrateFileCount = MetricsSystem.getMetricValue(MetricKey.MASTER_MIGRATE_JOB_FILE_COUNT.getName()).getValue();
    double completedMigrateFileSize = MetricsSystem.getMetricValue(MetricKey.MASTER_MIGRATE_JOB_FILE_SIZE.getName()).getValue();
    // Metrics for Persist job type
    double completedPersistCount = MetricsSystem.getMetricValue(MetricKey.MASTER_ASYNC_PERSIST_SUCCESS.getName()).getValue();
    double completedPersistFileCount = MetricsSystem.getMetricValue(MetricKey.MASTER_ASYNC_PERSIST_FILE_COUNT.getName()).getValue();
    double completedPersistFileSize = MetricsSystem.getMetricValue(MetricKey.MASTER_ASYNC_PERSIST_FILE_SIZE.getName()).getValue();
    // test counters for distributed load on Complete status.
    // distributedLoad operation count equals 1.
    assertEquals(completedCount, 1, 0);
    // file count equals 1.
    assertEquals(fileCount, 1, 0);
    // file size equals $length.
    assertEquals(fileSize, length, 0);
    // test for other job types. Migrate counters, should all be 0.
    assertEquals(completedMigrateCount, 0, 0);
    assertEquals(completedMigrateFileCount, 0, 0);
    assertEquals(completedMigrateFileSize, 0, 0);
    // test AsyncPersist counters, should all be 0.
    assertEquals(completedPersistCount, 0, 0);
    assertEquals(completedPersistFileCount, 0, 0);
    assertEquals(completedPersistFileSize, 0, 0);
}