Examples with PersistConfig alluxio.job.plan.persist.PersistConfig used on opensource projects

Example 1 with PersistConfig

use of alluxio.job.plan.persist.PersistConfig in project alluxio by Alluxio.

the class BatchedJobDefinitionTest method batchPersist.

@Test
public void batchPersist() throws Exception {
    AlluxioURI uri = new AlluxioURI("/test");
    PersistConfig config = new PersistConfig(uri.getPath(), -1, true, "");
    HashSet<Map<String, String>> configs = Sets.newHashSet();
    ObjectMapper oMapper = new ObjectMapper();
    Map<String, String> map = oMapper.convertValue(config, Map.class);
    configs.add(map);
    BatchedJobConfig batchedJobConfig = new BatchedJobConfig("Persist", configs);
    WorkerNetAddress workerNetAddress = new WorkerNetAddress().setDataPort(10);
    WorkerInfo workerInfo = new WorkerInfo().setAddress(workerNetAddress);
    long blockId = 1;
    BlockInfo blockInfo = new BlockInfo().setBlockId(blockId);
    FileBlockInfo fileBlockInfo = new FileBlockInfo().setBlockInfo(blockInfo);
    BlockLocation location = new BlockLocation();
    location.setWorkerAddress(workerNetAddress);
    blockInfo.setLocations(Lists.newArrayList(location));
    FileInfo testFileInfo = new FileInfo();
    testFileInfo.setFileBlockInfos(Lists.newArrayList(fileBlockInfo));
    Mockito.when(mMockFileSystem.getStatus(uri)).thenReturn(new URIStatus(testFileInfo));
    Set<Pair<WorkerInfo, BatchedJobDefinition.BatchedJobTask>> result = new BatchedJobDefinition().selectExecutors(batchedJobConfig, Lists.newArrayList(workerInfo), new SelectExecutorsContext(1, mJobServerContext));
    System.out.println(result);
    Assert.assertNull(result.iterator().next().getSecond().getJobTaskArgs());
    Assert.assertEquals(1, result.size());
    Assert.assertEquals(workerInfo, result.iterator().next().getFirst());
}

Example 2 with PersistConfig

use of alluxio.job.plan.persist.PersistConfig in project alluxio by Alluxio.

the class JobShellTest method runPersistJob.

protected long runPersistJob(String pathStr) throws Exception {
    // write a file in alluxio only
    AlluxioURI filePath = new AlluxioURI(pathStr);
    FileOutStream os = sFileSystem.createFile(filePath, CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build());
    os.write((byte) 0);
    os.write((byte) 1);
    os.close();
    // persist the file
    URIStatus status = sFileSystem.getStatus(filePath);
    return sJobMaster.run(new PersistConfig(pathStr, 1, true, status.getUfsPath()));
}

Example 3 with PersistConfig

use of alluxio.job.plan.persist.PersistConfig in project alluxio by Alluxio.

the class DistributedCommandsStatsTest method testAsyncPersistCancelStats.

@Test
public void testAsyncPersistCancelStats() throws Exception {
    FileSystemTestUtils.createByteFile(sFileSystem, "/testFile", WritePType.THROUGH, 10);
    long jobId = sJobMaster.run(new PersistConfig("/testFile", 0, false, "/testUfsPath"));
    sJobShell.run("cancel", Long.toString(jobId));
    JobTestUtils.waitForJobStatus(sJobMaster, jobId, Sets.newHashSet(Status.CANCELED), TEST_TIMEOUT);
    sJobShell.run("stat", "-v", Long.toString(jobId));
    String[] output = mOutput.toString().split("\n");
    assertEquals(String.format("ID: %s", jobId), output[0]);
    assertEquals(String.format("Name: Persist"), output[1]);
    assertTrue(output[2].contains("Description: PersistConfig"));
    assertTrue(output[2].contains("/testFile"));
    assertEquals("Status: CANCELED", output[3]);
    assertEquals("Task 0", output[4]);
    assertTrue(output[5].contains("\tWorker: "));
    assertEquals("\tStatus: CANCELED", output[6]);
    double cancelledCount = MetricsSystem.getMetricValue(MetricKey.MASTER_ASYNC_PERSIST_CANCEL.getName()).getValue();
    double failedCount = MetricsSystem.getMetricValue(MetricKey.MASTER_ASYNC_PERSIST_FAIL.getName()).getValue();
    double completedCount = MetricsSystem.getMetricValue(MetricKey.MASTER_ASYNC_PERSIST_SUCCESS.getName()).getValue();
    assertEquals(cancelledCount, 1, 0);
    assertEquals(failedCount, 0, 0);
    assertEquals(completedCount, 0, 0);
}