Examples with JobMaster alluxio.master.job.JobMaster used on opensource projects

Example 1 with JobMaster

use of alluxio.master.job.JobMaster in project alluxio by Alluxio.

the class TableMasterJournalIntegrationTest method journalTransformDb.

@Test
public void journalTransformDb() throws Exception {
    LocalAlluxioCluster mCluster = sClusterResource.get();
    TableMaster tableMaster = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(TableMaster.class);
    LocalAlluxioJobCluster jobCluster = new LocalAlluxioJobCluster();
    jobCluster.start();
    JobMaster jobMaster = jobCluster.getMaster().getJobMaster();
    genTable(1, 2, true);
    tableMaster.attachDatabase(TestUdbFactory.TYPE, "connect", DB_NAME, DB_NAME, Collections.emptyMap(), false);
    List<String> tables = tableMaster.getAllTables(DB_NAME);
    assertFalse(tables.isEmpty());
    // all partitions are not transformed, so baselayout is the same as layout
    String tableName = tables.get(0);
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() == partition.getLayout()));
    long jobid = tableMaster.transformTable(DB_NAME, tableName, null);
    assertNotEquals(0, jobid);
    JobTestUtils.waitForJobStatus(jobMaster, jobid, ImmutableSet.of(Status.COMPLETED, Status.CANCELED, Status.FAILED));
    final JobInfo status = jobMaster.getStatus(jobid);
    assertEquals("", status.getErrorMessage());
    assertEquals(Status.COMPLETED, status.getStatus());
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_TABLE_TRANSFORMATION_MONITOR);
    // all partitions are transformed, so baselayout should be different as layout
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() != partition.getLayout()));
    restartMaster();
    genTable(1, 4, true);
    TableMaster tableMasterRestart = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(TableMaster.class);
    Table table = tableMaster.getTable(DB_NAME, tableName);
    // all partitions remain transformed
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> partition.getBaseLayout() != partition.getLayout()));
    tableMasterRestart.syncDatabase(DB_NAME);
    // The first two partitions should remain transformed, the new partitions are not transformed
    assertTrue(tableMaster.getTable(DB_NAME, tableName).getPartitions().stream().allMatch(partition -> (partition.getSpec().endsWith("0") || partition.getSpec().endsWith("1")) == (partition.getBaseLayout() != partition.getLayout())));
}

Example 2 with JobMaster

use of alluxio.master.job.JobMaster in project alluxio by Alluxio.

the class PersistIntegrationTest method persistOnlyCompleteFiles.

@Test(timeout = 30000)
public void persistOnlyCompleteFiles() throws Exception {
    AlluxioURI path = new AlluxioURI("/" + CommonUtils.randomAlphaNumString(10));
    // Create file, but do not complete
    FileOutStream os = mFileSystem.createFile(path, CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).setMode(TEST_MODE.toProto()).build());
    URIStatus status = mFileSystem.getStatus(path);
    // Generate a temporary path to be used by the persist job.
    String tempUfsPath = PathUtils.temporaryFileName(System.currentTimeMillis(), status.getUfsPath());
    JobMaster jobMaster = mLocalAlluxioJobCluster.getMaster().getJobMaster();
    // Run persist job on incomplete file (expected to fail)
    long failId = jobMaster.run(new PersistConfig(path.toString(), status.getMountId(), false, tempUfsPath));
    CommonUtils.waitFor("Wait for persist job to complete", () -> {
        try {
            JobInfo jobInfo = jobMaster.getStatus(failId);
            Assert.assertNotEquals("Persist should not succeed for incomplete file", COMPLETED, jobInfo.getStatus());
            if (jobInfo.getStatus() == FAILED) {
                // failed job is expected
                Assert.assertTrue("Failure expected to be about incomplete files", jobInfo.getErrorMessage().toLowerCase().contains("incomplete"));
                return true;
            }
            return false;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }, WaitForOptions.defaults().setTimeoutMs(10 * Constants.SECOND_MS).setInterval(100));
    // close the file to allow persist to happen
    os.close();
    // Run persist job on complete file (expected to succeed)
    long successId = jobMaster.run(new PersistConfig(path.toString(), status.getMountId(), false, tempUfsPath));
    CommonUtils.waitFor("Wait for persist job to complete", () -> {
        try {
            JobInfo jobInfo = jobMaster.getStatus(successId);
            Assert.assertNotEquals("Persist should not fail", FAILED, jobInfo.getStatus());
            return jobInfo.getStatus() == COMPLETED;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }, WaitForOptions.defaults().setTimeoutMs(10 * Constants.SECOND_MS).setInterval(100));
}