Examples with JobInfo alluxio.job.wire.JobInfo used on opensource projects

Example 6 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class PersistenceTest method retryPersistJobRenameDelete.

/**
 * Tests that a persist file job is retried after the file is renamed and the src directory is
 * deleted.
 */
@Test(timeout = 20000)
public void retryPersistJobRenameDelete() throws Exception {
    UserState s = UserState.Factory.create(ServerConfiguration.global());
    AuthenticatedClientUser.set(s.getUser().getName());
    // Create src file and directory, checking the internal state.
    AlluxioURI alluxioDirSrc = new AlluxioURI("/src");
    mFileSystemMaster.createDirectory(alluxioDirSrc, CreateDirectoryContext.defaults().setWriteType(WriteType.CACHE_THROUGH));
    AlluxioURI alluxioFileSrc = new AlluxioURI("/src/in_alluxio");
    FileInfo info = mFileSystemMaster.createFile(alluxioFileSrc, CreateFileContext.defaults().setWriteType(WriteType.MUST_CACHE));
    Assert.assertEquals(PersistenceState.NOT_PERSISTED.toString(), info.getPersistenceState());
    mFileSystemMaster.completeFile(alluxioFileSrc, CompleteFileContext.defaults());
    // Schedule the async persistence, checking the internal state.
    mFileSystemMaster.scheduleAsyncPersistence(alluxioFileSrc, ScheduleAsyncPersistenceContext.defaults());
    checkPersistenceRequested(alluxioFileSrc);
    // Mock the job service interaction.
    Random random = new Random();
    long jobId = random.nextLong();
    Mockito.when(mMockJobMasterClient.run(any(JobConfig.class))).thenReturn(jobId);
    // Execute the persistence scheduler heartbeat, checking the internal state.
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
    CommonUtils.waitFor("Scheduler heartbeat", (() -> getPersistJobs().size() > 0));
    checkPersistenceInProgress(alluxioFileSrc, jobId);
    // Mock the job service interaction.
    JobInfo jobInfo = createJobInfo(Status.CREATED);
    Mockito.when(mMockJobMasterClient.getJobStatus(Mockito.anyLong())).thenReturn(jobInfo);
    // Execute the persistence checker heartbeat, checking the internal state.
    HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
    CommonUtils.waitFor("Checker heartbeat", (() -> getPersistJobs().size() > 0));
    checkPersistenceInProgress(alluxioFileSrc, jobId);
    // Mock the job service interaction.
    jobInfo = createJobInfo(Status.COMPLETED);
    Mockito.when(mMockJobMasterClient.getJobStatus(Mockito.anyLong())).thenReturn(jobInfo);
    // Create the temporary UFS file.
    {
        Map<Long, PersistJob> persistJobs = getPersistJobs();
        PersistJob job = persistJobs.get(info.getFileId());
        UnderFileSystem ufs = UnderFileSystem.Factory.create(job.getTempUfsPath().toString(), UnderFileSystemConfiguration.defaults(ServerConfiguration.global()));
        UnderFileSystemUtils.touch(ufs, job.getTempUfsPath());
    }
    // Rename the src file before the persist is commited.
    mFileSystemMaster.createDirectory(new AlluxioURI("/dst"), CreateDirectoryContext.defaults().setWriteType(WriteType.CACHE_THROUGH));
    AlluxioURI alluxioFileDst = new AlluxioURI("/dst/in_alluxio");
    mFileSystemMaster.rename(alluxioFileSrc, alluxioFileDst, RenameContext.defaults());
    // Delete the src directory recursively.
    mFileSystemMaster.delete(alluxioDirSrc, DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true)));
    // Repeatedly execute the persistence checker heartbeat, checking the internal state.
    {
        // Execute the persistence checker heartbeat, checking the internal state. This should
        // write the persist file to renamed destination.
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        waitUntilPersisted(alluxioFileDst);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        waitUntilPersisted(alluxioFileDst);
    }
}

Example 7 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class PersistenceTest method retryFailed.

/**
 * Tests that a failed persist job is retried multiple times.
 */
@Test
public void retryFailed() throws Exception {
    // Create a file and check the internal state.
    AlluxioURI testFile = createTestFile();
    FileInfo fileInfo = mFileSystemMaster.getFileInfo(testFile, GET_STATUS_CONTEXT);
    Assert.assertEquals(PersistenceState.NOT_PERSISTED.toString(), fileInfo.getPersistenceState());
    // schedule the async persistence, checking the internal state.
    mFileSystemMaster.scheduleAsyncPersistence(testFile, ScheduleAsyncPersistenceContext.defaults());
    checkPersistenceRequested(testFile);
    // Mock the job service interaction.
    Random random = new Random();
    long jobId = random.nextLong();
    Mockito.when(mMockJobMasterClient.run(any(JobConfig.class))).thenReturn(jobId);
    // Repeatedly execute the persistence checker heartbeat, checking the internal state.
    {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
        checkPersistenceInProgress(testFile, jobId);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
        checkPersistenceInProgress(testFile, jobId);
    }
    // Mock the job service interaction.
    JobInfo jobInfo = createJobInfo(Status.FAILED);
    Mockito.when(mMockJobMasterClient.getJobStatus(Mockito.anyLong())).thenReturn(jobInfo);
    // has been cancelled.
    while (true) {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        checkPersistenceRequested(testFile);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
        if (getPersistJobs().size() != 0) {
            checkPersistenceInProgress(testFile, jobId);
        } else {
            checkEmpty();
            break;
        }
        CommonUtils.sleepMs(100);
    }
    fileInfo = mFileSystemMaster.getFileInfo(testFile, GET_STATUS_CONTEXT);
    Assert.assertEquals(PersistenceState.NOT_PERSISTED.toString(), fileInfo.getPersistenceState());
}

Example 8 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class PersistenceTest method successfulAsyncPersistence.

/**
 * Tests the progression of a successful persist job.
 */
@Test
public void successfulAsyncPersistence() throws Exception {
    // Create a file and check the internal state.
    AlluxioURI testFile = createTestFile();
    FileInfo fileInfo = mFileSystemMaster.getFileInfo(testFile, GET_STATUS_CONTEXT);
    Assert.assertEquals(PersistenceState.NOT_PERSISTED.toString(), fileInfo.getPersistenceState());
    // schedule the async persistence, checking the internal state.
    mFileSystemMaster.scheduleAsyncPersistence(testFile, ScheduleAsyncPersistenceContext.defaults());
    checkPersistenceRequested(testFile);
    // Mock the job service interaction.
    Random random = new Random();
    long jobId = random.nextLong();
    Mockito.when(mMockJobMasterClient.run(any(JobConfig.class))).thenReturn(jobId);
    // Repeatedly execute the persistence checker heartbeat, checking the internal state.
    {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
        checkPersistenceInProgress(testFile, jobId);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
        checkPersistenceInProgress(testFile, jobId);
    }
    // Mock the job service interaction.
    JobInfo jobInfo = createJobInfo(Status.CREATED);
    Mockito.when(mMockJobMasterClient.getJobStatus(Mockito.anyLong())).thenReturn(jobInfo);
    // Repeatedly execute the persistence checker heartbeat, checking the internal state.
    {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        checkPersistenceInProgress(testFile, jobId);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        checkPersistenceInProgress(testFile, jobId);
    }
    // Mock the job service interaction.
    jobInfo = createJobInfo(Status.RUNNING);
    Mockito.when(mMockJobMasterClient.getJobStatus(Mockito.anyLong())).thenReturn(jobInfo);
    // Repeatedly execute the persistence checker heartbeat, checking the internal state.
    {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        checkPersistenceInProgress(testFile, jobId);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        checkPersistenceInProgress(testFile, jobId);
    }
    // Mock the job service interaction.
    jobInfo = createJobInfo(Status.COMPLETED);
    Mockito.when(mMockJobMasterClient.getJobStatus(Mockito.anyLong())).thenReturn(jobInfo);
    {
        // Create the temporary UFS file.
        fileInfo = mFileSystemMaster.getFileInfo(testFile, GET_STATUS_CONTEXT);
        Map<Long, PersistJob> persistJobs = getPersistJobs();
        PersistJob job = persistJobs.get(fileInfo.getFileId());
        UnderFileSystem ufs = UnderFileSystem.Factory.create(job.getTempUfsPath().toString(), UnderFileSystemConfiguration.defaults(ServerConfiguration.global()));
        UnderFileSystemUtils.touch(ufs, job.getTempUfsPath());
    }
    // Repeatedly execute the persistence checker heartbeat, checking the internal state.
    {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        waitUntilPersisted(testFile);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        waitUntilPersisted(testFile);
    }
}

Example 9 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class PersistenceTest method noRetryCanceled.

/**
 * Tests that a canceled persist job is not retried.
 */
@Test
public void noRetryCanceled() throws Exception {
    // Create a file and check the internal state.
    AlluxioURI testFile = createTestFile();
    FileInfo fileInfo = mFileSystemMaster.getFileInfo(testFile, GET_STATUS_CONTEXT);
    Assert.assertEquals(PersistenceState.NOT_PERSISTED.toString(), fileInfo.getPersistenceState());
    // schedule the async persistence, checking the internal state.
    mFileSystemMaster.scheduleAsyncPersistence(testFile, ScheduleAsyncPersistenceContext.defaults());
    checkPersistenceRequested(testFile);
    // Mock the job service interaction.
    Random random = new Random();
    long jobId = random.nextLong();
    Mockito.when(mMockJobMasterClient.run(any(JobConfig.class))).thenReturn(jobId);
    // Repeatedly execute the persistence checker heartbeat, checking the internal state.
    {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
        checkPersistenceInProgress(testFile, jobId);
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_SCHEDULER);
        checkPersistenceInProgress(testFile, jobId);
    }
    // Mock the job service interaction.
    JobInfo jobInfo = createJobInfo(Status.CANCELED);
    Mockito.when(mMockJobMasterClient.getJobStatus(Mockito.anyLong())).thenReturn(jobInfo);
    // Execute the persistence checker heartbeat, checking the internal state.
    {
        HeartbeatScheduler.execute(HeartbeatContext.MASTER_PERSISTENCE_CHECKER);
        checkEmpty();
    }
}

Example 10 with JobInfo

use of alluxio.job.wire.JobInfo in project alluxio by Alluxio.

the class JobMaster method getStatus.

/**
 * Gets information of the given job id (verbose = True).
 *
 * @param jobId the id of the job
 * @return the job information
 * @throws JobDoesNotExistException if the job does not exist
 */
public JobInfo getStatus(long jobId) throws JobDoesNotExistException {
    try (JobMasterAuditContext auditContext = createAuditContext("getStatus")) {
        auditContext.setJobId(jobId);
        JobInfo jobInfo = getStatus(jobId, true);
        if (jobInfo != null) {
            auditContext.setJobName(jobInfo.getName());
            auditContext.setSucceeded(true);
        }
        return jobInfo;
    }
}