Examples with ApplicationHistoryData org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData used on opensource projects

Example 1 with ApplicationHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData in project hadoop by apache.

the class FileSystemApplicationHistoryStore method getApplication.

@Override
public ApplicationHistoryData getApplication(ApplicationId appId) throws IOException {
    HistoryFileReader hfReader = getHistoryFileReader(appId);
    try {
        boolean readStartData = false;
        boolean readFinishData = false;
        ApplicationHistoryData historyData = ApplicationHistoryData.newInstance(appId, null, null, null, null, Long.MIN_VALUE, Long.MIN_VALUE, Long.MAX_VALUE, null, FinalApplicationStatus.UNDEFINED, null);
        while ((!readStartData || !readFinishData) && hfReader.hasNext()) {
            HistoryFileReader.Entry entry = hfReader.next();
            if (entry.key.id.equals(appId.toString())) {
                if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
                    ApplicationStartData startData = parseApplicationStartData(entry.value);
                    mergeApplicationHistoryData(historyData, startData);
                    readStartData = true;
                } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
                    ApplicationFinishData finishData = parseApplicationFinishData(entry.value);
                    mergeApplicationHistoryData(historyData, finishData);
                    readFinishData = true;
                }
            }
        }
        if (!readStartData && !readFinishData) {
            return null;
        }
        if (!readStartData) {
            LOG.warn("Start information is missing for application " + appId);
        }
        if (!readFinishData) {
            LOG.warn("Finish information is missing for application " + appId);
        }
        LOG.info("Completed reading history information of application " + appId);
        return historyData;
    } catch (IOException e) {
        LOG.error("Error when reading history file of application " + appId, e);
        throw e;
    } finally {
        hfReader.close();
    }
}

Example 2 with ApplicationHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData in project hadoop by apache.

the class TestRMApplicationHistoryWriter method testWriteApplication.

@Test
public void testWriteApplication() throws Exception {
    RMApp app = createRMApp(ApplicationId.newInstance(0, 1));
    writer.applicationStarted(app);
    ApplicationHistoryData appHD = null;
    for (int i = 0; i < MAX_RETRIES; ++i) {
        appHD = store.getApplication(ApplicationId.newInstance(0, 1));
        if (appHD != null) {
            break;
        } else {
            Thread.sleep(100);
        }
    }
    Assert.assertNotNull(appHD);
    Assert.assertEquals("test app", appHD.getApplicationName());
    Assert.assertEquals("test app type", appHD.getApplicationType());
    Assert.assertEquals("test user", appHD.getUser());
    Assert.assertEquals("test queue", appHD.getQueue());
    Assert.assertEquals(0L, appHD.getSubmitTime());
    Assert.assertEquals(1L, appHD.getStartTime());
    writer.applicationFinished(app, RMAppState.FINISHED);
    for (int i = 0; i < MAX_RETRIES; ++i) {
        appHD = store.getApplication(ApplicationId.newInstance(0, 1));
        if (appHD.getYarnApplicationState() != null) {
            break;
        } else {
            Thread.sleep(100);
        }
    }
    Assert.assertEquals(2L, appHD.getFinishTime());
    Assert.assertEquals("test diagnostics info", appHD.getDiagnosticsInfo());
    Assert.assertEquals(FinalApplicationStatus.UNDEFINED, appHD.getFinalApplicationStatus());
    Assert.assertEquals(YarnApplicationState.FINISHED, appHD.getYarnApplicationState());
}

Example 3 with ApplicationHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData in project hadoop by apache.

the class FileSystemApplicationHistoryStore method getAllApplications.

@Override
public Map<ApplicationId, ApplicationHistoryData> getAllApplications() throws IOException {
    Map<ApplicationId, ApplicationHistoryData> historyDataMap = new HashMap<ApplicationId, ApplicationHistoryData>();
    FileStatus[] files = fs.listStatus(rootDirPath);
    for (FileStatus file : files) {
        ApplicationId appId = ApplicationId.fromString(file.getPath().getName());
        try {
            ApplicationHistoryData historyData = getApplication(appId);
            if (historyData != null) {
                historyDataMap.put(appId, historyData);
            }
        } catch (IOException e) {
            // Eat the exception not to disturb the getting the next
            // ApplicationHistoryData
            LOG.error("History information of application " + appId + " is not included into the result due to the exception", e);
        }
    }
    return historyDataMap;
}

Example 4 with ApplicationHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData in project hadoop by apache.

the class MemoryApplicationHistoryStore method applicationFinished.

@Override
public void applicationFinished(ApplicationFinishData appFinish) throws IOException {
    ApplicationHistoryData data = applicationData.get(appFinish.getApplicationId());
    if (data == null) {
        throw new IOException("The finish information of application " + appFinish.getApplicationId() + " is stored before the start" + " information.");
    }
    // the finish information is already recorded
    if (data.getYarnApplicationState() != null) {
        throw new IOException("The finish information of application " + appFinish.getApplicationId() + " is already stored.");
    }
    data.setFinishTime(appFinish.getFinishTime());
    data.setDiagnosticsInfo(appFinish.getDiagnosticsInfo());
    data.setFinalApplicationStatus(appFinish.getFinalApplicationStatus());
    data.setYarnApplicationState(appFinish.getYarnApplicationState());
}

Example 5 with ApplicationHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData in project hadoop by apache.

the class TestFileSystemApplicationHistoryStore method testReadHistoryData.

@SuppressWarnings("deprecation")
private void testReadHistoryData(int num, boolean missingContainer, boolean missingApplicationAttempt) throws IOException {
    // read application history data
    Assert.assertEquals(num, store.getAllApplications().size());
    for (int i = 1; i <= num; ++i) {
        ApplicationId appId = ApplicationId.newInstance(0, i);
        ApplicationHistoryData appData = store.getApplication(appId);
        Assert.assertNotNull(appData);
        Assert.assertEquals(appId.toString(), appData.getApplicationName());
        Assert.assertEquals(appId.toString(), appData.getDiagnosticsInfo());
        // read application attempt history data
        Assert.assertEquals(num, store.getApplicationAttempts(appId).size());
        for (int j = 1; j <= num; ++j) {
            ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, j);
            ApplicationAttemptHistoryData attemptData = store.getApplicationAttempt(appAttemptId);
            Assert.assertNotNull(attemptData);
            Assert.assertEquals(appAttemptId.toString(), attemptData.getHost());
            if (missingApplicationAttempt && j == num) {
                Assert.assertNull(attemptData.getDiagnosticsInfo());
                continue;
            } else {
                Assert.assertEquals(appAttemptId.toString(), attemptData.getDiagnosticsInfo());
            }
            // read container history data
            Assert.assertEquals(num, store.getContainers(appAttemptId).size());
            for (int k = 1; k <= num; ++k) {
                ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
                ContainerHistoryData containerData = store.getContainer(containerId);
                Assert.assertNotNull(containerData);
                Assert.assertEquals(Priority.newInstance(containerId.getId()), containerData.getPriority());
                if (missingContainer && k == num) {
                    Assert.assertNull(containerData.getDiagnosticsInfo());
                } else {
                    Assert.assertEquals(containerId.toString(), containerData.getDiagnosticsInfo());
                }
            }
            ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId);
            Assert.assertNotNull(masterContainer);
            Assert.assertEquals(ContainerId.newContainerId(appAttemptId, 1), masterContainer.getContainerId());
        }
    }
}