Examples with ApplicationAttemptHistoryData org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData used on opensource projects

Example 1 with ApplicationAttemptHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData in project hadoop by apache.

the class TestMemoryApplicationHistoryStore method testReadWriteApplicationAttemptHistory.

@Test
public void testReadWriteApplicationAttemptHistory() throws Exception {
    // Out of order
    ApplicationId appId = ApplicationId.newInstance(0, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    try {
        writeApplicationAttemptFinishData(appAttemptId);
        Assert.fail();
    } catch (IOException e) {
        Assert.assertTrue(e.getMessage().contains("is stored before the start information"));
    }
    // Normal
    int numAppAttempts = 5;
    writeApplicationStartData(appId);
    for (int i = 1; i <= numAppAttempts; ++i) {
        appAttemptId = ApplicationAttemptId.newInstance(appId, i);
        writeApplicationAttemptStartData(appAttemptId);
        writeApplicationAttemptFinishData(appAttemptId);
    }
    Assert.assertEquals(numAppAttempts, store.getApplicationAttempts(appId).size());
    for (int i = 1; i <= numAppAttempts; ++i) {
        appAttemptId = ApplicationAttemptId.newInstance(appId, i);
        ApplicationAttemptHistoryData data = store.getApplicationAttempt(appAttemptId);
        Assert.assertNotNull(data);
        Assert.assertEquals(appAttemptId.toString(), data.getHost());
        Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
    }
    writeApplicationFinishData(appId);
    // Write again
    appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    try {
        writeApplicationAttemptStartData(appAttemptId);
        Assert.fail();
    } catch (IOException e) {
        Assert.assertTrue(e.getMessage().contains("is already stored"));
    }
    try {
        writeApplicationAttemptFinishData(appAttemptId);
        Assert.fail();
    } catch (IOException e) {
        Assert.assertTrue(e.getMessage().contains("is already stored"));
    }
}

Example 2 with ApplicationAttemptHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData in project hadoop by apache.

the class FileSystemApplicationHistoryStore method getApplicationAttempts.

@Override
public Map<ApplicationAttemptId, ApplicationAttemptHistoryData> getApplicationAttempts(ApplicationId appId) throws IOException {
    Map<ApplicationAttemptId, ApplicationAttemptHistoryData> historyDataMap = new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
    HistoryFileReader hfReader = getHistoryFileReader(appId);
    try {
        while (hfReader.hasNext()) {
            HistoryFileReader.Entry entry = hfReader.next();
            if (entry.key.id.startsWith(ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
                ApplicationAttemptId appAttemptId = ApplicationAttemptId.fromString(entry.key.id);
                if (appAttemptId.getApplicationId().equals(appId)) {
                    ApplicationAttemptHistoryData historyData = historyDataMap.get(appAttemptId);
                    if (historyData == null) {
                        historyData = ApplicationAttemptHistoryData.newInstance(appAttemptId, null, -1, null, null, null, FinalApplicationStatus.UNDEFINED, null);
                        historyDataMap.put(appAttemptId, historyData);
                    }
                    if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
                        mergeApplicationAttemptHistoryData(historyData, parseApplicationAttemptStartData(entry.value));
                    } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
                        mergeApplicationAttemptHistoryData(historyData, parseApplicationAttemptFinishData(entry.value));
                    }
                }
            }
        }
        LOG.info("Completed reading history information of all application" + " attempts of application " + appId);
    } catch (IOException e) {
        LOG.info("Error when reading history information of some application" + " attempts of application " + appId);
    } finally {
        hfReader.close();
    }
    return historyDataMap;
}

Example 3 with ApplicationAttemptHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData in project hadoop by apache.

the class FileSystemApplicationHistoryStore method getApplicationAttempt.

@Override
public ApplicationAttemptHistoryData getApplicationAttempt(ApplicationAttemptId appAttemptId) throws IOException {
    HistoryFileReader hfReader = getHistoryFileReader(appAttemptId.getApplicationId());
    try {
        boolean readStartData = false;
        boolean readFinishData = false;
        ApplicationAttemptHistoryData historyData = ApplicationAttemptHistoryData.newInstance(appAttemptId, null, -1, null, null, null, FinalApplicationStatus.UNDEFINED, null);
        while ((!readStartData || !readFinishData) && hfReader.hasNext()) {
            HistoryFileReader.Entry entry = hfReader.next();
            if (entry.key.id.equals(appAttemptId.toString())) {
                if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
                    ApplicationAttemptStartData startData = parseApplicationAttemptStartData(entry.value);
                    mergeApplicationAttemptHistoryData(historyData, startData);
                    readStartData = true;
                } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
                    ApplicationAttemptFinishData finishData = parseApplicationAttemptFinishData(entry.value);
                    mergeApplicationAttemptHistoryData(historyData, finishData);
                    readFinishData = true;
                }
            }
        }
        if (!readStartData && !readFinishData) {
            return null;
        }
        if (!readStartData) {
            LOG.warn("Start information is missing for application attempt " + appAttemptId);
        }
        if (!readFinishData) {
            LOG.warn("Finish information is missing for application attempt " + appAttemptId);
        }
        LOG.info("Completed reading history information of application attempt " + appAttemptId);
        return historyData;
    } catch (IOException e) {
        LOG.error("Error when reading history file of application attempt" + appAttemptId, e);
        throw e;
    } finally {
        hfReader.close();
    }
}

Example 4 with ApplicationAttemptHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData in project hadoop by apache.

the class ApplicationHistoryManagerImpl method getLastAttempt.

private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId) throws IOException {
    Map<ApplicationAttemptId, ApplicationAttemptHistoryData> attempts = historyStore.getApplicationAttempts(appId);
    ApplicationAttemptId prevMaxAttemptId = null;
    for (ApplicationAttemptId attemptId : attempts.keySet()) {
        if (prevMaxAttemptId == null) {
            prevMaxAttemptId = attemptId;
        } else {
            if (prevMaxAttemptId.getAttemptId() < attemptId.getAttemptId()) {
                prevMaxAttemptId = attemptId;
            }
        }
    }
    return attempts.get(prevMaxAttemptId);
}

Example 5 with ApplicationAttemptHistoryData

use of org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData in project hadoop by apache.

the class ApplicationHistoryManagerImpl method convertToApplicationReport.

private ApplicationReport convertToApplicationReport(ApplicationHistoryData appHistory) throws IOException {
    ApplicationAttemptId currentApplicationAttemptId = null;
    String trackingUrl = UNAVAILABLE;
    String host = UNAVAILABLE;
    int rpcPort = -1;
    ApplicationAttemptHistoryData lastAttempt = getLastAttempt(appHistory.getApplicationId());
    if (lastAttempt != null) {
        currentApplicationAttemptId = lastAttempt.getApplicationAttemptId();
        trackingUrl = lastAttempt.getTrackingURL();
        host = lastAttempt.getHost();
        rpcPort = lastAttempt.getRPCPort();
    }
    return ApplicationReport.newInstance(appHistory.getApplicationId(), currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(), appHistory.getApplicationName(), host, rpcPort, null, appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(), trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(), null, "", 100, appHistory.getApplicationType(), null);
}