Examples with JobStartEventLog com.microsoft.azure.hdinsight.sdk.rest.spark.event.JobStartEventLog used on opensource projects

Example 1 with JobStartEventLog

use of com.microsoft.azure.hdinsight.sdk.rest.spark.event.JobStartEventLog in project azure-tools-for-java by Microsoft.

the class SparkRestUtil method getSparkEventLogs.

public static List<JobStartEventLog> getSparkEventLogs(@NotNull ApplicationKey key) throws HDIException, IOException {
    String url = String.format("%s/logs", key.getAppId());
    String eventLogsPath = String.format("%s/SparkEventLogs/%s/eventLogs.zip", HDInsightLoader.getHDInsightHelper().getPluginRootPath(), key.getAppId());
    File file = new File(eventLogsPath);
    HttpEntity entity = getSparkRestEntity(key.getClusterDetails(), url);
    InputStream inputStream = entity.getContent();
    FileUtils.copyInputStreamToFile(inputStream, file);
    IOUtils.closeQuietly(inputStream);
    ZipFile zipFile = new ZipFile(file);
    List<? extends ZipEntry> entities = Collections.list(zipFile.entries());
    // every application has an attempt in event log
    // and the entity name should be in formation "{appId}_{attemptId}"
    String entityName = String.format("%s_%s", key.getAppId(), entities.size());
    ZipEntry lastEntity = zipFile.getEntry(entityName);
    if (lastEntity == null) {
        throw new HDIException(String.format("No Spark event log entity found for app: %s", key.getAppId()));
    }
    InputStream zipFileInputStream = zipFile.getInputStream(lastEntity);
    String entityContent = IOUtils.toString(zipFileInputStream, Charset.forName("utf-8"));
    String[] lines = entityContent.split("\n");
    List<JobStartEventLog> jobStartEvents = Arrays.stream(lines).filter(line -> {
        JSONObject jsonObject = new JSONObject(line);
        String eventName = jsonObject.getString("Event");
        return eventName.equalsIgnoreCase("SparkListenerJobStart");
    }).map(oneLine -> ObjectConvertUtils.convertToObjectQuietly(oneLine, JobStartEventLog.class)).filter(Objects::nonNull).collect(Collectors.toList());
    return jobStartEvents;
}