Examples with SparkJobLog com.microsoft.azure.hdinsight.spark.common.SparkJobLog used on opensource projects

Example 1 with SparkJobLog

use of com.microsoft.azure.hdinsight.spark.common.SparkJobLog in project azure-tools-for-java by Microsoft.

the class SparkSubmitHelper method printoutJobLog.

private int printoutJobLog(/*Project project,*/
int id, int from_index, IClusterDetail clusterDetail) throws IOException {
    HttpResponse httpResponse = SparkBatchSubmission.getInstance().getBatchJobFullLog(clusterDetail.getConnectionUrl() + "/livy/batches", id);
    sparkJobLog = new Gson().fromJson(httpResponse.getMessage(), new TypeToken<SparkJobLog>() {
    }.getType());
    if (!HDInsightUtil.getSparkSubmissionToolWindowView().getJobStatusManager().isApplicationGenerated()) {
        String sparkLogs = Joiner.on("").join(sparkJobLog.getLog());
        String applicationId = getApplicationIdFromYarnLog(sparkLogs);
        if (applicationId != null) {
            HDInsightUtil.getSparkSubmissionToolWindowView().setBrowserButtonState(true);
            HDInsightUtil.getSparkSubmissionToolWindowView().getJobStatusManager().setApplicationIdGenerated();
            HDInsightUtil.getSparkSubmissionToolWindowView().getJobStatusManager().setApplicationId(applicationId);
        }
    }
    int counter = 0;
    if (sparkJobLog.getLog().size() > 0) {
        for (String line : sparkJobLog.getLog()) {
            if (counter >= from_index && !StringHelper.isNullOrWhiteSpace(line)) {
                HDInsightUtil.getSparkSubmissionToolWindowView().setInfo(line, true);
            }
            counter++;
        }
    }
    return sparkJobLog.getTotal();
}