Examples with Error com.microsoft.azure.hdinsight.common.MessageInfoType.Error used on opensource projects

Example 1 with Error

use of com.microsoft.azure.hdinsight.common.MessageInfoType.Error in project azure-tools-for-java by Microsoft.

the class SparkBatchJob method getSubmissionLog.

@Override
@NotNull
public Observable<SparkLogLine> getSubmissionLog() {
    if (getConnectUri() == null) {
        return Observable.error(new SparkJobNotConfiguredException("Can't get Spark job connection URI, " + "please configure Spark cluster which the Spark job will be submitted."));
    }
    // Those lines are carried per response,
    // if there is no value followed, the line should not be sent to console
    final Set<String> ignoredEmptyLines = new HashSet<>(Arrays.asList("stdout:", "stderr:", "yarn diagnostics:"));
    return Observable.create(ob -> {
        try {
            final int maxLinesPerGet = 128;
            int linesGot;
            boolean isFetching = true;
            while (isFetching) {
                final int start = nextLivyLogOffset;
                final boolean isAppIdAllocated = !this.getSparkJobApplicationIdObservable().isEmpty().toBlocking().lastOrDefault(true);
                final String logUrl = String.format("%s/%d/log?from=%d&size=%d", this.getConnectUri().toString(), batchId, start, maxLinesPerGet);
                final HttpResponse httpResponse = this.getSubmission().getHttpResponseViaGet(logUrl);
                final SparkJobLog sparkJobLog = ObjectConvertUtils.convertJsonToObject(httpResponse.getMessage(), SparkJobLog.class).orElseThrow(() -> new UnknownServiceException("Bad spark log response: " + httpResponse.getMessage()));
                synchronized (livyLogOffsetLock) {
                    if (start != nextLivyLogOffset) {
                        // The offset is moved by another fetching thread, re-do it with new offset
                        continue;
                    }
                    // To subscriber
                    sparkJobLog.getLog().stream().filter(line -> !ignoredEmptyLines.contains(line.trim().toLowerCase())).forEach(line -> ob.onNext(new SparkLogLine(LIVY, Log, line)));
                    linesGot = sparkJobLog.getLog().size();
                    nextLivyLogOffset += linesGot;
                }
                // Retry interval
                if (linesGot == 0) {
                    isFetching = "starting".equals(this.getState()) && !isAppIdAllocated;
                    sleep(TimeUnit.SECONDS.toMillis(this.getDelaySeconds()));
                }
            }
        } catch (final IOException ex) {
            ob.onNext(new SparkLogLine(TOOL, Error, ex.getMessage()));
        } catch (final InterruptedException ignored) {
        } finally {
            ob.onCompleted();
        }
    });
}