Examples with SparkLogLine com.microsoft.azure.hdinsight.spark.common.log.SparkLogLine used on opensource projects

Example 6 with SparkLogLine

use of com.microsoft.azure.hdinsight.spark.common.log.SparkLogLine in project azure-tools-for-java by Microsoft.

the class SparkBatchJob method getSubmissionLog.

@Override
@NotNull
public Observable<SparkLogLine> getSubmissionLog() {
    if (getConnectUri() == null) {
        return Observable.error(new SparkJobNotConfiguredException("Can't get Spark job connection URI, " + "please configure Spark cluster which the Spark job will be submitted."));
    }
    // Those lines are carried per response,
    // if there is no value followed, the line should not be sent to console
    final Set<String> ignoredEmptyLines = new HashSet<>(Arrays.asList("stdout:", "stderr:", "yarn diagnostics:"));
    return Observable.create(ob -> {
        try {
            final int maxLinesPerGet = 128;
            int linesGot;
            boolean isFetching = true;
            while (isFetching) {
                final int start = nextLivyLogOffset;
                final boolean isAppIdAllocated = !this.getSparkJobApplicationIdObservable().isEmpty().toBlocking().lastOrDefault(true);
                final String logUrl = String.format("%s/%d/log?from=%d&size=%d", this.getConnectUri().toString(), batchId, start, maxLinesPerGet);
                final HttpResponse httpResponse = this.getSubmission().getHttpResponseViaGet(logUrl);
                final SparkJobLog sparkJobLog = ObjectConvertUtils.convertJsonToObject(httpResponse.getMessage(), SparkJobLog.class).orElseThrow(() -> new UnknownServiceException("Bad spark log response: " + httpResponse.getMessage()));
                synchronized (livyLogOffsetLock) {
                    if (start != nextLivyLogOffset) {
                        // The offset is moved by another fetching thread, re-do it with new offset
                        continue;
                    }
                    // To subscriber
                    sparkJobLog.getLog().stream().filter(line -> !ignoredEmptyLines.contains(line.trim().toLowerCase())).forEach(line -> ob.onNext(new SparkLogLine(LIVY, Log, line)));
                    linesGot = sparkJobLog.getLog().size();
                    nextLivyLogOffset += linesGot;
                }
                // Retry interval
                if (linesGot == 0) {
                    isFetching = "starting".equals(this.getState()) && !isAppIdAllocated;
                    sleep(TimeUnit.SECONDS.toMillis(this.getDelaySeconds()));
                }
            }
        } catch (final IOException ex) {
            ob.onNext(new SparkLogLine(TOOL, Error, ex.getMessage()));
        } catch (final InterruptedException ignored) {
        } finally {
            ob.onCompleted();
        }
    });
}

Example 7 with SparkLogLine

use of com.microsoft.azure.hdinsight.spark.common.log.SparkLogLine in project azure-tools-for-java by Microsoft.

the class WebHDFSDeploy method deploy.

@Override
public Observable<String> deploy(File src, Observer<SparkLogLine> logSubject) {
    // three steps to upload via webhdfs
    // 1.put request to create new dir
    // 2.put request to get 307 redirect uri from response
    // 3.put redirect request with file content as setEntity
    final URI dest = getUploadDir();
    final HttpPut req = new HttpPut(dest.toString());
    return http.request(req, null, this.createDirReqParams, null).doOnNext(resp -> {
        if (resp.getStatusLine().getStatusCode() != 200) {
            Exceptions.propagate(new UnknownServiceException("Can not create directory to save artifact using webHDFS storage type"));
        }
    }).map(ignored -> new HttpPut(dest.resolve(src.getName()).toString())).flatMap(put -> http.request(put, null, this.uploadReqParams, null)).map(resp -> resp.getFirstHeader("Location").getValue()).doOnNext(redirectedUri -> {
        if (StringUtils.isBlank(redirectedUri)) {
            Exceptions.propagate(new UnknownServiceException("Can not get valid redirect uri using webHDFS storage type"));
        }
    }).map(HttpPut::new).flatMap(put -> {
        try {
            InputStreamEntity reqEntity = new InputStreamEntity(new FileInputStream(src), -1, ContentType.APPLICATION_OCTET_STREAM);
            reqEntity.setChunked(true);
            return http.request(put, new BufferedHttpEntity(reqEntity), URLEncodedUtils.parse(put.getURI(), "UTF-8"), null);
        } catch (IOException ex) {
            throw new RuntimeException(new IllegalArgumentException("Can not get local artifact when uploading" + ex.toString()));
        }
    }).map(ignored -> {
        try {
            return getArtifactUploadedPath(dest.resolve(src.getName()).toString());
        } catch (final URISyntaxException ex) {
            throw new RuntimeException(new IllegalArgumentException("Can not get valid artifact upload path" + ex.toString()));
        }
    });
}

Example 8 with SparkLogLine

use of com.microsoft.azure.hdinsight.spark.common.log.SparkLogLine in project azure-tools-for-java by Microsoft.

the class Session method deploy.

/*
     * Observable APIs, all IO operations
     */
public Observable<Session> deploy() {
    final Deployable deployDelegate = getDeploy();
    if (deployDelegate == null) {
        return Observable.just(this);
    }
    return Observable.from(getArtifactsToDeploy()).doOnNext(artifactPath -> ctrlSubject.onNext(new SparkLogLine(TOOL, Info, "Start uploading artifact " + artifactPath))).flatMap(artifactPath -> deployDelegate.deploy(new File(artifactPath), ctrlSubject)).doOnNext(uri -> ctrlSubject.onNext(new SparkLogLine(TOOL, Info, "Uploaded to " + uri))).toList().onErrorResumeNext(err -> {
        ctrlSubject.onNext(new SparkLogLine(TOOL, Warning, "Failed to upload artifact: " + err));
        ctrlSubject.onNext(new SparkLogLine(TOOL, Warning, "Try to start interactive session without those artifacts dependency..."));
        return Observable.empty();
    }).map(uploadedUris -> {
        this.createParameters.uploadedArtifactsUris.addAll(uploadedUris);
        return this;
    }).defaultIfEmpty(this);
}

Example 9 with SparkLogLine

use of com.microsoft.azure.hdinsight.spark.common.log.SparkLogLine in project azure-tools-for-java by Microsoft.

the class CosmosServerlessSparkBatchJob method getSubmissionLog.

@NotNull
@Override
public Observable<SparkLogLine> getSubmissionLog() {
    final ImmutableSet<String> ignoredEmptyLines = ImmutableSet.of("stdout:", "stderr:", "yarn diagnostics:");
    final int GET_LIVY_URL_REPEAT_DELAY_MILLISECONDS = 3000;
    final int MAX_LOG_LINES_PER_REQUEST = 128;
    final int GET_LOG_REPEAT_DELAY_MILLISECONDS = 1000;
    // We need to repeatly call getSparkBatchJobRequest() since "livyServerApi" field does not always exist in response but
    // only appeared for a while and before that we can't get the "livyServerApi" field.
    ctrlInfo("Trying to get livy URL...");
    return getSparkBatchJobRequest().flatMap(batchResp -> getJobSchedulerState(batchResp) == null ? Observable.error(new IOException("Failed to get scheduler state of the job.")) : Observable.just(batchResp)).retryWhen(err -> err.zipWith(Observable.range(1, getRetriesMax()), (n, i) -> i).delay(getDelaySeconds(), TimeUnit.SECONDS)).repeatWhen(ob -> ob.delay(GET_LIVY_URL_REPEAT_DELAY_MILLISECONDS, TimeUnit.MILLISECONDS)).takeUntil(batchResp -> isJobEnded(batchResp) || StringUtils.isNotEmpty(getLivyAPI(batchResp))).filter(batchResp -> isJobEnded(batchResp) || StringUtils.isNotEmpty(getLivyAPI(batchResp))).flatMap(job -> {
        if (isJobEnded(job)) {
            final String jobState = getJobState(job);
            final String schedulerState = getJobSchedulerState(job);
            final String message = String.format("Job scheduler state: %s. Job running state: %s.", schedulerState, jobState);
            return Observable.just(new SparkLogLine(TOOL, Info, message));
        } else {
            return Observable.just(job).doOnNext(batchResp -> {
                ctrlInfo("Successfully get livy URL: " + batchResp.properties().livyServerAPI());
                ctrlInfo("Trying to retrieve livy submission logs...");
                // After test we find batch id won't be provided until the job is in running state
                // However, since only one spark job will be run on the cluster, the batch ID should always be 0
                setBatchId(0);
            }).map(batchResp -> batchResp.properties().livyServerAPI()).flatMap(livyUrl -> Observable.defer(() -> getSubmissionLogRequest(livyUrl, getBatchId(), getLogStartIndex(), MAX_LOG_LINES_PER_REQUEST)).map(sparkJobLog -> Optional.ofNullable(sparkJobLog.getLog()).orElse(Collections.<String>emptyList())).doOnNext(logs -> setLogStartIndex(getLogStartIndex() + logs.size())).map(logs -> logs.stream().filter(logLine -> !ignoredEmptyLines.contains(logLine.trim().toLowerCase())).collect(Collectors.toList())).flatMap(logLines -> {
                if (logLines.size() > 0) {
                    return Observable.just(Triple.of(logLines, SparkBatchJobState.STARTING.toString(), SchedulerState.SCHEDULED.toString()));
                } else {
                    return getSparkBatchJobRequest().map(batchResp -> Triple.of(logLines, getJobState(batchResp), getJobSchedulerState(batchResp)));
                }
            }).onErrorResumeNext(errors -> getSparkBatchJobRequest().delay(getDelaySeconds(), TimeUnit.SECONDS).map(batchResp -> Triple.of(new ArrayList<>(), getJobState(batchResp), getJobSchedulerState(batchResp)))).repeatWhen(ob -> ob.delay(GET_LOG_REPEAT_DELAY_MILLISECONDS, TimeUnit.MILLISECONDS)).takeUntil(logAndStatesTriple -> {
                String jobRunningState = logAndStatesTriple.getMiddle();
                String jobSchedulerState = logAndStatesTriple.getRight();
                return jobRunningState != null && !jobRunningState.equalsIgnoreCase(SparkBatchJobState.STARTING.toString()) || jobSchedulerState != null && jobSchedulerState.equalsIgnoreCase(SchedulerState.ENDED.toString());
            }).flatMap(logAndStatesTriple -> {
                final String jobRunningState = logAndStatesTriple.getMiddle();
                final String jobSchedulerState = logAndStatesTriple.getRight();
                if (jobRunningState != null && !jobRunningState.equalsIgnoreCase(SparkBatchJobState.STARTING.toString()) || jobSchedulerState != null && jobSchedulerState.equalsIgnoreCase(SchedulerState.ENDED.toString())) {
                    final String message = String.format("Job scheduler state: %s. Job running state: %s.", jobSchedulerState, jobRunningState);
                    return Observable.just(new SparkLogLine(TOOL, Info, message));
                } else {
                    return Observable.from(logAndStatesTriple.getLeft()).map(line -> new SparkLogLine(LIVY, Log, line));
                }
            }));
        }
    });
}

Example 10 with SparkLogLine

use of com.microsoft.azure.hdinsight.spark.common.log.SparkLogLine in project azure-tools-for-java by Microsoft.

the class JobUtils method uploadFileToEmulator.

public static String uploadFileToEmulator(@NotNull IClusterDetail selectedClusterDetail, @NotNull String buildJarPath, @NotNull Observer<SparkLogLine> logSubject) throws Exception {
    logSubject.onNext(new SparkLogLine(TOOL, Info, String.format("Get target jar from %s.", buildJarPath)));
    final String uniqueFolderId = UUID.randomUUID().toString();
    final String folderPath = String.format("../opt/livy/SparkSubmission/%s", uniqueFolderId);
    return String.format("/opt/livy/SparkSubmission/%s/%s", uniqueFolderId, sftpFileToEmulator(buildJarPath, folderPath, selectedClusterDetail));
}