Examples with RetryableException io.cdap.cdap.api.retry.RetryableException used on opensource projects

Example 26 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by cdapio.

the class AbstractArtifactLocalizer method fetchArtifact.

/**
 * fetchArtifact attempts to connect to app fabric to download the given artifact. This method will throw
 * {@link RetryableException} in certain circumstances.
 *
 * @param artifactId The ArtifactId of the artifact to fetch
 * @param remoteClient The remote client used to connect to appfabric on the peer
 * @param artifactDir The directory where the artifact will be stored
 * @return The Local Location for this artifact
 * @throws IOException If an unexpected error occurs while writing the artifact to the filesystem
 * @throws ArtifactNotFoundException If the given artifact does not exist
 */
protected File fetchArtifact(ArtifactId artifactId, RemoteClient remoteClient, File artifactDir) throws IOException, ArtifactNotFoundException {
    Long lastModifiedTimestamp = getCurrentLastModifiedTimestamp(artifactDir);
    HttpURLConnection urlConn = openConnection(artifactId, remoteClient);
    try {
        if (lastModifiedTimestamp != null) {
            LOG.debug("Found existing local version for {} with timestamp {}", artifactId, lastModifiedTimestamp);
            ZonedDateTime lastModifiedDate = ZonedDateTime.ofInstant(Instant.ofEpochMilli(lastModifiedTimestamp), ZoneId.of("GMT"));
            urlConn.setRequestProperty(HttpHeaders.IF_MODIFIED_SINCE, lastModifiedDate.format(DateTimeFormatter.RFC_1123_DATE_TIME));
        }
        // If we get this response that means we already have the most up to date artifact
        if (lastModifiedTimestamp != null && urlConn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
            LOG.debug("Call to app fabric returned NOT_MODIFIED for {} with lastModifiedTimestamp of {}", artifactId, lastModifiedTimestamp);
            File artifactJarLocation = getArtifactJarLocation(artifactDir, lastModifiedTimestamp);
            if (!artifactJarLocation.exists()) {
                throw new RetryableException(String.format("Locally cached artifact jar for %s is missing.", artifactId));
            }
            return artifactJarLocation;
        }
        throwIfError(urlConn, artifactId);
        ZonedDateTime newModifiedDate = getLastModifiedHeader(urlConn);
        long newTimestamp = newModifiedDate.toInstant().toEpochMilli();
        File newLocation = getArtifactJarLocation(artifactDir, newTimestamp);
        DirUtils.mkdirs(newLocation.getParentFile());
        // Download the artifact to a temporary file then atomically rename it to the final name to
        // avoid race conditions with multiple threads.
        Path tempFile = Files.createTempFile(newLocation.getParentFile().toPath(), String.valueOf(newTimestamp), ".jar");
        return downloadArtifact(urlConn, newLocation.toPath(), tempFile);
    } finally {
        urlConn.disconnect();
    }
}

Example 27 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by cdapio.

the class AbstractServiceRetryableMacroEvaluator method evaluateMap.

@Override
public Map<String, String> evaluateMap(String macroFunction, String... args) throws InvalidMacroException {
    if (!functionName.equals(macroFunction)) {
        // This shouldn't happen
        throw new IllegalArgumentException("Invalid function name " + macroFunction + ". Expecting " + functionName);
    }
    // Make call with exponential delay on failure retry.
    long delay = RETRY_BASE_DELAY_MILLIS;
    double minMultiplier = RETRY_DELAY_MULTIPLIER - RETRY_DELAY_MULTIPLIER * RETRY_RANDOMIZE_FACTOR;
    double maxMultiplier = RETRY_DELAY_MULTIPLIER + RETRY_DELAY_MULTIPLIER * RETRY_RANDOMIZE_FACTOR;
    Stopwatch stopWatch = new Stopwatch().start();
    try {
        while (stopWatch.elapsedTime(TimeUnit.MILLISECONDS) < TIMEOUT_MILLIS) {
            try {
                return evaluateMacroMap(macroFunction, args);
            } catch (RetryableException e) {
                TimeUnit.MILLISECONDS.sleep(delay);
                delay = (long) (delay * (minMultiplier + Math.random() * (maxMultiplier - minMultiplier + 1)));
                delay = Math.min(delay, RETRY_MAX_DELAY_MILLIS);
            } catch (IOException e) {
                throw new RuntimeException("Failed to evaluate the macro function '" + functionName + "' with args " + Arrays.asList(args), e);
            }
        }
    } catch (InterruptedException e) {
        throw new RuntimeException("Thread interrupted while trying to evaluate the macro function '" + functionName + "' with args " + Arrays.asList(args), e);
    }
    throw new IllegalStateException("Timed out when trying to evaluate the macro function '" + functionName + "' with args " + Arrays.asList(args));
}

Example 28 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by cdapio.

the class UpgradeJobMain method suspendSchedulesAndStopPipelines.

private static void suspendSchedulesAndStopPipelines(ClientConfig clientConfig) throws Exception {
    ApplicationClient applicationClient = new ApplicationClient(clientConfig);
    ScheduleClient scheduleClient = new ScheduleClient(clientConfig);
    ProgramClient programClient = new ProgramClient(clientConfig);
    NamespaceClient namespaceClient = new NamespaceClient(clientConfig);
    boolean shouldRetry = false;
    List<NamespaceId> namespaceIdList = namespaceClient.list().stream().map(NamespaceMeta::getNamespaceId).collect(Collectors.toList());
    namespaceIdList.add(NamespaceId.SYSTEM);
    for (NamespaceId namespaceId : namespaceIdList) {
        for (ApplicationRecord record : applicationClient.list(namespaceId)) {
            ApplicationId applicationId = new ApplicationId(namespaceId.getNamespace(), record.getName(), record.getAppVersion());
            LOG.debug("Trying to stop schedule and workflows for application " + applicationId);
            List<WorkflowId> workflowIds = applicationClient.get(applicationId).getPrograms().stream().filter(programRecord -> programRecord.getType().equals(ProgramType.WORKFLOW)).map(programRecord -> new WorkflowId(applicationId, programRecord.getName())).collect(Collectors.toList());
            for (WorkflowId workflowId : workflowIds) {
                List<ScheduleId> scheduleIds = scheduleClient.listSchedules(workflowId).stream().map(scheduleDetail -> new ScheduleId(namespaceId.getNamespace(), record.getName(), record.getAppVersion(), scheduleDetail.getName())).collect(Collectors.toList());
                for (ScheduleId scheduleId : scheduleIds) {
                    if (scheduleClient.getStatus(scheduleId).equals(SCHEDULED)) {
                        scheduleClient.suspend(scheduleId);
                    }
                }
                // Need to stop workflows first or else the program will fail to stop below
                if (!programClient.getStatus(workflowId).equals(ProgramStatus.STOPPED.toString())) {
                    try {
                        programClient.stop(workflowId);
                    } catch (BadRequestException e) {
                        // transitioned to stop state since it was checked earlier or not.
                        if (!programClient.getStatus(workflowId).equals(ProgramStatus.STOPPED.toString())) {
                            // Pipeline still in running state. Continue with stopping rest of the pipelines in this namespace and
                            // next retry should try to stop/verify status for this pipeline.
                            shouldRetry = true;
                        }
                    }
                }
            }
        }
        // At least one pipeline is still in running state so retry to verify pipeline status .
        if (shouldRetry) {
            throw new RetryableException("At least one pipeline in namespace " + namespaceId + " is still running.");
        }
        // All schedules are stopped, now stop all programs
        programClient.stopAll(namespaceId);
    }
}