Examples with RetryableException io.cdap.cdap.api.retry.RetryableException used on opensource projects

Example 11 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by cdapio.

the class AbstractArtifactLocalizer method getLastModifiedHeader.

/**
 * Helper function for verifying, extracting and converting the Last-Modified header from the URL connection.
 */
private ZonedDateTime getLastModifiedHeader(HttpURLConnection urlConn) {
    Map<String, List<String>> headers = urlConn.getHeaderFields();
    ZonedDateTime lastModified = headers.entrySet().stream().filter(headerEntry -> HttpHeaders.LAST_MODIFIED.equalsIgnoreCase(headerEntry.getKey())).map(Map.Entry::getValue).flatMap(Collection::stream).findFirst().map(s -> ZonedDateTime.parse(s, DateTimeFormatter.RFC_1123_DATE_TIME)).orElse(null);
    if (lastModified == null) {
        // If it does happen we should retry.
        throw new RetryableException(String.format("The response from %s did not contain the %s header.", urlConn.getURL(), HttpHeaders.LAST_MODIFIED));
    }
    return lastModified;
}

Example 12 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by caskdata.

the class AbstractArtifactLocalizer method fetchArtifact.

/**
 * fetchArtifact attempts to connect to app fabric to download the given artifact. This method will throw
 * {@link RetryableException} in certain circumstances.
 *
 * @param artifactId The ArtifactId of the artifact to fetch
 * @param remoteClient The remote client used to connect to appfabric on the peer
 * @param artifactDir The directory where the artifact will be stored
 * @return The Local Location for this artifact
 * @throws IOException If an unexpected error occurs while writing the artifact to the filesystem
 * @throws ArtifactNotFoundException If the given artifact does not exist
 */
protected File fetchArtifact(ArtifactId artifactId, RemoteClient remoteClient, File artifactDir) throws IOException, ArtifactNotFoundException {
    Long lastModifiedTimestamp = getCurrentLastModifiedTimestamp(artifactDir);
    HttpURLConnection urlConn = openConnection(artifactId, remoteClient);
    try {
        if (lastModifiedTimestamp != null) {
            LOG.debug("Found existing local version for {} with timestamp {}", artifactId, lastModifiedTimestamp);
            ZonedDateTime lastModifiedDate = ZonedDateTime.ofInstant(Instant.ofEpochMilli(lastModifiedTimestamp), ZoneId.of("GMT"));
            urlConn.setRequestProperty(HttpHeaders.IF_MODIFIED_SINCE, lastModifiedDate.format(DateTimeFormatter.RFC_1123_DATE_TIME));
        }
        // If we get this response that means we already have the most up to date artifact
        if (lastModifiedTimestamp != null && urlConn.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
            LOG.debug("Call to app fabric returned NOT_MODIFIED for {} with lastModifiedTimestamp of {}", artifactId, lastModifiedTimestamp);
            File artifactJarLocation = getArtifactJarLocation(artifactDir, lastModifiedTimestamp);
            if (!artifactJarLocation.exists()) {
                throw new RetryableException(String.format("Locally cached artifact jar for %s is missing.", artifactId));
            }
            return artifactJarLocation;
        }
        throwIfError(urlConn, artifactId);
        ZonedDateTime newModifiedDate = getLastModifiedHeader(urlConn);
        long newTimestamp = newModifiedDate.toInstant().toEpochMilli();
        File newLocation = getArtifactJarLocation(artifactDir, newTimestamp);
        DirUtils.mkdirs(newLocation.getParentFile());
        // Download the artifact to a temporary file then atomically rename it to the final name to
        // avoid race conditions with multiple threads.
        Path tempFile = Files.createTempFile(newLocation.getParentFile().toPath(), String.valueOf(newTimestamp), ".jar");
        return downloadArtifact(urlConn, newLocation.toPath(), tempFile);
    } finally {
        urlConn.disconnect();
    }
}

Example 13 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by caskdata.

the class UpgradeJobMain method suspendSchedulesAndStopPipelines.

private static void suspendSchedulesAndStopPipelines(ClientConfig clientConfig) throws Exception {
    ApplicationClient applicationClient = new ApplicationClient(clientConfig);
    ScheduleClient scheduleClient = new ScheduleClient(clientConfig);
    ProgramClient programClient = new ProgramClient(clientConfig);
    NamespaceClient namespaceClient = new NamespaceClient(clientConfig);
    boolean shouldRetry = false;
    List<NamespaceId> namespaceIdList = namespaceClient.list().stream().map(NamespaceMeta::getNamespaceId).collect(Collectors.toList());
    namespaceIdList.add(NamespaceId.SYSTEM);
    for (NamespaceId namespaceId : namespaceIdList) {
        for (ApplicationRecord record : applicationClient.list(namespaceId)) {
            ApplicationId applicationId = new ApplicationId(namespaceId.getNamespace(), record.getName(), record.getAppVersion());
            LOG.debug("Trying to stop schedule and workflows for application " + applicationId);
            List<WorkflowId> workflowIds = applicationClient.get(applicationId).getPrograms().stream().filter(programRecord -> programRecord.getType().equals(ProgramType.WORKFLOW)).map(programRecord -> new WorkflowId(applicationId, programRecord.getName())).collect(Collectors.toList());
            for (WorkflowId workflowId : workflowIds) {
                List<ScheduleId> scheduleIds = scheduleClient.listSchedules(workflowId).stream().map(scheduleDetail -> new ScheduleId(namespaceId.getNamespace(), record.getName(), record.getAppVersion(), scheduleDetail.getName())).collect(Collectors.toList());
                for (ScheduleId scheduleId : scheduleIds) {
                    if (scheduleClient.getStatus(scheduleId).equals(SCHEDULED)) {
                        scheduleClient.suspend(scheduleId);
                    }
                }
                // Need to stop workflows first or else the program will fail to stop below
                if (!programClient.getStatus(workflowId).equals(ProgramStatus.STOPPED.toString())) {
                    try {
                        programClient.stop(workflowId);
                    } catch (BadRequestException e) {
                        // transitioned to stop state since it was checked earlier or not.
                        if (!programClient.getStatus(workflowId).equals(ProgramStatus.STOPPED.toString())) {
                            // Pipeline still in running state. Continue with stopping rest of the pipelines in this namespace and
                            // next retry should try to stop/verify status for this pipeline.
                            shouldRetry = true;
                        }
                    }
                }
            }
        }
        // At least one pipeline is still in running state so retry to verify pipeline status .
        if (shouldRetry) {
            throw new RetryableException("At least one pipeline in namespace " + namespaceId + " is still running.");
        }
        // All schedules are stopped, now stop all programs
        programClient.stopAll(namespaceId);
    }
}

Example 14 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by caskdata.

the class RuntimeServiceMainTest method createProgramStateWriter.

/**
 * Creates a {@link ProgramStateWriter} that writes to {@link RuntimeClient} directly.
 *
 * @param injector the injector for creating the {@link RuntimeClient}
 * @param programRunId the {@link ProgramRunId} for the program state change
 * @return a {@link ProgramStateWriter}
 */
private ProgramStateWriter createProgramStateWriter(Injector injector, ProgramRunId programRunId) {
    RuntimeClient runtimeClient = injector.getInstance(RuntimeClient.class);
    // We write to the record event directly to skip the app-fabric to process it
    // This is because we don't follow the normal event flow here for testing
    TopicId topicId = NamespaceId.SYSTEM.topic(injector.getInstance(CConfiguration.class).get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
    RetryStrategy retryStrategy = RetryStrategies.timeLimit(5, TimeUnit.SECONDS, RetryStrategies.fixDelay(200, TimeUnit.MILLISECONDS));
    return new MessagingProgramStateWriter((notificationType, properties) -> {
        Notification notification = new Notification(notificationType, properties);
        try {
            Retries.callWithRetries((Retries.Callable<Void, Exception>) () -> {
                runtimeClient.sendMessages(programRunId, topicId, Collections.singleton(createMessage(notification)).iterator());
                return null;
            }, retryStrategy, t -> t instanceof IOException || t instanceof RetryableException);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
}

Example 15 with RetryableException

use of io.cdap.cdap.api.retry.RetryableException in project cdap by caskdata.

the class DatasetServiceStore method startUp.

@Override
protected void startUp() throws Exception {
    final DatasetId serviceStoreDatasetInstanceId = NamespaceId.SYSTEM.dataset(Constants.Service.SERVICE_INSTANCE_TABLE_NAME);
    table = Retries.supplyWithRetries(() -> {
        try {
            return DatasetsUtil.getOrCreateDataset(dsFramework, serviceStoreDatasetInstanceId, NoTxKeyValueTable.class.getName(), DatasetProperties.EMPTY, null);
        } catch (Exception e) {
            // Throwing RetryableException here is just to make it retry getting the dataset
            // an exception here usually means there is an hbase problem
            LOG.warn("Error getting service store dataset {}. Will retry after some time: {}", serviceStoreDatasetInstanceId, e.getMessage());
            throw new RetryableException(e);
        }
    }, RetryStrategies.exponentialDelay(1, 30, TimeUnit.SECONDS));
}