Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 81 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class LocalJobPlanner method createStreams.

/**
 * Create intermediate streams using {@link StreamManager}.
 * If {@link CoordinationUtils} is provided, this function will first invoke leader election, and the leader
 * will create the streams. All the runner processes will wait on the latch that is released after the leader finishes
 * stream creation.
 * @param planId a unique identifier representing the plan used for coordination purpose
 * @param intStreams list of intermediate {@link StreamSpec}s
 * @param streamManager the {@link StreamManager} used to create streams
 */
private void createStreams(String planId, List<StreamSpec> intStreams, StreamManager streamManager) {
    if (intStreams.isEmpty()) {
        LOG.info("Set of intermediate streams is empty. Nothing to create.");
        return;
    }
    LOG.info("A single processor must create the intermediate streams. Processor {} will attempt to acquire the lock.", processorId);
    // Refer SAMZA-1385 for more details
    if (coordinationUtils == null) {
        LOG.warn("Processor {} failed to create utils. Each processor will attempt to create streams.", processorId);
        // each application process will try creating the streams, which
        // requires stream creation to be idempotent
        streamManager.createStreams(intStreams);
        return;
    }
    // If BATCH, then need to create new intermediate streams every run.
    // planId does not change every run and hence, need to use runid
    // as the lockId to create a new lock with state each run
    // to create new streams each run.
    // If run.id is null, defaults to old behavior of using planId
    boolean isAppModeBatch = new ApplicationConfig(userConfig).getAppMode() == ApplicationConfig.ApplicationMode.BATCH;
    String lockId = planId;
    if (isAppModeBatch && runId != null) {
        lockId = runId;
    }
    try {
        checkAndCreateStreams(lockId, intStreams, streamManager);
    } catch (TimeoutException te) {
        throw new SamzaException(String.format("Processor {} failed to get the lock for stream initialization within timeout.", processorId), te);
    } finally {
        if (!isAppModeBatch && coordinationUtils != null) {
            coordinationUtils.close();
        }
    }
}

Example 82 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class LocalJobPlanner method checkAndCreateStreams.

private void checkAndCreateStreams(String lockId, List<StreamSpec> intStreams, StreamManager streamManager) throws TimeoutException {
    MetadataStore metadataStore = getMetadataStore();
    DistributedLock distributedLock = coordinationUtils.getLock(lockId);
    if (distributedLock == null || metadataStore == null) {
        LOG.warn("Processor {} failed to create utils. Each processor will attempt to create streams.", processorId);
        // each application process will try creating the streams, which requires stream creation to be idempotent
        streamManager.createStreams(intStreams);
        return;
    }
    // Start timer for timeout
    long startTime = System.currentTimeMillis();
    long lockTimeout = TimeUnit.MILLISECONDS.convert(CoordinationConstants.LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    // does not die of timeout exception and comes back and checks for state and proceeds
    while ((System.currentTimeMillis() - startTime) < lockTimeout) {
        if (metadataStore.get(String.format(STREAM_CREATED_STATE_KEY, lockId)) != null) {
            LOG.info("Processor {} found streams created state data. They must've been created by another processor.", processorId);
            break;
        }
        try {
            if (distributedLock.lock(Duration.ofMillis(10000))) {
                LOG.info("lock acquired for streams creation by Processor " + processorId);
                streamManager.createStreams(intStreams);
                String streamCreatedMessage = "Streams created by processor " + processorId;
                metadataStore.put(String.format(STREAM_CREATED_STATE_KEY, lockId), streamCreatedMessage.getBytes("UTF-8"));
                metadataStore.flush();
                distributedLock.unlock();
                break;
            } else {
                LOG.info("Processor {} failed to get the lock for stream initialization. Will try again until time out", processorId);
            }
        } catch (UnsupportedEncodingException e) {
            String msg = String.format("Processor {} failed to encode string for stream initialization", processorId);
            throw new SamzaException(msg, e);
        }
    }
    if ((System.currentTimeMillis() - startTime) >= lockTimeout) {
        throw new TimeoutException(String.format("Processor {} failed to get the lock for stream initialization within {} milliseconds.", processorId, CoordinationConstants.LOCK_TIMEOUT_MS));
    }
}

Example 83 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class OperatorImpl method onMessageAsync.

public final CompletionStage<Void> onMessageAsync(M message, MessageCollector collector, TaskCoordinator coordinator) {
    this.numMessage.inc();
    long startNs = this.highResClock.nanoTime();
    CompletionStage<Collection<RM>> completableResultsFuture;
    try {
        completableResultsFuture = handleMessageAsync(message, collector, coordinator);
    } catch (ClassCastException e) {
        String actualType = e.getMessage().replaceFirst(" cannot be cast to .*", "");
        String expectedType = e.getMessage().replaceFirst(".* cannot be cast to ", "");
        throw new SamzaException(String.format("Error applying operator %s (created at %s) to its input message. " + "Expected input message to be of type %s, but found it to be of type %s. " + "Are Serdes for the inputs to this operator configured correctly?", getOpImplId(), getOperatorSpec().getSourceLocation(), expectedType, actualType), e);
    }
    CompletionStage<Void> result = completableResultsFuture.thenCompose(results -> {
        long endNs = this.highResClock.nanoTime();
        this.handleMessageNs.update(endNs - startNs);
        return CompletableFuture.allOf(results.stream().flatMap(r -> this.registeredOperators.stream().map(op -> op.onMessageAsync(r, collector, coordinator))).toArray(CompletableFuture[]::new));
    });
    WatermarkFunction watermarkFn = getOperatorSpec().getWatermarkFn();
    if (watermarkFn != null) {
        // check whether there is new watermark emitted from the user function
        Long outputWm = watermarkFn.getOutputWatermark();
        return result.thenCompose(ignored -> propagateWatermark(outputWm, collector, coordinator));
    }
    return result;
}

Example 84 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class MetricsSnapshotReporter method stop.

@Override
public void stop() {
    // Scheduling an event with 0 delay to ensure flushing of metrics one last time before shutdown
    this.executor.schedule(this, 0, TimeUnit.SECONDS);
    LOG.info("Stopping reporter timer.");
    // Allow the scheduled task above to finish, and block for termination (for max 60 seconds)
    this.executor.shutdown();
    try {
        this.executor.awaitTermination(60, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        throw new SamzaException(e);
    }
    LOG.info("Stopping producer.");
    this.producer.stop();
    if (!this.executor.isTerminated()) {
        LOG.warn("Unable to shutdown reporter timer.");
    }
}

Example 85 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class JobCoordinatorMetadataManager method writeJobCoordinatorMetadata.

/**
 * Persist the {@link JobCoordinatorMetadata} in metadata store. The job coordinator metadata is associated
 * with the cluster type specified at the creation of the manager.
 *
 * @param metadata metadata to be persisted
 *
 * @throws SamzaException in case of exception encountered during the writes to underlying metadata store
 */
public void writeJobCoordinatorMetadata(JobCoordinatorMetadata metadata) {
    Preconditions.checkNotNull(metadata, "Job coordinator metadata cannot be null");
    try {
        String metadataValueString = metadataMapper.writeValueAsString(metadata);
        metadataStore.put(clusterType.name(), valueSerde.toBytes(metadataValueString));
        LOG.info("Successfully written job coordinator metadata: {} for cluster {}.", metadata, clusterType);
    } catch (Exception e) {
        metrics.incrementMetadataWriteFailedCount();
        LOG.error("Failed to write the job coordinator metadata to metadata store due to ", e);
        throw new SamzaException("Failed to write the job coordinator metadata.", e);
    }
}