Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 51 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class CoordinatorStreamSystemProducer method send.

/**
   * Serialize and send a coordinator stream message.
   * 
   * @param message
   *          The message to send.
   */
public void send(CoordinatorStreamMessage message) {
    log.debug("Sending {}", message);
    try {
        String source = message.getSource();
        byte[] key = keySerde.toBytes(Arrays.asList(message.getKeyArray()));
        byte[] value = null;
        if (!message.isDelete()) {
            value = messageSerde.toBytes(message.getMessageMap());
        }
        OutgoingMessageEnvelope envelope = new OutgoingMessageEnvelope(systemStream, Integer.valueOf(0), key, value);
        systemProducer.send(source, envelope);
    } catch (Exception e) {
        throw new SamzaException(e);
    }
}

Example 52 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class ExecutionPlanner method calculateJoinInputPartitions.

/**
   * Calculate the partitions for the input streams of join operators
   */
/* package private */
static void calculateJoinInputPartitions(StreamGraphImpl streamGraph, JobGraph jobGraph) {
    // mapping from a source stream to all join specs reachable from it
    Multimap<OperatorSpec, StreamEdge> joinSpecToStreamEdges = HashMultimap.create();
    // reverse mapping of the above
    Multimap<StreamEdge, OperatorSpec> streamEdgeToJoinSpecs = HashMultimap.create();
    // Mapping from the output stream to the join spec. Since StreamGraph creates two partial join operators for a join and they
    // will have the same output stream, this mapping is used to choose one of them as the unique join spec representing this join
    // (who register first in the map wins).
    Map<MessageStream, OperatorSpec> outputStreamToJoinSpec = new HashMap<>();
    // A queue of joins with known input partitions
    Queue<OperatorSpec> joinQ = new LinkedList<>();
    // The visited set keeps track of the join specs that have been already inserted in the queue before
    Set<OperatorSpec> visited = new HashSet<>();
    streamGraph.getInputStreams().entrySet().forEach(entry -> {
        StreamEdge streamEdge = jobGraph.getOrCreateStreamEdge(entry.getKey());
        findReachableJoins(entry.getValue(), streamEdge, joinSpecToStreamEdges, streamEdgeToJoinSpecs, outputStreamToJoinSpec, joinQ, visited);
    });
    // At this point, joinQ contains joinSpecs where at least one of the input stream edge partitions is known.
    while (!joinQ.isEmpty()) {
        OperatorSpec join = joinQ.poll();
        int partitions = StreamEdge.PARTITIONS_UNKNOWN;
        // loop through the input streams to the join and find the partition count
        for (StreamEdge edge : joinSpecToStreamEdges.get(join)) {
            int edgePartitions = edge.getPartitionCount();
            if (edgePartitions != StreamEdge.PARTITIONS_UNKNOWN) {
                if (partitions == StreamEdge.PARTITIONS_UNKNOWN) {
                    //if the partition is not assigned
                    partitions = edgePartitions;
                } else if (partitions != edgePartitions) {
                    throw new SamzaException(String.format("Unable to resolve input partitions of stream %s for join. Expected: %d, Actual: %d", edge.getFormattedSystemStream(), partitions, edgePartitions));
                }
            }
        }
        // assign the partition count for intermediate streams
        for (StreamEdge edge : joinSpecToStreamEdges.get(join)) {
            if (edge.getPartitionCount() <= 0) {
                edge.setPartitionCount(partitions);
                // find other joins can be inferred by setting this edge
                for (OperatorSpec op : streamEdgeToJoinSpecs.get(edge)) {
                    if (!visited.contains(op)) {
                        joinQ.add(op);
                        visited.add(op);
                    }
                }
            }
        }
    }
}

Example 53 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class StreamManager method getStreamPartitionCounts.

Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) {
    Map<String, Integer> streamToPartitionCount = new HashMap<>();
    SystemAdmin systemAdmin = sysAdmins.get(systemName);
    if (systemAdmin == null) {
        throw new SamzaException(String.format("System %s does not exist.", systemName));
    }
    // retrieve the metadata for the streams in this system
    Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames);
    // set the partitions of a stream to its StreamEdge
    streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size()));
    return streamToPartitionCount;
}

Example 54 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class LocalContainerRunner method main.

public static void main(String[] args) throws Exception {
    Thread.setDefaultUncaughtExceptionHandler(new SamzaContainerExceptionHandler(() -> {
        log.info("Exiting process now.");
        System.exit(1);
    }));
    String containerId = System.getenv(ShellCommandConfig.ENV_CONTAINER_ID());
    log.info(String.format("Got container ID: %s", containerId));
    String coordinatorUrl = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
    log.info(String.format("Got coordinator URL: %s", coordinatorUrl));
    int delay = new Random().nextInt(SamzaContainer.DEFAULT_READ_JOBMODEL_DELAY_MS()) + 1;
    JobModel jobModel = SamzaContainer.readJobModel(coordinatorUrl, delay);
    Config config = jobModel.getConfig();
    JobConfig jobConfig = new JobConfig(config);
    if (jobConfig.getName().isEmpty()) {
        throw new SamzaException("can not find the job name");
    }
    String jobName = jobConfig.getName().get();
    String jobId = jobConfig.getJobId().getOrElse(ScalaToJavaUtils.defaultValue("1"));
    MDC.put("containerName", "samza-container-" + containerId);
    MDC.put("jobName", jobName);
    MDC.put("jobId", jobId);
    StreamApplication streamApp = TaskFactoryUtil.createStreamApplication(config);
    LocalContainerRunner localContainerRunner = new LocalContainerRunner(jobModel, containerId);
    localContainerRunner.run(streamApp);
}

Example 55 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class RemoteApplicationRunner method run.

/**
   * Run the {@link StreamApplication} on the remote cluster
   * @param app a StreamApplication
   */
@Override
public void run(StreamApplication app) {
    try {
        // 1. initialize and plan
        ExecutionPlan plan = getExecutionPlan(app);
        writePlanJsonFile(plan.getPlanAsJson());
        // 2. create the necessary streams
        getStreamManager().createStreams(plan.getIntermediateStreams());
        // 3. submit jobs for remote execution
        plan.getJobConfigs().forEach(jobConfig -> {
            log.info("Starting job {} with config {}", jobConfig.getName(), jobConfig);
            JobRunner runner = new JobRunner(jobConfig);
            runner.run(true);
        });
    } catch (Throwable t) {
        throw new SamzaException("Failed to run application", t);
    }
}