Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 91 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TestJobCoordinatorMetadataManager method testGenerateJobCoordinatorMetadataFailed.

@Test
public void testGenerateJobCoordinatorMetadataFailed() {
    doThrow(new RuntimeException("Failed to generate epoch id")).when(jobCoordinatorMetadataManager).fetchEpochIdForJobCoordinator();
    try {
        jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(new JobModel(OLD_CONFIG, containerModelMap), OLD_CONFIG);
        fail("Expected generate job coordinator metadata to throw exception");
    } catch (Exception e) {
        assertTrue("Expecting SamzaException to be thrown", e instanceof SamzaException);
        assertEquals("Metadata generation failed count should be 1", 1, jobCoordinatorMetadataManager.getMetrics().getMetadataGenerationFailedCount().getValue().intValue());
    }
}

Example 92 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class StaticResourceJobCoordinator method start.

@Override
public void start() {
    LOG.info("Starting job coordinator");
    this.systemAdmins.start();
    this.startpointManager.ifPresent(StartpointManager::start);
    try {
        JobModel jobModel = newJobModel();
        doSetLoggingContextConfig(jobModel.getConfig());
        // monitors should be created right after job model is calculated (see jobModelMonitors() for more details)
        JobModelMonitors jobModelMonitors = jobModelMonitors(jobModel);
        Optional<DiagnosticsManager> diagnosticsManager = diagnosticsManager(jobModel);
        JobCoordinatorMetadata newMetadata = this.jobCoordinatorMetadataManager.generateJobCoordinatorMetadata(jobModel, jobModel.getConfig());
        Set<JobMetadataChange> jobMetadataChanges = checkForMetadataChanges(newMetadata);
        if (!jobMetadataChanges.isEmpty() && !jobMetadataChanges.contains(JobMetadataChange.NEW_DEPLOYMENT)) {
            /*
         * If the job coordinator comes up, but not due to a new deployment, and the metadata changed, then trigger a
         * restart. This case applies if the job coordinator died and the job model needed to change while it was down.
         * If there were no metadata changes, then just let the current workers continue to run.
         * If there was a new deployment (which includes the case where the coordinator requested a restart), then we
         * rely on the external resource manager to make sure the previous workers restarted, so we don't need to
         * restart again.
         */
            LOG.info("Triggering job restart");
            this.jobRestartSignal.restartJob();
        } else {
            prepareWorkerExecution(jobModel, newMetadata, jobMetadataChanges);
            // save components that depend on job model in order to manage lifecycle or access later
            this.currentDiagnosticsManager = diagnosticsManager;
            this.currentJobModelMonitors = Optional.of(jobModelMonitors);
            this.currentJobModel = Optional.of(jobModel);
            // lifecycle: start components
            this.coordinatorCommunication.start();
            this.jobCoordinatorListener.ifPresent(listener -> listener.onNewJobModel(this.processorId, jobModel));
            this.currentDiagnosticsManager.ifPresent(DiagnosticsManager::start);
            jobModelMonitors.start();
            this.jobPreparationComplete.set(true);
        }
    } catch (Exception e) {
        LOG.error("Error while running job coordinator; exiting", e);
        throw new SamzaException("Error while running job coordinator", e);
    }
}

Example 93 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class CoordinatorStreamSystemConsumer method register.

/**
 * Retrieves the oldest offset in the coordinator stream, and registers the
 * coordinator stream with the SystemConsumer using the earliest offset.
 */
public void register() {
    if (isStarted) {
        log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition);
        return;
    }
    log.debug("Attempting to register: {}", coordinatorSystemStreamPartition);
    Set<String> streamNames = new HashSet<String>();
    String streamName = coordinatorSystemStreamPartition.getStream();
    streamNames.add(streamName);
    Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames);
    log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString()));
    if (systemStreamMetadataMap == null) {
        throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal.");
    }
    SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName);
    if (systemStreamMetadata == null) {
        throw new SamzaException("Expected " + streamName + " to be in system stream metadata.");
    }
    SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition());
    if (systemStreamPartitionMetadata == null) {
        throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist.");
    }
    String startingOffset = systemStreamPartitionMetadata.getOldestOffset();
    log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset);
    systemConsumer.register(coordinatorSystemStreamPartition, startingOffset);
}

Example 94 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class CoordinatorStreamSystemConsumer method bootstrap.

/**
 * Read all messages from the earliest offset, all the way to the latest.
 * Currently, this method only pays attention to config messages.
 */
public void bootstrap() {
    synchronized (bootstrapLock) {
        // Make a copy so readers aren't affected while we modify the set.
        final LinkedHashSet<CoordinatorStreamMessage> bootstrappedMessages = new LinkedHashSet<>(bootstrappedStreamSet);
        log.info("Bootstrapping configuration from coordinator stream.");
        SystemStreamPartitionIterator iterator = new SystemStreamPartitionIterator(systemConsumer, coordinatorSystemStreamPartition);
        try {
            while (iterator.hasNext()) {
                IncomingMessageEnvelope envelope = iterator.next();
                Object[] keyArray = keySerde.fromBytes((byte[]) envelope.getKey()).toArray();
                Map<String, Object> valueMap = null;
                if (envelope.getMessage() != null) {
                    valueMap = messageSerde.fromBytes((byte[]) envelope.getMessage());
                }
                CoordinatorStreamMessage coordinatorStreamMessage = new CoordinatorStreamMessage(keyArray, valueMap);
                log.debug("Received coordinator stream message: {}", coordinatorStreamMessage);
                // Remove any existing entry. Set.add() does not add if the element already exists.
                if (bootstrappedMessages.remove(coordinatorStreamMessage)) {
                    log.debug("Removed duplicate message: {}", coordinatorStreamMessage);
                }
                bootstrappedMessages.add(coordinatorStreamMessage);
                if (SetConfig.TYPE.equals(coordinatorStreamMessage.getType())) {
                    String configKey = coordinatorStreamMessage.getKey();
                    if (coordinatorStreamMessage.isDelete()) {
                        configMap.remove(configKey);
                    } else {
                        String configValue = new SetConfig(coordinatorStreamMessage).getConfigValue();
                        configMap.put(configKey, configValue);
                    }
                }
            }
            bootstrappedStreamSet = Collections.unmodifiableSet(bootstrappedMessages);
            log.debug("Bootstrapped configuration: {}", configMap);
            isBootstrapped = true;
        } catch (Exception e) {
            throw new SamzaException(e);
        }
    }
}

Example 95 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class CoordinatorStreamValueSerde method fromBytes.

@Override
public String fromBytes(byte[] bytes) {
    Map<String, Object> values = messageSerde.fromBytes(bytes);
    CoordinatorStreamMessage message = new CoordinatorStreamMessage(new Object[] {}, values);
    if (type.equalsIgnoreCase(SetExecutionEnvContainerIdMapping.TYPE)) {
        SetExecutionEnvContainerIdMapping executionContainerIdMapping = new SetExecutionEnvContainerIdMapping(message);
        return executionContainerIdMapping.getExecutionEnvironmentContainerId();
    } else if (type.equalsIgnoreCase(SetContainerHostMapping.TYPE)) {
        SetContainerHostMapping hostMapping = new SetContainerHostMapping(message);
        return hostMapping.getHostLocality();
    } else if (type.equalsIgnoreCase(SetTaskContainerMapping.TYPE)) {
        SetTaskContainerMapping setTaskContainerMapping = new SetTaskContainerMapping(message);
        return setTaskContainerMapping.getTaskAssignment();
    } else if (type.equalsIgnoreCase(SetChangelogMapping.TYPE)) {
        SetChangelogMapping changelogMapping = new SetChangelogMapping(message);
        return (changelogMapping.getPartition() != null) ? String.valueOf(changelogMapping.getPartition()) : null;
    } else if (type.equalsIgnoreCase(SetConfig.TYPE)) {
        SetConfig setConfig = new SetConfig(message);
        return setConfig.getConfigValue();
    } else if (type.equalsIgnoreCase(SetTaskModeMapping.TYPE)) {
        SetTaskModeMapping setTaskModeMapping = new SetTaskModeMapping(message);
        return String.valueOf(setTaskModeMapping.getTaskMode());
    } else if (type.equalsIgnoreCase(SetTaskPartitionMapping.TYPE)) {
        SetTaskPartitionMapping setTaskPartitionMapping = new SetTaskPartitionMapping(message);
        return setTaskPartitionMapping.getTaskNames();
    } else if (type.equalsIgnoreCase(SetJobCoordinatorMetadataMessage.TYPE)) {
        SetJobCoordinatorMetadataMessage jobCoordinatorMetadataMessage = new SetJobCoordinatorMetadataMessage(message);
        return jobCoordinatorMetadataMessage.getJobCoordinatorMetadata();
    } else {
        throw new SamzaException(String.format("Unknown coordinator stream message type: %s", type));
    }
}