Examples with SamzaException org.apache.samza.SamzaException used on opensource projects

Example 96 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class StreamRegexMonitorFactory method build.

/**
 * Build a {@link StreamRegexMonitor} for input streams for the job model.
 */
public Optional<StreamRegexMonitor> build(JobModel jobModel, Config config, StreamRegexMonitor.Callback callback) {
    JobConfig jobConfig = new JobConfig(config);
    // if input regex monitor is not enabled return empty
    if (jobConfig.getMonitorRegexDisabled()) {
        LOG.info("StreamRegexMonitor is disabled.");
        return Optional.empty();
    }
    Set<SystemStream> inputStreamsToMonitor = JobModelUtil.getSystemStreams(jobModel);
    if (inputStreamsToMonitor.isEmpty()) {
        throw new SamzaException("Input streams to a job can not be empty.");
    }
    // First list all rewriters
    Optional<String> rewritersList = jobConfig.getConfigRewriters();
    // if no rewriter is defined, there is nothing to monitor
    if (!rewritersList.isPresent()) {
        LOG.warn("No config rewriters are defined. No StreamRegexMonitor created.");
        return Optional.empty();
    }
    // Compile a map of each input-system to its corresponding input-monitor-regex patterns
    Map<String, Pattern> inputRegexesToMonitor = jobConfig.getMonitorRegexPatternMap(rewritersList.get());
    // if there are no regexes to monitor
    if (inputRegexesToMonitor.isEmpty()) {
        LOG.info("No input regexes are defined. No StreamRegexMonitor created.");
        return Optional.empty();
    }
    return Optional.of(new StreamRegexMonitor(inputStreamsToMonitor, inputRegexesToMonitor, this.streamMetadataCache, this.metrics, jobConfig.getMonitorRegexFrequency(), callback));
}

Example 97 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TaskPartitionAssignmentManager method readTaskPartitionAssignments.

/**
 * Reads the task partition assignments from the underlying storage layer.
 * @return the task partition assignments.
 */
public Map<SystemStreamPartition, List<String>> readTaskPartitionAssignments() {
    try {
        Map<SystemStreamPartition, List<String>> sspToTaskNamesMap = new HashMap<>();
        Map<String, byte[]> allMetadataEntries = metadataStore.all();
        for (Map.Entry<String, byte[]> entry : allMetadataEntries.entrySet()) {
            SystemStreamPartition systemStreamPartition = deserializeSSPFromJson(entry.getKey());
            String taskNamesAsJson = valueSerde.fromBytes(entry.getValue());
            List<String> taskNames = taskNamesMapper.readValue(taskNamesAsJson, new TypeReference<List<String>>() {
            });
            sspToTaskNamesMap.put(systemStreamPartition, taskNames);
        }
        return sspToTaskNamesMap;
    } catch (Exception e) {
        throw new SamzaException("Exception occurred when reading task partition assignments.", e);
    }
}

Example 98 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TestStreamTableJoinOperatorImpl method testHandleMessage.

@Test
public void testHandleMessage() {
    String tableId = "t1";
    StreamTableJoinOperatorSpec mockJoinOpSpec = mock(StreamTableJoinOperatorSpec.class);
    when(mockJoinOpSpec.getTableId()).thenReturn(tableId);
    when(mockJoinOpSpec.getArgs()).thenReturn(new Object[0]);
    when(mockJoinOpSpec.getJoinFn()).thenReturn(new StreamTableJoinFunction<String, KV<String, String>, KV<String, String>, String>() {

        @Override
        public String apply(KV<String, String> message, KV<String, String> record) {
            if ("1".equals(message.getKey())) {
                Assert.assertEquals("m1", message.getValue());
                Assert.assertEquals("r1", record.getValue());
                return "m1r1";
            } else if ("2".equals(message.getKey())) {
                Assert.assertEquals("m2", message.getValue());
                Assert.assertNull(record);
                return null;
            }
            throw new SamzaException("Should never reach here!");
        }

        @Override
        public String getMessageKey(KV<String, String> message) {
            return message.getKey();
        }

        @Override
        public String getRecordKey(KV<String, String> record) {
            return record.getKey();
        }
    });
    ReadWriteUpdateTable table = mock(ReadWriteUpdateTable.class);
    when(table.getAsync("1")).thenReturn(CompletableFuture.completedFuture("r1"));
    when(table.getAsync("2")).thenReturn(CompletableFuture.completedFuture(null));
    Context context = new MockContext();
    when(context.getTaskContext().getUpdatableTable(tableId)).thenReturn(table);
    MessageCollector mockMessageCollector = mock(MessageCollector.class);
    TaskCoordinator mockTaskCoordinator = mock(TaskCoordinator.class);
    StreamTableJoinOperatorImpl streamTableJoinOperator = new StreamTableJoinOperatorImpl(mockJoinOpSpec, context);
    // Table has the key
    Collection<TestMessageEnvelope> result;
    result = streamTableJoinOperator.handleMessage(KV.of("1", "m1"), mockMessageCollector, mockTaskCoordinator);
    Assert.assertEquals(1, result.size());
    Assert.assertEquals("m1r1", result.iterator().next());
    // Table doesn't have the key
    result = streamTableJoinOperator.handleMessage(KV.of("2", "m2"), mockMessageCollector, mockTaskCoordinator);
    Assert.assertEquals(0, result.size());
}

Example 99 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class TestOperatorSpecGraph method testCloneWithSerializationError.

@Test(expected = NotSerializableException.class)
public void testCloneWithSerializationError() throws Throwable {
    OperatorSpec mockFailedOpSpec = PowerMockito.mock(OperatorSpec.class);
    when(mockFailedOpSpec.getOpId()).thenReturn("test-failed-op-4");
    allOpSpecs.add(mockFailedOpSpec);
    inputOpSpecMap.values().stream().findFirst().get().registerNextOperatorSpec(mockFailedOpSpec);
    // failed with serialization error
    try {
        new OperatorSpecGraph(mockAppDesc);
        fail("Should have failed with serialization error");
    } catch (SamzaException se) {
        throw se.getCause();
    }
}

Example 100 with SamzaException

use of org.apache.samza.SamzaException in project samza by apache.

the class StreamAppender method setupSystem.

/**
 * This should only be called after verifying that the {@link LoggingContextHolder} has the config.
 */
protected void setupSystem() {
    config = getConfig();
    Log4jSystemConfig log4jSystemConfig = new Log4jSystemConfig(config);
    if (streamName == null) {
        streamName = getStreamName(log4jSystemConfig.getJobName(), log4jSystemConfig.getJobId());
    }
    // TODO we need the ACTUAL metrics registry, or the metrics won't get reported by the metric reporters!
    MetricsRegistry metricsRegistry = new MetricsRegistryMap();
    metrics = new StreamAppenderMetrics("stream-appender", metricsRegistry);
    String systemName = log4jSystemConfig.getSystemName();
    String systemFactoryName = log4jSystemConfig.getSystemFactory(systemName).orElseThrow(() -> new SamzaException("Could not figure out \"" + systemName + "\" system factory for log4j StreamAppender to use"));
    SystemFactory systemFactory = ReflectionUtil.getObj(systemFactoryName, SystemFactory.class);
    setSerde(log4jSystemConfig, systemName, streamName);
    if (config.getBoolean(CREATE_STREAM_ENABLED, false)) {
        int streamPartitionCount = getPartitionCount();
        System.out.println("[StreamAppender] creating stream " + streamName + " with partition count " + streamPartitionCount);
        StreamSpec streamSpec = StreamSpec.createStreamAppenderStreamSpec(streamName, systemName, streamPartitionCount);
        // SystemAdmin only needed for stream creation here.
        SystemAdmin systemAdmin = systemFactory.getAdmin(systemName, config);
        systemAdmin.start();
        systemAdmin.createStream(streamSpec);
        systemAdmin.stop();
    }
    systemProducer = systemFactory.getProducer(systemName, config, metricsRegistry, this.getClass().getSimpleName());
    systemStream = new SystemStream(systemName, streamName);
    systemProducer.register(SOURCE);
    systemProducer.start();
    log.info(SOURCE + " has been registered in " + systemName + ". So all the logs will be sent to " + streamName + " in " + systemName + ". Logs are partitioned by " + key);
    startTransferThread();
}