Examples with SystemStream org.apache.samza.system.SystemStream used on opensource projects

Example 46 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestCoordinatorStreamSystemProducer method testCoordinatorStreamSystemProducer.

@Test
public void testCoordinatorStreamSystemProducer() {
    MockCoordinatorStreamSystemFactory.enableMockConsumerCache();
    String source = "source";
    SystemStream systemStream = new SystemStream("system", "stream");
    MockCoordinatorSystemProducer systemProducer = new MockCoordinatorSystemProducer(source);
    MockSystemAdmin systemAdmin = new MockSystemAdmin();
    CoordinatorStreamSystemProducer producer = new CoordinatorStreamSystemProducer(systemStream, systemProducer, systemAdmin);
    SetConfig setConfig1 = new SetConfig(source, "job.name", "my-job-name");
    SetConfig setConfig2 = new SetConfig(source, "job.id", "1234");
    Delete delete = new Delete(source, "job.name", SetConfig.TYPE);
    assertFalse(systemProducer.isRegistered());
    producer.register(source);
    assertTrue(systemProducer.isRegistered());
    assertFalse(systemProducer.isStarted());
    producer.start();
    assertTrue(systemProducer.isStarted());
    producer.send(setConfig1);
    producer.send(setConfig2);
    producer.send(delete);
    assertFalse(systemProducer.isStopped());
    producer.stop();
    assertTrue(systemProducer.isStopped());
    List<OutgoingMessageEnvelope> envelopes = systemProducer.getEnvelopes();
    OutgoingMessageEnvelope envelope0 = envelopes.get(0);
    OutgoingMessageEnvelope envelope1 = envelopes.get(1);
    OutgoingMessageEnvelope envelope2 = envelopes.get(2);
    TypeReference<Object[]> keyRef = new TypeReference<Object[]>() {
    };
    TypeReference<Map<String, Object>> msgRef = new TypeReference<Map<String, Object>>() {
    };
    assertEquals(3, envelopes.size());
    assertEquals(new CoordinatorStreamMessage(setConfig1), new CoordinatorStreamMessage(deserialize((byte[]) envelope0.getKey(), keyRef), deserialize((byte[]) envelope0.getMessage(), msgRef)));
    assertEquals(new CoordinatorStreamMessage(setConfig2), new CoordinatorStreamMessage(deserialize((byte[]) envelope1.getKey(), keyRef), deserialize((byte[]) envelope1.getMessage(), msgRef)));
    assertEquals(new CoordinatorStreamMessage(delete), new CoordinatorStreamMessage(deserialize((byte[]) envelope2.getKey(), keyRef), deserialize((byte[]) envelope2.getMessage(), msgRef)));
}

Example 47 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestJobModelCalculator method testWithRegexTopicRewriters.

@Test
public void testWithRegexTopicRewriters() {
    // this is the SystemStream that is directly in the config
    SystemStream existingSystemStream = new SystemStream("existingSystem", "existingStream");
    addStreamMetadataCacheMetadata(this.streamMetadataCache, ImmutableMap.of(SYSTEM_STREAM0, buildSystemStreamMetadata(4), SYSTEM_STREAM1, buildSystemStreamMetadata(3), existingSystemStream, buildSystemStreamMetadata(1)));
    Map<TaskName, Integer> changelogPartitionMapping = changelogPartitionMapping(4);
    PowerMockito.mockStatic(ConfigUtil.class);
    // add SYSTEM_STREAM0 for one rewriter
    PowerMockito.when(ConfigUtil.applyRewriter(any(), eq(REGEX_REWRITER0))).thenAnswer(invocation -> addSystemStreamInput(SYSTEM_STREAM0, invocation.getArgumentAt(0, Config.class)));
    // add SYSTEM_STREAM1 for another rewriter
    PowerMockito.when(ConfigUtil.applyRewriter(any(), eq(REGEX_REWRITER1))).thenAnswer(invocation -> addSystemStreamInput(SYSTEM_STREAM1, invocation.getArgumentAt(0, Config.class)));
    Config config = config(ImmutableList.of(existingSystemStream), ImmutableMap.of(JobConfig.CONFIG_REWRITERS, String.format("%s,%s", REGEX_REWRITER0, REGEX_REWRITER1), String.format(JobConfig.CONFIG_REWRITER_CLASS, REGEX_REWRITER0), RegExTopicGenerator.class.getName(), String.format(JobConfig.CONFIG_REWRITER_CLASS, REGEX_REWRITER1), RegExTopicGenerator.class.getName()));
    Set<SystemStreamPartition> sspsForTask0 = ImmutableSet.of(new SystemStreamPartition(SYSTEM_STREAM0, new Partition(0)), new SystemStreamPartition(SYSTEM_STREAM1, new Partition(0)), new SystemStreamPartition(existingSystemStream, new Partition(0)));
    TaskModel taskModel0 = new TaskModel(taskName(0), sspsForTask0, new Partition(0));
    Map<String, ContainerModel> containerModels = ImmutableMap.of("0", new ContainerModel("0", ImmutableMap.of(taskName(0), taskModel0, taskName(2), taskModel(2, 2, 2))), "1", new ContainerModel("1", ImmutableMap.of(taskName(1), taskModel(1, 1, 1), taskName(3), taskModel(3, 3))));
    Map<String, String> expectedConfigMap = new HashMap<>(config);
    expectedConfigMap.put(TaskConfig.INPUT_STREAMS, String.format("%s,%s,%s", taskInputString(existingSystemStream), taskInputString(SYSTEM_STREAM0), taskInputString(SYSTEM_STREAM1)));
    JobModel expected = new JobModel(new MapConfig(expectedConfigMap), containerModels);
    JobModel actual = JobModelCalculator.INSTANCE.calculateJobModel(config, changelogPartitionMapping, this.streamMetadataCache, this.grouperMetadata);
    assertEquals(expected, actual);
}

Example 48 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class JobModelCalculator method getInputStreamPartitions.

/**
 * Computes the input system stream partitions of a samza job using the provided {@param config}
 * and {@param streamMetadataCache}.
 * @param config the configuration of the job.
 * @param streamMetadataCache to query the partition metadata of the input streams.
 * @return the input {@see SystemStreamPartition} of the samza job.
 */
private static Set<SystemStreamPartition> getInputStreamPartitions(Config config, StreamMetadataCache streamMetadataCache) {
    TaskConfig taskConfig = new TaskConfig(config);
    // Get the set of partitions for each SystemStream from the stream metadata
    Map<SystemStream, SystemStreamMetadata> allMetadata = JavaConverters.mapAsJavaMapConverter(streamMetadataCache.getStreamMetadata(JavaConverters.asScalaSetConverter(taskConfig.getInputStreams()).asScala().toSet(), true)).asJava();
    Set<SystemStreamPartition> inputStreamPartitions = new HashSet<>();
    allMetadata.forEach((systemStream, systemStreamMetadata) -> systemStreamMetadata.getSystemStreamPartitionMetadata().keySet().forEach(partition -> inputStreamPartitions.add(new SystemStreamPartition(systemStream, partition))));
    return inputStreamPartitions;
}

Example 49 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class StreamRegexMonitorFactory method build.

/**
 * Build a {@link StreamRegexMonitor} for input streams for the job model.
 */
public Optional<StreamRegexMonitor> build(JobModel jobModel, Config config, StreamRegexMonitor.Callback callback) {
    JobConfig jobConfig = new JobConfig(config);
    // if input regex monitor is not enabled return empty
    if (jobConfig.getMonitorRegexDisabled()) {
        LOG.info("StreamRegexMonitor is disabled.");
        return Optional.empty();
    }
    Set<SystemStream> inputStreamsToMonitor = JobModelUtil.getSystemStreams(jobModel);
    if (inputStreamsToMonitor.isEmpty()) {
        throw new SamzaException("Input streams to a job can not be empty.");
    }
    // First list all rewriters
    Optional<String> rewritersList = jobConfig.getConfigRewriters();
    // if no rewriter is defined, there is nothing to monitor
    if (!rewritersList.isPresent()) {
        LOG.warn("No config rewriters are defined. No StreamRegexMonitor created.");
        return Optional.empty();
    }
    // Compile a map of each input-system to its corresponding input-monitor-regex patterns
    Map<String, Pattern> inputRegexesToMonitor = jobConfig.getMonitorRegexPatternMap(rewritersList.get());
    // if there are no regexes to monitor
    if (inputRegexesToMonitor.isEmpty()) {
        LOG.info("No input regexes are defined. No StreamRegexMonitor created.");
        return Optional.empty();
    }
    return Optional.of(new StreamRegexMonitor(inputStreamsToMonitor, inputRegexesToMonitor, this.streamMetadataCache, this.metrics, jobConfig.getMonitorRegexFrequency(), callback));
}

Example 50 with SystemStream

use of org.apache.samza.system.SystemStream in project samza by apache.

the class TestOperatorImplGraph method testBroadcastChain.

@Test
public void testBroadcastChain() {
    String inputStreamId = "input";
    String inputSystem = "input-system";
    String inputPhysicalName = "input-stream";
    HashMap<String, String> configMap = new HashMap<>();
    configMap.put(JobConfig.JOB_NAME, "test-job");
    configMap.put(JobConfig.JOB_ID, "1");
    StreamTestUtils.addStreamConfigs(configMap, inputStreamId, inputSystem, inputPhysicalName);
    Config config = new MapConfig(configMap);
    when(this.context.getJobContext().getConfig()).thenReturn(config);
    StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> {
        GenericSystemDescriptor sd = new GenericSystemDescriptor(inputSystem, "mockFactoryClass");
        GenericInputDescriptor inputDescriptor = sd.getInputDescriptor(inputStreamId, mock(Serde.class));
        MessageStream<Object> inputStream = appDesc.getInputStream(inputDescriptor);
        inputStream.filter(mock(FilterFunction.class));
        inputStream.map(mock(MapFunction.class));
    }, config);
    OperatorImplGraph opImplGraph = new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), this.context, mock(Clock.class));
    InputOperatorImpl inputOpImpl = opImplGraph.getInputOperator(new SystemStream(inputSystem, inputPhysicalName));
    assertEquals(2, inputOpImpl.registeredOperators.size());
    assertTrue(inputOpImpl.registeredOperators.stream().anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.FILTER));
    assertTrue(inputOpImpl.registeredOperators.stream().anyMatch(opImpl -> ((OperatorImpl) opImpl).getOperatorSpec().getOpCode() == OpCode.MAP));
}