Examples with StreamValidationException org.apache.samza.system.StreamValidationException used on opensource projects

Example 1 with StreamValidationException

use of org.apache.samza.system.StreamValidationException in project samza by apache.

the class KafkaSystemAdmin method toKafkaSpec.

/**
 * Converts a StreamSpec into a KafkaStreamSpec. Special handling for coordinator and changelog stream.
 * @param spec a StreamSpec object
 * @return KafkaStreamSpec object
 */
public KafkaStreamSpec toKafkaSpec(StreamSpec spec) {
    KafkaStreamSpec kafkaSpec;
    if (spec.isChangeLogStream()) {
        String topicName = spec.getPhysicalName();
        ChangelogInfo topicMeta = changelogTopicMetaInformation.get(topicName);
        if (topicMeta == null) {
            throw new StreamValidationException("Unable to find topic information for topic " + topicName);
        }
        kafkaSpec = new KafkaStreamSpec(spec.getId(), topicName, systemName, spec.getPartitionCount(), topicMeta.getReplicationFactor(), topicMeta.getKafkaProperties());
    } else if (spec.isCoordinatorStream()) {
        kafkaSpec = new KafkaStreamSpec(spec.getId(), spec.getPhysicalName(), systemName, 1, coordinatorStreamReplicationFactor, coordinatorStreamProperties);
    } else if (spec.isCheckpointStream()) {
        Properties checkpointTopicProperties = new Properties();
        checkpointTopicProperties.putAll(spec.getConfig());
        kafkaSpec = KafkaStreamSpec.fromSpec(StreamSpec.createCheckpointStreamSpec(spec.getPhysicalName(), spec.getSystemName())).copyWithReplicationFactor(Integer.parseInt(new KafkaConfig(config).getCheckpointReplicationFactor().get())).copyWithProperties(checkpointTopicProperties);
    } else if (intermediateStreamProperties.containsKey(spec.getId())) {
        kafkaSpec = KafkaStreamSpec.fromSpec(spec);
        Properties properties = kafkaSpec.getProperties();
        properties.putAll(intermediateStreamProperties.get(spec.getId()));
        kafkaSpec = kafkaSpec.copyWithProperties(properties);
    } else {
        kafkaSpec = KafkaStreamSpec.fromSpec(spec);
        // we check if there is a system-level rf config specified, else we use KafkaConfig.topic-default-rf
        int replicationFactorFromSystemConfig = Integer.valueOf(new KafkaConfig(config).getSystemDefaultReplicationFactor(spec.getSystemName(), KafkaConfig.TOPIC_DEFAULT_REPLICATION_FACTOR()));
        LOG.info("Using replication-factor: {} for StreamSpec: {}", replicationFactorFromSystemConfig, spec);
        return new KafkaStreamSpec(kafkaSpec.getId(), kafkaSpec.getPhysicalName(), kafkaSpec.getSystemName(), kafkaSpec.getPartitionCount(), replicationFactorFromSystemConfig, kafkaSpec.getProperties());
    }
    return kafkaSpec;
}

Example 2 with StreamValidationException

use of org.apache.samza.system.StreamValidationException in project samza by apache.

the class TestKafkaCheckpointManager method testCreateResourcesTopicValidationError.

@Test(expected = StreamValidationException.class)
public void testCreateResourcesTopicValidationError() {
    setupSystemFactory(config());
    // throw an exception during validateStream
    doThrow(new StreamValidationException("invalid stream")).when(this.createResourcesSystemAdmin).validateStream(CHECKPOINT_SPEC);
    KafkaCheckpointManager checkpointManager = buildKafkaCheckpointManager(true, config());
    // expect an exception during startup
    checkpointManager.createResources();
}

Example 3 with StreamValidationException

use of org.apache.samza.system.StreamValidationException in project samza by apache.

the class KafkaSystemAdmin method validateStream.

@Override
public void validateStream(StreamSpec streamSpec) throws StreamValidationException {
    LOG.info("About to validate stream = " + streamSpec);
    String streamName = streamSpec.getPhysicalName();
    SystemStreamMetadata systemStreamMetadata = getSystemStreamMetadata(Collections.singleton(streamName)).get(streamName);
    if (systemStreamMetadata == null) {
        throw new StreamValidationException("Failed to obtain metadata for stream " + streamName + ". Validation failed.");
    }
    int actualPartitionCounter = systemStreamMetadata.getSystemStreamPartitionMetadata().size();
    int expectedPartitionCounter = streamSpec.getPartitionCount();
    LOG.info("actualCount=" + actualPartitionCounter + "; expectedCount=" + expectedPartitionCounter);
    if (actualPartitionCounter != expectedPartitionCounter) {
        throw new StreamValidationException(String.format("Mismatch of partitions for stream %s. Expected %d, got %d. Validation failed.", streamName, expectedPartitionCounter, actualPartitionCounter));
    }
}

Example 4 with StreamValidationException

use of org.apache.samza.system.StreamValidationException in project samza by apache.

the class TestKafkaSystemAdminJava method testToKafkaSpec.

@Test
public void testToKafkaSpec() {
    String topicName = "testStream";
    int defaultPartitionCount = 2;
    int changeLogPartitionFactor = 5;
    Map<String, String> map = new HashMap<>();
    Config config = new MapConfig(map);
    StreamSpec spec = new StreamSpec("id", topicName, SYSTEM, defaultPartitionCount, config);
    KafkaSystemAdmin kafkaAdmin = systemAdmin();
    KafkaStreamSpec kafkaSpec = kafkaAdmin.toKafkaSpec(spec);
    Assert.assertEquals("id", kafkaSpec.getId());
    Assert.assertEquals(topicName, kafkaSpec.getPhysicalName());
    Assert.assertEquals(SYSTEM, kafkaSpec.getSystemName());
    Assert.assertEquals(defaultPartitionCount, kafkaSpec.getPartitionCount());
    // validate that conversion is using coordination metadata
    map.put("job.coordinator.segment.bytes", "123");
    map.put("job.coordinator.cleanup.policy", "superCompact");
    int coordReplicatonFactor = 4;
    map.put(org.apache.samza.config.KafkaConfig.JOB_COORDINATOR_REPLICATION_FACTOR(), String.valueOf(coordReplicatonFactor));
    KafkaSystemAdmin admin = Mockito.spy(createSystemAdmin(SYSTEM, map));
    spec = StreamSpec.createCoordinatorStreamSpec(topicName, SYSTEM);
    kafkaSpec = admin.toKafkaSpec(spec);
    Assert.assertEquals(coordReplicatonFactor, kafkaSpec.getReplicationFactor());
    Assert.assertEquals("123", kafkaSpec.getProperties().getProperty("segment.bytes"));
    // cleanup policy is overridden in the KafkaAdmin
    Assert.assertEquals("compact", kafkaSpec.getProperties().getProperty("cleanup.policy"));
    // validate that conversion is using changeLog metadata
    map = new HashMap<>();
    map.put(JobConfig.JOB_DEFAULT_SYSTEM, SYSTEM);
    map.put(String.format("stores.%s.changelog", "fakeStore"), topicName);
    int changeLogReplicationFactor = 3;
    map.put(String.format("stores.%s.changelog.replication.factor", "fakeStore"), String.valueOf(changeLogReplicationFactor));
    admin = Mockito.spy(createSystemAdmin(SYSTEM, map));
    spec = StreamSpec.createChangeLogStreamSpec(topicName, SYSTEM, changeLogPartitionFactor);
    kafkaSpec = admin.toKafkaSpec(spec);
    Assert.assertEquals(changeLogReplicationFactor, kafkaSpec.getReplicationFactor());
    // same, but with missing topic info
    try {
        admin = Mockito.spy(createSystemAdmin(SYSTEM, map));
        spec = StreamSpec.createChangeLogStreamSpec("anotherTopic", SYSTEM, changeLogPartitionFactor);
        kafkaSpec = admin.toKafkaSpec(spec);
        Assert.fail("toKafkaSpec should've failed for missing topic");
    } catch (StreamValidationException e) {
    // expected
    }
    // validate that conversion is using intermediate streams properties
    String interStreamId = "isId";
    Map<String, String> interStreamMap = new HashMap<>();
    interStreamMap.put("app.mode", ApplicationConfig.ApplicationMode.BATCH.toString());
    interStreamMap.put(String.format("streams.%s.samza.intermediate", interStreamId), "true");
    interStreamMap.put(String.format("streams.%s.samza.system", interStreamId), "testSystem");
    interStreamMap.put(String.format("streams.%s.p1", interStreamId), "v1");
    interStreamMap.put(String.format("streams.%s.retention.ms", interStreamId), "123");
    // legacy format
    interStreamMap.put(String.format("systems.%s.streams.%s.p2", "testSystem", interStreamId), "v2");
    admin = Mockito.spy(createSystemAdmin(SYSTEM, interStreamMap));
    spec = new StreamSpec(interStreamId, topicName, SYSTEM, defaultPartitionCount, config);
    kafkaSpec = admin.toKafkaSpec(spec);
    Assert.assertEquals("v1", kafkaSpec.getProperties().getProperty("p1"));
    Assert.assertEquals("v2", kafkaSpec.getProperties().getProperty("p2"));
    Assert.assertEquals("123", kafkaSpec.getProperties().getProperty("retention.ms"));
    Assert.assertEquals(defaultPartitionCount, kafkaSpec.getPartitionCount());
}