Examples with SystemAdmin org.apache.samza.system.SystemAdmin used on opensource projects

Example 6 with SystemAdmin

use of org.apache.samza.system.SystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testValidateChangelogStreamDelegatesToCreateStream_specialCharsInTopicName.

@Test
public void testValidateChangelogStreamDelegatesToCreateStream_specialCharsInTopicName() {
    final String STREAM = "test.Change_Log.Validate";
    Properties coordProps = new Properties();
    Map<String, ChangelogInfo> changeLogMap = new HashMap<>();
    changeLogMap.put(STREAM, new ChangelogInfo(3, new Properties()));
    KafkaSystemAdmin systemAdmin = createSystemAdmin(coordProps, 3, Util.javaMapAsScalaMap(changeLogMap));
    SystemAdmin admin = Mockito.spy(systemAdmin);
    StreamSpec spec = new StreamSpec("testId", STREAM, "testSystem", 12);
    admin.createChangelogStream(spec.getPhysicalName(), spec.getPartitionCount());
    admin.validateStream(spec);
    // Should not throw
    admin.validateChangelogStream(STREAM, spec.getPartitionCount());
    Mockito.verify(admin).createStream(Mockito.any());
    Mockito.verify(admin, Mockito.times(3)).validateStream(Mockito.any());
}

Example 7 with SystemAdmin

use of org.apache.samza.system.SystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testCreateChangelogStreamDelegatesToCreateStream_specialCharsInTopicName.

@Test
public void testCreateChangelogStreamDelegatesToCreateStream_specialCharsInTopicName() {
    final String STREAM = "test.Change_Log.Stream";
    final int PARTITIONS = 12;
    final int REP_FACTOR = 3;
    Properties coordProps = new Properties();
    Properties changeLogProps = new Properties();
    changeLogProps.setProperty("cleanup.policy", "compact");
    changeLogProps.setProperty("segment.bytes", "139");
    Map<String, ChangelogInfo> changeLogMap = new HashMap<>();
    changeLogMap.put(STREAM, new ChangelogInfo(REP_FACTOR, changeLogProps));
    SystemAdmin admin = Mockito.spy(createSystemAdmin(coordProps, 3, Util.javaMapAsScalaMap(changeLogMap)));
    StreamSpec spec = new StreamSpec(KafkaSystemAdmin.CHANGELOG_STREAMID(), STREAM, SYSTEM(), PARTITIONS);
    admin.createChangelogStream(STREAM, PARTITIONS);
    admin.validateStream(spec);
    ArgumentCaptor<StreamSpec> specCaptor = ArgumentCaptor.forClass(StreamSpec.class);
    Mockito.verify(admin).createStream(specCaptor.capture());
    StreamSpec internalSpec = specCaptor.getValue();
    // KafkaStreamSpec is used to carry replication factor
    assertTrue(internalSpec instanceof KafkaStreamSpec);
    assertEquals(KafkaSystemAdmin.CHANGELOG_STREAMID(), internalSpec.getId());
    assertEquals(SYSTEM(), internalSpec.getSystemName());
    assertEquals(STREAM, internalSpec.getPhysicalName());
    assertEquals(REP_FACTOR, ((KafkaStreamSpec) internalSpec).getReplicationFactor());
    assertEquals(PARTITIONS, internalSpec.getPartitionCount());
    assertEquals(changeLogProps, ((KafkaStreamSpec) internalSpec).getProperties());
}

Example 8 with SystemAdmin

use of org.apache.samza.system.SystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testCreateChangelogStreamDelegatesToCreateStream.

@Test
public void testCreateChangelogStreamDelegatesToCreateStream() {
    final String STREAM = "testChangeLogStream";
    final int PARTITIONS = 12;
    final int REP_FACTOR = 3;
    Properties coordProps = new Properties();
    Properties changeLogProps = new Properties();
    changeLogProps.setProperty("cleanup.policy", "compact");
    changeLogProps.setProperty("segment.bytes", "139");
    Map<String, ChangelogInfo> changeLogMap = new HashMap<>();
    changeLogMap.put(STREAM, new ChangelogInfo(REP_FACTOR, changeLogProps));
    SystemAdmin admin = Mockito.spy(createSystemAdmin(coordProps, 3, Util.javaMapAsScalaMap(changeLogMap)));
    StreamSpec spec = new StreamSpec(KafkaSystemAdmin.CHANGELOG_STREAMID(), STREAM, SYSTEM(), PARTITIONS);
    admin.createChangelogStream(STREAM, PARTITIONS);
    admin.validateStream(spec);
    ArgumentCaptor<StreamSpec> specCaptor = ArgumentCaptor.forClass(StreamSpec.class);
    Mockito.verify(admin).createStream(specCaptor.capture());
    StreamSpec internalSpec = specCaptor.getValue();
    // KafkaStreamSpec is used to carry replication factor
    assertTrue(internalSpec instanceof KafkaStreamSpec);
    assertEquals(KafkaSystemAdmin.CHANGELOG_STREAMID(), internalSpec.getId());
    assertEquals(SYSTEM(), internalSpec.getSystemName());
    assertEquals(STREAM, internalSpec.getPhysicalName());
    assertEquals(REP_FACTOR, ((KafkaStreamSpec) internalSpec).getReplicationFactor());
    assertEquals(PARTITIONS, internalSpec.getPartitionCount());
    assertEquals(changeLogProps, ((KafkaStreamSpec) internalSpec).getProperties());
}

Example 9 with SystemAdmin

use of org.apache.samza.system.SystemAdmin in project samza by apache.

the class SamzaTaskProxy method getJobModel.

/**
   * Retrieves the jobModel from the jobCoordinator.
   * @param jobInstance the job instance (jobId, jobName).
   * @return the JobModel fetched from the coordinator stream.
   */
protected JobModel getJobModel(JobInstance jobInstance) {
    CoordinatorStreamSystemConsumer coordinatorSystemConsumer = null;
    CoordinatorStreamSystemProducer coordinatorSystemProducer = null;
    try {
        CoordinatorStreamSystemFactory coordinatorStreamSystemFactory = new CoordinatorStreamSystemFactory();
        Config coordinatorSystemConfig = getCoordinatorSystemConfig(jobInstance);
        LOG.info("Using config: {} to create coordinatorStream producer and consumer.", coordinatorSystemConfig);
        coordinatorSystemConsumer = coordinatorStreamSystemFactory.getCoordinatorStreamSystemConsumer(coordinatorSystemConfig, METRICS_REGISTRY);
        coordinatorSystemProducer = coordinatorStreamSystemFactory.getCoordinatorStreamSystemProducer(coordinatorSystemConfig, METRICS_REGISTRY);
        LOG.info("Registering coordinator system stream consumer.");
        coordinatorSystemConsumer.register();
        LOG.debug("Starting coordinator system stream consumer.");
        coordinatorSystemConsumer.start();
        LOG.debug("Bootstrapping coordinator system stream consumer.");
        coordinatorSystemConsumer.bootstrap();
        LOG.info("Registering coordinator system stream producer.");
        coordinatorSystemProducer.register(SOURCE);
        Config config = coordinatorSystemConsumer.getConfig();
        LOG.info("Got config from coordinatorSystemConsumer: {}.", config);
        ChangelogPartitionManager changelogManager = new ChangelogPartitionManager(coordinatorSystemProducer, coordinatorSystemConsumer, SOURCE);
        changelogManager.start();
        LocalityManager localityManager = new LocalityManager(coordinatorSystemProducer, coordinatorSystemConsumer);
        localityManager.start();
        String jobCoordinatorSystemName = config.get(JobConfig.JOB_COORDINATOR_SYSTEM());
        /**
       * Select job coordinator system properties from config and instantiate SystemAdmin for it alone.
       * Instantiating SystemAdmin's for other input/output systems defined in config is unnecessary.
       */
        Config systemAdminConfig = config.subset(String.format("systems.%s", jobCoordinatorSystemName), false);
        scala.collection.immutable.Map<String, SystemAdmin> systemAdmins = JobModelManager.getSystemAdmins(systemAdminConfig);
        StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance());
        Map<TaskName, Integer> changeLogPartitionMapping = changelogManager.readChangeLogPartitionMapping();
        return JobModelManager.readJobModel(config, changeLogPartitionMapping, localityManager, streamMetadataCache, null);
    } finally {
        if (coordinatorSystemConsumer != null) {
            coordinatorSystemConsumer.stop();
        }
        if (coordinatorSystemProducer != null) {
            coordinatorSystemProducer.stop();
        }
    }
}

Example 10 with SystemAdmin

use of org.apache.samza.system.SystemAdmin in project samza by apache.

the class TestKafkaSystemAdminJava method testValidateStreamDoesNotExist.

@Test(expected = StreamValidationException.class)
public void testValidateStreamDoesNotExist() {
    SystemAdmin admin = this.basicSystemAdmin;
    StreamSpec spec = new StreamSpec("testId", "testStreamNameExist", "testSystem", 8);
    admin.validateStream(spec);
}