Examples with KafkaServiceConfiguration io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration used on opensource projects

Example 1 with KafkaServiceConfiguration

use of io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration in project kop by streamnative.

the class SaslPlainEndToEndTest method setup.

@BeforeClass
@Override
protected void setup() throws Exception {
    SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256);
    AuthenticationProviderToken provider = new AuthenticationProviderToken();
    Properties properties = new Properties();
    properties.setProperty("tokenSecretKey", AuthTokenUtils.encodeKeyBase64(secretKey));
    ServiceConfiguration authConf = new ServiceConfiguration();
    authConf.setProperties(properties);
    provider.initialize(authConf);
    String adminToken = AuthTokenUtils.createToken(secretKey, ADMIN_USER, Optional.empty());
    userToken = AuthTokenUtils.createToken(secretKey, SIMPLE_USER, Optional.empty());
    anotherToken = AuthTokenUtils.createToken(secretKey, ANOTHER_USER, Optional.empty());
    super.resetConfig();
    conf.setKopAllowedNamespaces(Collections.singleton(TENANT + "/" + NAMESPACE));
    ((KafkaServiceConfiguration) conf).setSaslAllowedMechanisms(Sets.newHashSet("PLAIN"));
    ((KafkaServiceConfiguration) conf).setKafkaMetadataTenant("internal");
    ((KafkaServiceConfiguration) conf).setKafkaMetadataNamespace("__kafka");
    conf.setClusterName(super.configClusterName);
    conf.setAuthorizationEnabled(true);
    conf.setAuthenticationEnabled(true);
    conf.setAuthorizationAllowWildcardsMatching(true);
    conf.setSuperUserRoles(Sets.newHashSet(ADMIN_USER));
    conf.setAuthenticationProviders(Sets.newHashSet("org.apache.pulsar.broker.authentication." + "AuthenticationProviderToken"));
    conf.setBrokerClientAuthenticationPlugin(AuthenticationToken.class.getName());
    conf.setBrokerClientAuthenticationParameters("token:" + adminToken);
    conf.setProperties(properties);
    super.internalSetup();
    admin.topics().createPartitionedTopic(TOPIC, 1);
    admin.namespaces().grantPermissionOnNamespace(TENANT + "/" + NAMESPACE, SIMPLE_USER, Sets.newHashSet(AuthAction.consume, AuthAction.produce));
}

Example 2 with KafkaServiceConfiguration

use of io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration in project kop by streamnative.

the class MetadataUtilsTest method testCreateKafkaMetadataIfMissing.

@Test(timeOut = 30000)
public void testCreateKafkaMetadataIfMissing() throws Exception {
    String namespacePrefix = "public/default";
    KafkaServiceConfiguration conf = new KafkaServiceConfiguration();
    assertTrue(conf.isKafkaManageSystemNamespaces());
    ClusterData clusterData = ClusterData.builder().build();
    conf.setClusterName("test");
    conf.setKafkaMetadataTenant("public");
    conf.setKafkaMetadataNamespace("default");
    conf.setSuperUserRoles(Sets.newHashSet("admin"));
    conf.setOffsetsTopicNumPartitions(8);
    final KopTopic offsetsTopic = new KopTopic(MetadataUtils.constructOffsetsTopicBaseName(conf.getKafkaMetadataTenant(), conf), namespacePrefix);
    final KopTopic txnTopic = new KopTopic(MetadataUtils.constructTxnLogTopicBaseName(conf.getKafkaMetadataTenant(), conf), namespacePrefix);
    List<String> emptyList = Lists.newArrayList();
    List<String> existingClusters = Lists.newArrayList("test");
    Clusters mockClusters = mock(Clusters.class);
    doReturn(existingClusters).when(mockClusters).getClusters();
    Tenants mockTenants = mock(Tenants.class);
    doReturn(emptyList).when(mockTenants).getTenants();
    Namespaces mockNamespaces = mock(Namespaces.class);
    doReturn(emptyList).when(mockNamespaces).getNamespaces("public");
    PartitionedTopicMetadata offsetTopicMetadata = new PartitionedTopicMetadata();
    Topics mockTopics = mock(Topics.class);
    doReturn(offsetTopicMetadata).when(mockTopics).getPartitionedTopicMetadata(eq(offsetsTopic.getFullName()));
    doReturn(offsetTopicMetadata).when(mockTopics).getPartitionedTopicMetadata(eq(txnTopic.getFullName()));
    PulsarAdmin mockPulsarAdmin = mock(PulsarAdmin.class);
    doReturn(mockClusters).when(mockPulsarAdmin).clusters();
    doReturn(mockTenants).when(mockPulsarAdmin).tenants();
    doReturn(mockNamespaces).when(mockPulsarAdmin).namespaces();
    doReturn(mockTopics).when(mockPulsarAdmin).topics();
    TenantInfo partialTenant = TenantInfo.builder().build();
    doReturn(partialTenant).when(mockTenants).getTenantInfo(eq(conf.getKafkaMetadataTenant()));
    MetadataUtils.createOffsetMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    // After call the createOffsetMetadataIfMissing, these methods should return expected data.
    doReturn(Lists.newArrayList(conf.getKafkaMetadataTenant())).when(mockTenants).getTenants();
    String namespace = conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace();
    doReturn(Lists.newArrayList(namespace)).when(mockNamespaces).getNamespaces(conf.getKafkaMetadataTenant());
    doReturn(Lists.newArrayList(conf.getClusterName())).when(mockNamespaces).getNamespaceReplicationClusters(eq(namespace));
    MetadataUtils.createTxnMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    verify(mockTenants, times(1)).createTenant(eq(conf.getKafkaMetadataTenant()), any(TenantInfo.class));
    verify(mockNamespaces, times(1)).createNamespace(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Set.class));
    verify(mockNamespaces, times(1)).setNamespaceReplicationClusters(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Set.class));
    verify(mockNamespaces, times(2)).setRetention(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(RetentionPolicies.class));
    verify(mockNamespaces, times(2)).setNamespaceMessageTTL(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Integer.class));
    verify(mockTopics, times(1)).createPartitionedTopic(eq(offsetsTopic.getFullName()), eq(conf.getOffsetsTopicNumPartitions()));
    verify(mockTopics, times(1)).createPartitionedTopic(eq(txnTopic.getFullName()), eq(conf.getKafkaTxnLogTopicNumPartitions()));
    // Test that cluster is added to existing Tenant if missing
    // Test that the cluster is added to the namespace replication cluster list if it is missing
    // Test that missing offset topic partitions are created
    reset(mockTenants);
    reset(mockNamespaces);
    reset(mockTopics);
    doReturn(Lists.newArrayList("public")).when(mockTenants).getTenants();
    partialTenant = TenantInfo.builder().adminRoles(conf.getSuperUserRoles()).allowedClusters(Sets.newHashSet("other-cluster")).build();
    doReturn(partialTenant).when(mockTenants).getTenantInfo(eq(conf.getKafkaMetadataTenant()));
    doReturn(Lists.newArrayList("test")).when(mockNamespaces).getNamespaces("public");
    doReturn(emptyList).when(mockNamespaces).getNamespaceReplicationClusters(eq(conf.getKafkaMetadataTenant()));
    List<String> incompletePartitionList = new ArrayList<String>(conf.getOffsetsTopicNumPartitions());
    for (int i = 0; i < conf.getOffsetsTopicNumPartitions() - 2; i++) {
        incompletePartitionList.add(offsetsTopic.getPartitionName(i));
    }
    for (int i = 0; i < conf.getKafkaTxnLogTopicNumPartitions() - 2; i++) {
        incompletePartitionList.add(txnTopic.getPartitionName(i));
    }
    doReturn(new PartitionedTopicMetadata(8)).when(mockTopics).getPartitionedTopicMetadata(eq(offsetsTopic.getFullName()));
    doReturn(new PartitionedTopicMetadata(8)).when(mockTopics).getPartitionedTopicMetadata(eq(txnTopic.getFullName()));
    doReturn(incompletePartitionList).when(mockTopics).getList(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()));
    MetadataUtils.createOffsetMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    MetadataUtils.createTxnMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    verify(mockTenants, times(1)).updateTenant(eq(conf.getKafkaMetadataTenant()), any(TenantInfo.class));
    verify(mockNamespaces, times(2)).setNamespaceReplicationClusters(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Set.class));
    verify(mockTopics, times(1)).createMissedPartitions(contains(offsetsTopic.getOriginalName()));
    verify(mockTopics, times(1)).createMissedPartitions(contains(txnTopic.getOriginalName()));
}

Example 3 with KafkaServiceConfiguration

use of io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration in project starlight-for-kafka by datastax.

the class MetadataUtilsTest method testDisableCreateKafkaMetadata.

@Test(timeOut = 30000)
public void testDisableCreateKafkaMetadata() throws Exception {
    KafkaServiceConfiguration conf = new KafkaServiceConfiguration();
    conf.setKafkaManageSystemNamespaces(false);
    Clusters mockClusters = mock(Clusters.class);
    Tenants mockTenants = mock(Tenants.class);
    Namespaces mockNamespaces = mock(Namespaces.class);
    Topics mockTopics = mock(Topics.class);
    PulsarAdmin mockPulsarAdmin = mock(PulsarAdmin.class);
    doReturn(mockClusters).when(mockPulsarAdmin).clusters();
    doReturn(mockTenants).when(mockPulsarAdmin).tenants();
    doReturn(mockNamespaces).when(mockPulsarAdmin).namespaces();
    doReturn(mockTopics).when(mockPulsarAdmin).topics();
    MetadataUtils.createOffsetMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, ClusterData.builder().build(), conf);
    verify(mockTenants, times(0)).createTenant(any(), any());
    verify(mockNamespaces, times(0)).createNamespace(any(), any(Set.class));
}

Example 4 with KafkaServiceConfiguration

use of io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration in project starlight-for-kafka by datastax.

the class MetadataUtilsTest method testCreateKafkaMetadataIfMissing.

@Test(timeOut = 30000)
public void testCreateKafkaMetadataIfMissing() throws Exception {
    String namespacePrefix = "public/default";
    KafkaServiceConfiguration conf = new KafkaServiceConfiguration();
    assertTrue(conf.isKafkaManageSystemNamespaces());
    ClusterData clusterData = ClusterData.builder().build();
    conf.setClusterName("test");
    conf.setKafkaMetadataTenant("public");
    conf.setKafkaMetadataNamespace("default");
    conf.setSuperUserRoles(Sets.newHashSet("admin"));
    conf.setOffsetsTopicNumPartitions(8);
    final KopTopic offsetsTopic = new KopTopic(MetadataUtils.constructOffsetsTopicBaseName(conf.getKafkaMetadataTenant(), conf), namespacePrefix);
    final KopTopic txnTopic = new KopTopic(MetadataUtils.constructTxnLogTopicBaseName(conf.getKafkaMetadataTenant(), conf), namespacePrefix);
    List<String> emptyList = Lists.newArrayList();
    List<String> existingClusters = Lists.newArrayList("test");
    Clusters mockClusters = mock(Clusters.class);
    doReturn(existingClusters).when(mockClusters).getClusters();
    Tenants mockTenants = mock(Tenants.class);
    doReturn(emptyList).when(mockTenants).getTenants();
    Namespaces mockNamespaces = mock(Namespaces.class);
    doReturn(emptyList).when(mockNamespaces).getNamespaces("public");
    PartitionedTopicMetadata offsetTopicMetadata = new PartitionedTopicMetadata();
    Topics mockTopics = mock(Topics.class);
    doReturn(offsetTopicMetadata).when(mockTopics).getPartitionedTopicMetadata(eq(offsetsTopic.getFullName()));
    doReturn(offsetTopicMetadata).when(mockTopics).getPartitionedTopicMetadata(eq(txnTopic.getFullName()));
    PulsarAdmin mockPulsarAdmin = mock(PulsarAdmin.class);
    doReturn(mockClusters).when(mockPulsarAdmin).clusters();
    doReturn(mockTenants).when(mockPulsarAdmin).tenants();
    doReturn(mockNamespaces).when(mockPulsarAdmin).namespaces();
    doReturn(mockTopics).when(mockPulsarAdmin).topics();
    TenantInfo partialTenant = TenantInfo.builder().build();
    doReturn(partialTenant).when(mockTenants).getTenantInfo(eq(conf.getKafkaMetadataTenant()));
    MetadataUtils.createOffsetMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    // After call the createOffsetMetadataIfMissing, these methods should return expected data.
    doReturn(Lists.newArrayList(conf.getKafkaMetadataTenant())).when(mockTenants).getTenants();
    String namespace = conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace();
    doReturn(Lists.newArrayList(namespace)).when(mockNamespaces).getNamespaces(conf.getKafkaMetadataTenant());
    doReturn(Lists.newArrayList(conf.getClusterName())).when(mockNamespaces).getNamespaceReplicationClusters(eq(namespace));
    MetadataUtils.createTxnMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    verify(mockTenants, times(1)).createTenant(eq(conf.getKafkaMetadataTenant()), any(TenantInfo.class));
    verify(mockNamespaces, times(1)).createNamespace(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Set.class));
    verify(mockNamespaces, times(1)).setNamespaceReplicationClusters(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Set.class));
    verify(mockNamespaces, times(2)).setRetention(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(RetentionPolicies.class));
    verify(mockNamespaces, times(2)).setNamespaceMessageTTL(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Integer.class));
    verify(mockTopics, times(1)).createPartitionedTopic(eq(offsetsTopic.getFullName()), eq(conf.getOffsetsTopicNumPartitions()));
    verify(mockTopics, times(1)).createPartitionedTopic(eq(txnTopic.getFullName()), eq(conf.getKafkaTxnLogTopicNumPartitions()));
    // Test that cluster is added to existing Tenant if missing
    // Test that the cluster is added to the namespace replication cluster list if it is missing
    // Test that missing offset topic partitions are created
    reset(mockTenants);
    reset(mockNamespaces);
    reset(mockTopics);
    doReturn(Lists.newArrayList("public")).when(mockTenants).getTenants();
    partialTenant = TenantInfo.builder().adminRoles(conf.getSuperUserRoles()).allowedClusters(Sets.newHashSet("other-cluster")).build();
    doReturn(partialTenant).when(mockTenants).getTenantInfo(eq(conf.getKafkaMetadataTenant()));
    doReturn(Lists.newArrayList("test")).when(mockNamespaces).getNamespaces("public");
    doReturn(emptyList).when(mockNamespaces).getNamespaceReplicationClusters(eq(conf.getKafkaMetadataTenant()));
    List<String> incompletePartitionList = new ArrayList<String>(conf.getOffsetsTopicNumPartitions());
    for (int i = 0; i < conf.getOffsetsTopicNumPartitions() - 2; i++) {
        incompletePartitionList.add(offsetsTopic.getPartitionName(i));
    }
    for (int i = 0; i < conf.getKafkaTxnLogTopicNumPartitions() - 2; i++) {
        incompletePartitionList.add(txnTopic.getPartitionName(i));
    }
    doReturn(new PartitionedTopicMetadata(8)).when(mockTopics).getPartitionedTopicMetadata(eq(offsetsTopic.getFullName()));
    doReturn(new PartitionedTopicMetadata(8)).when(mockTopics).getPartitionedTopicMetadata(eq(txnTopic.getFullName()));
    doReturn(incompletePartitionList).when(mockTopics).getList(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()));
    MetadataUtils.createOffsetMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    MetadataUtils.createTxnMetadataIfMissing(conf.getKafkaMetadataTenant(), mockPulsarAdmin, clusterData, conf);
    verify(mockTenants, times(1)).updateTenant(eq(conf.getKafkaMetadataTenant()), any(TenantInfo.class));
    verify(mockNamespaces, times(2)).setNamespaceReplicationClusters(eq(conf.getKafkaMetadataTenant() + "/" + conf.getKafkaMetadataNamespace()), any(Set.class));
    verify(mockTopics, times(1)).createMissedPartitions(contains(offsetsTopic.getOriginalName()));
    verify(mockTopics, times(1)).createMissedPartitions(contains(txnTopic.getOriginalName()));
}

Example 5 with KafkaServiceConfiguration

use of io.streamnative.pulsar.handlers.kop.KafkaServiceConfiguration in project starlight-for-kafka by datastax.

the class TestSSLUtils method testKConfigTLSCiphers.

@Test
public void testKConfigTLSCiphers() throws Exception {
    // test KafkaServiceConfiguration get cipher config right from config file.
    KafkaServiceConfiguration kConfig = ConfigurationUtils.create(new FileInputStream(createConfigFileWithCiphers()), KafkaServiceConfiguration.class);
    Set<String> ciphers = kConfig.getKopSslCipherSuites();
    Assert.assertEquals(ciphers.size(), 2);
    ciphers.forEach(s -> {
        Assert.assertTrue(s.equals(cipherA) || s.equals(cipherB));
        log.debug("cipher: {}", s);
    });
    // test factory set ciphers right.
    SslContextFactory sslContextFactory = SSLUtils.createSslContextFactory(kConfig);
    String[] sslCiphers = sslContextFactory.getIncludeCipherSuites();
    Assert.assertEquals(sslCiphers.length, 2);
    List<String> ciphersList = Lists.newArrayList(sslCiphers);
    ciphers.forEach(c -> {
        Assert.assertTrue(c.equals(cipherA) || c.equals(cipherB));
        log.debug("ssl factory cipher: {}", c);
    });
}