Examples with SubscriptionMode org.apache.pulsar.client.api.SubscriptionMode used on opensource projects

Example 1 with SubscriptionMode

use of org.apache.pulsar.client.api.SubscriptionMode in project cdc-apache-cassandra by datastax.

the class PulsarCassandraSourceTests method testSchema.

// docker exec -it pulsar cat /pulsar/logs/functions/public/default/cassandra-source-ks1-table3/cassandra-source-ks1-table3-0.log
public void testSchema(String ksName, Class<? extends Converter> keyConverter, Class<? extends Converter> valueConverter) throws InterruptedException, IOException {
    try {
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS " + ksName + " WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};");
            cqlSession.execute("CREATE TYPE IF NOT EXISTS " + ksName + ".zudt (" + "ztext text, zascii ascii, zboolean boolean, zblob blob, ztimestamp timestamp, ztime time, zdate date, zuuid uuid, ztimeuuid timeuuid, " + "ztinyint tinyint, zsmallint smallint, zint int, zbigint bigint, zvarint varint, zdecimal decimal, zduration duration, zdouble double, " + "zfloat float, zinet4 inet, zinet6 inet, zlist frozen<list<text>>, zset frozen<set<int>>, zmap frozen<map<text, double>>" + ");");
            UserDefinedType zudt = cqlSession.getMetadata().getKeyspace(ksName).flatMap(ks -> ks.getUserDefinedType("zudt")).orElseThrow(() -> new IllegalArgumentException("Missing UDT zudt definition"));
            UdtValue zudtValue = zudt.newValue(dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("duration").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue, dataSpecMap.get("list").cqlValue, dataSpecMap.get("set").cqlValue, dataSpecMap.get("map").cqlValue);
            cqlSession.execute("CREATE TABLE IF NOT EXISTS " + ksName + ".table3 (" + "xtext text, xascii ascii, xboolean boolean, xblob blob, xtimestamp timestamp, xtime time, xdate date, xuuid uuid, xtimeuuid timeuuid, xtinyint tinyint, xsmallint smallint, xint int, xbigint bigint, xvarint varint, xdecimal decimal, xdouble double, xfloat float, xinet4 inet, xinet6 inet, " + "ytext text, yascii ascii, yboolean boolean, yblob blob, ytimestamp timestamp, ytime time, ydate date, yuuid uuid, ytimeuuid timeuuid, ytinyint tinyint, ysmallint smallint, yint int, ybigint bigint, yvarint varint, ydecimal decimal, ydouble double, yfloat float, yinet4 inet, yinet6 inet, yduration duration, yudt zudt, ylist list<text>, yset set<int>, ymap map<text, double>, ylistofmap list<frozen<map<text,double>>>, ysetofudt set<frozen<zudt>>," + "primary key (xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6)) " + "WITH CLUSTERING ORDER BY (xascii ASC, xboolean DESC, xblob ASC, xtimestamp DESC, xtime DESC, xdate ASC, xuuid DESC, xtimeuuid ASC, xtinyint DESC, xsmallint ASC, xint DESC, xbigint ASC, xvarint DESC, xdecimal ASC, xdouble DESC, xfloat ASC, xinet4 ASC, xinet6 DESC) AND cdc=true");
            cqlSession.execute("INSERT INTO " + ksName + ".table3 (" + "xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6, " + "ytext, yascii, yboolean, yblob, ytimestamp, ytime, ydate, yuuid, ytimeuuid, ytinyint, ysmallint, yint, ybigint, yvarint, ydecimal, ydouble, yfloat, yinet4, yinet6, yduration, yudt, ylist, yset, ymap, ylistofmap, ysetofudt" + ") VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?, ?,?,?,?,?)", dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue, dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue, dataSpecMap.get("duration").cqlValue, zudtValue, dataSpecMap.get("list").cqlValue, dataSpecMap.get("set").cqlValue, dataSpecMap.get("map").cqlValue, dataSpecMap.get("listofmap").cqlValue, ImmutableSet.of(zudtValue, zudtValue));
        }
        deployConnector(ksName, "table3", keyConverter, valueConverter);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build()) {
            try (Consumer<GenericRecord> consumer = pulsarClient.newConsumer(org.apache.pulsar.client.api.Schema.AUTO_CONSUME()).topic(String.format(Locale.ROOT, "data-%s.table3", ksName)).subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
                int mutationTable3Count = 0;
                Message<GenericRecord> msg;
                while ((msg = consumer.receive(120, TimeUnit.SECONDS)) != null && mutationTable3Count < 1) {
                    GenericObject genericObject = msg.getValue();
                    mutationTable3Count++;
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    // check primary key fields
                    Map<String, Object> keyMap = genericRecordToMap(key);
                    for (Field field : key.getFields()) {
                        assertField(field.getName(), keyMap.get(field.getName()));
                    }
                    // check regular columns.
                    Map<String, Object> valueMap = genericRecordToMap(value);
                    for (Field field : value.getFields()) {
                        assertField(field.getName(), valueMap.get(field.getName()));
                    }
                    consumer.acknowledge(msg);
                }
                assertEquals(1, mutationTable3Count);
            }
        }
    } finally {
        dumpFunctionLogs("cassandra-source-" + ksName + "-table3");
        undeployConnector(ksName, "table3");
    }
}

Example 2 with SubscriptionMode

use of org.apache.pulsar.client.api.SubscriptionMode in project cdc-apache-cassandra by datastax.

the class PulsarCassandraSourceTests method testCompoundPk.

// docker exec -it pulsar cat /pulsar/logs/functions/public/default/cassandra-source-ks1-table2/cassandra-source-ks1-table2-0.log
public void testCompoundPk(String ksName, Class<? extends Converter> keyConverter, Class<? extends Converter> valueConverter) throws InterruptedException, IOException {
    try {
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS " + ksName + " WITH replication = {'class':'SimpleStrategy','replication_factor':'2'};");
            cqlSession.execute("CREATE TABLE IF NOT EXISTS " + ksName + ".table2 (a text, b int, c int, PRIMARY KEY(a,b)) WITH cdc=true");
            cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c) VALUES('1',1,1)");
            cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c) VALUES('2',1,1)");
            cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c) VALUES('3',1,1)");
        }
        deployConnector(ksName, "table2", keyConverter, valueConverter);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build()) {
            Map<String, Integer> mutationTable2 = new HashMap<>();
            try (Consumer<GenericRecord> consumer = pulsarClient.newConsumer(org.apache.pulsar.client.api.Schema.AUTO_CONSUME()).topic(String.format(Locale.ROOT, "data-%s.table2", ksName)).subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
                Message<GenericRecord> msg;
                while ((msg = consumer.receive(60, TimeUnit.SECONDS)) != null && mutationTable2.values().stream().mapToInt(i -> i).sum() < 4) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals((Integer) 0, mutationTable2.computeIfAbsent((String) key.getField("a"), k -> 0));
                    assertEquals(1, key.getField("b"));
                    assertEquals(1, value.getField("c"));
                    mutationTable2.compute((String) key.getField("a"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 1, mutationTable2.get("1"));
                assertEquals((Integer) 1, mutationTable2.get("2"));
                assertEquals((Integer) 1, mutationTable2.get("3"));
                // trigger a schema update
                try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
                    cqlSession.execute("CREATE TYPE " + ksName + ".type2 (a2 int, b2 boolean);");
                    cqlSession.execute("ALTER TABLE " + ksName + ".table2 ADD d type2");
                    cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c,d) VALUES('1',1,1,{a2:1,b2:true})");
                }
                while ((msg = consumer.receive(30, TimeUnit.SECONDS)) != null && mutationTable2.values().stream().mapToInt(i -> i).sum() < 5) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals("1", key.getField("a"));
                    assertEquals(1, key.getField("b"));
                    assertEquals(1, value.getField("c"));
                    GenericRecord udtGenericRecord = (GenericRecord) value.getField("d");
                    assertEquals(1, udtGenericRecord.getField("a2"));
                    assertEquals(true, udtGenericRecord.getField("b2"));
                    mutationTable2.compute((String) key.getField("a"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 2, mutationTable2.get("1"));
                assertEquals((Integer) 1, mutationTable2.get("2"));
                assertEquals((Integer) 1, mutationTable2.get("3"));
                // delete a row
                try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
                    cqlSession.execute("DELETE FROM " + ksName + ".table2 WHERE a = '1' AND b = 1");
                }
                while ((msg = consumer.receive(30, TimeUnit.SECONDS)) != null && mutationTable2.values().stream().mapToInt(i -> i).sum() < 6) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals("1", key.getField("a"));
                    assertEquals(1, key.getField("b"));
                    assertNull(value);
                    mutationTable2.compute((String) key.getField("a"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 3, mutationTable2.get("1"));
                assertEquals((Integer) 1, mutationTable2.get("2"));
                assertEquals((Integer) 1, mutationTable2.get("3"));
            }
        }
    } finally {
        dumpFunctionLogs("cassandra-source-" + ksName + "-table2");
        undeployConnector(ksName, "table2");
    }
}

Example 3 with SubscriptionMode

use of org.apache.pulsar.client.api.SubscriptionMode in project cdc-apache-cassandra by datastax.

the class PulsarCassandraSourceTests method testSinglePk.

// docker exec -it pulsar cat /pulsar/logs/functions/public/default/cassandra-source-ks1-table1/cassandra-source-ks1-table1-0.log
public void testSinglePk(String ksName, Class<? extends Converter> keyConverter, Class<? extends Converter> valueConverter) throws InterruptedException, IOException {
    try {
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS " + ksName + " WITH replication = {'class':'SimpleStrategy','replication_factor':'2'};");
            cqlSession.execute("CREATE TABLE IF NOT EXISTS " + ksName + ".table1 (id text PRIMARY KEY, a int) WITH cdc=true");
            cqlSession.execute("INSERT INTO " + ksName + ".table1 (id, a) VALUES('1',1)");
            cqlSession.execute("INSERT INTO " + ksName + ".table1 (id, a) VALUES('2',1)");
            cqlSession.execute("INSERT INTO " + ksName + ".table1 (id, a) VALUES('3',1)");
        }
        deployConnector(ksName, "table1", keyConverter, valueConverter);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build()) {
            Map<String, Integer> mutationTable1 = new HashMap<>();
            try (Consumer<GenericRecord> consumer = pulsarClient.newConsumer(org.apache.pulsar.client.api.Schema.AUTO_CONSUME()).topic(String.format(Locale.ROOT, "data-%s.table1", ksName)).subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
                Message<GenericRecord> msg;
                while ((msg = consumer.receive(90, TimeUnit.SECONDS)) != null && mutationTable1.values().stream().mapToInt(i -> i).sum() < 4) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals((Integer) 0, mutationTable1.computeIfAbsent((String) key.getField("id"), k -> 0));
                    assertEquals(1, value.getField("a"));
                    mutationTable1.compute((String) key.getField("id"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 1, mutationTable1.get("1"));
                assertEquals((Integer) 1, mutationTable1.get("2"));
                assertEquals((Integer) 1, mutationTable1.get("3"));
                // trigger a schema update
                try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
                    cqlSession.execute("ALTER TABLE " + ksName + ".table1 ADD b double");
                    cqlSession.execute("INSERT INTO " + ksName + ".table1 (id,a,b) VALUES('1',1,1.0)");
                }
                while ((msg = consumer.receive(90, TimeUnit.SECONDS)) != null && mutationTable1.values().stream().mapToInt(i -> i).sum() < 5) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals("1", key.getField("id"));
                    assertEquals(1, value.getField("a"));
                    assertEquals(1.0D, value.getField("b"));
                    mutationTable1.compute((String) key.getField("id"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 2, mutationTable1.get("1"));
                assertEquals((Integer) 1, mutationTable1.get("2"));
                assertEquals((Integer) 1, mutationTable1.get("3"));
                // delete a row
                try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
                    cqlSession.execute("DELETE FROM " + ksName + ".table1 WHERE id = '1'");
                }
                while ((msg = consumer.receive(60, TimeUnit.SECONDS)) != null && mutationTable1.values().stream().mapToInt(i -> i).sum() < 6) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals("1", key.getField("id"));
                    assertNull(value);
                    mutationTable1.compute((String) key.getField("id"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 3, mutationTable1.get("1"));
                assertEquals((Integer) 1, mutationTable1.get("2"));
                assertEquals((Integer) 1, mutationTable1.get("3"));
            }
        }
    } finally {
        dumpFunctionLogs("cassandra-source-" + ksName + "-table1");
        undeployConnector(ksName, "table1");
    }
}