Examples with CqlSession com.datastax.oss.driver.api.core.CqlSession used on opensource projects

Example 56 with CqlSession

use of com.datastax.oss.driver.api.core.CqlSession in project cdc-apache-cassandra by datastax.

the class PulsarSingleNodeTests method testSchema.

@Test
@SuppressWarnings("unchecked")
public void testSchema() throws IOException, InterruptedException {
    final String pulsarServiceUrl = "pulsar://pulsar:" + pulsarContainer.BROKER_PORT;
    try (CassandraContainer<?> cassandraContainer1 = createCassandraContainer(1, pulsarServiceUrl, testNetwork)) {
        cassandraContainer1.start();
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS ks2 WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};");
            cqlSession.execute("CREATE TABLE IF NOT EXISTS ks2.table1 (" + "xtext text, xascii ascii, xboolean boolean, xblob blob, xtimestamp timestamp, xtime time, xdate date, xuuid uuid, xtimeuuid timeuuid, xtinyint tinyint, xsmallint smallint, xint int, xbigint bigint, xvarint varint, xdecimal decimal, xdouble double, xfloat float, xinet4 inet, xinet6 inet, " + "primary key (xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6)) " + "WITH CLUSTERING ORDER BY (xascii ASC, xboolean DESC, xblob ASC, xtimestamp DESC, xtime DESC, xdate ASC, xuuid DESC, xtimeuuid ASC, xtinyint DESC, xsmallint ASC, xint DESC, xbigint ASC, xvarint DESC, xdecimal ASC, xdouble DESC, xfloat ASC, xinet4 ASC, xinet6 DESC) AND cdc=true");
            cqlSession.execute("INSERT INTO ks2.table1 (xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue);
        }
        drain(cassandraContainer1);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build();
            Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic("events-ks2.table1").subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
            Message<GenericRecord> msg = consumer.receive(60, TimeUnit.SECONDS);
            Assert.assertNotNull("Expecting one message, check the agent log", msg);
            GenericRecord gr = msg.getValue();
            KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) gr.getNativeObject();
            GenericRecord key = kv.getKey();
            System.out.println("Consumer Record: topicName=" + msg.getTopicName() + " key=" + AgentTestUtil.genericRecordToString(key));
            Map<String, Object> keyMap = AgentTestUtil.genericRecordToMap(key);
            for (Field field : key.getFields()) {
                String vKey = field.getName().substring(1);
                Assert.assertTrue("Unknown field " + vKey, dataSpecMap.containsKey(vKey));
                if (keyMap.get(field.getName()) instanceof GenericRecord) {
                    AgentTestUtil.assertGenericRecords(vKey, (GenericRecord) keyMap.get(field.getName()), dataSpecMap);
                } else {
                    Assert.assertEquals("Wrong value for PK field " + field.getName(), dataSpecMap.get(vKey).avroValue, keyMap.get(field.getName()));
                }
            }
            consumer.acknowledgeAsync(msg);
        }
    }
}

Example 57 with CqlSession

use of com.datastax.oss.driver.api.core.CqlSession in project cdc-apache-cassandra by datastax.

the class PulsarSingleNodeTests method testStaticColumn.

@Test
@SuppressWarnings("unchecked")
public void testStaticColumn() throws IOException, InterruptedException {
    String pulsarServiceUrl = "pulsar://pulsar:" + pulsarContainer.BROKER_PORT;
    try (CassandraContainer<?> cassandraContainer1 = createCassandraContainer(1, pulsarServiceUrl, testNetwork)) {
        cassandraContainer1.start();
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS ks3 WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};");
            cqlSession.execute("CREATE TABLE IF NOT EXISTS ks3.table1 (a text, b text, c text, d text static, PRIMARY KEY ((a), b)) with cdc=true;");
            cqlSession.execute("INSERT INTO ks3.table1 (a,b,c,d) VALUES ('a','b','c','d1');");
            cqlSession.execute("INSERT INTO ks3.table1 (a,d) VALUES ('a','d2');");
            cqlSession.execute("DELETE FROM ks3.table1 WHERE a = 'a'");
        }
        drain(cassandraContainer1);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build();
            Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic("events-ks3.table1").subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
            Message<GenericRecord> msg = consumer.receive(120, TimeUnit.SECONDS);
            Assert.assertNotNull("Expecting one message, check the agent log", msg);
            GenericRecord gr = msg.getValue();
            KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) gr.getNativeObject();
            GenericRecord key = kv.getKey();
            Assert.assertEquals("a", key.getField("a"));
            Assert.assertEquals("b", key.getField("b"));
            consumer.acknowledgeAsync(msg);
            msg = consumer.receive(90, TimeUnit.SECONDS);
            Assert.assertNotNull("Expecting one message, check the agent log", msg);
            GenericRecord gr2 = msg.getValue();
            KeyValue<GenericRecord, GenericRecord> kv2 = (KeyValue<GenericRecord, GenericRecord>) gr2.getNativeObject();
            GenericRecord key2 = kv2.getKey();
            Assert.assertEquals("a", key2.getField("a"));
            Assert.assertEquals(null, key2.getField("b"));
            consumer.acknowledgeAsync(msg);
            msg = consumer.receive(90, TimeUnit.SECONDS);
            Assert.assertNotNull("Expecting one message, check the agent log", msg);
            GenericRecord gr3 = msg.getValue();
            KeyValue<GenericRecord, GenericRecord> kv3 = (KeyValue<GenericRecord, GenericRecord>) gr3.getNativeObject();
            GenericRecord key3 = kv3.getKey();
            Assert.assertEquals("a", key3.getField("a"));
            Assert.assertEquals(null, key3.getField("b"));
            consumer.acknowledgeAsync(msg);
        }
    }
}

Example 58 with CqlSession

use of com.datastax.oss.driver.api.core.CqlSession in project cdc-apache-cassandra by datastax.

the class PulsarSingleNodeTests method testPartitionedTopic.

@Test
@SuppressWarnings("unchecked")
public void testPartitionedTopic() throws IOException, InterruptedException {
    final String pulsarServiceUrl = "pulsar://pulsar:" + pulsarContainer.BROKER_PORT;
    Container.ExecResult result = pulsarContainer.execInContainer("/pulsar/bin/pulsar-admin", "topics", "create-partitioned-topic", "persistent://public/default/events-pt.table1", "--partitions", "3");
    assertEquals(0, result.getExitCode(), "create-partitioned-topic failed:" + result.getStdout());
    int numMutation = 100;
    try (CassandraContainer<?> cassandraContainer1 = createCassandraContainer(1, pulsarServiceUrl, testNetwork)) {
        cassandraContainer1.start();
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS pt WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};");
            cqlSession.execute("CREATE TABLE IF NOT EXISTS pt.table1 (a int, b blob, PRIMARY KEY (a)) with cdc=true;");
            for (int i = 0; i < 100; i++) cqlSession.execute("INSERT INTO pt.table1 (a,b) VALUES (?, ?);", i, AgentTestUtil.randomizeBuffer(i));
        }
        drain(cassandraContainer1);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build();
            Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic("events-pt.table1").subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
            Message msg;
            int numMessage = 0;
            while ((msg = consumer.receive(60, TimeUnit.SECONDS)) != null && numMessage < numMutation) {
                numMessage++;
                consumer.acknowledge(msg);
            }
            assertEquals(numMutation, numMessage);
        }
    }
}

Example 59 with CqlSession

use of com.datastax.oss.driver.api.core.CqlSession in project cdc-apache-cassandra by datastax.

the class PulsarCassandraSourceTests method testCompoundPk.

// docker exec -it pulsar cat /pulsar/logs/functions/public/default/cassandra-source-ks1-table2/cassandra-source-ks1-table2-0.log
public void testCompoundPk(String ksName, Class<? extends Converter> keyConverter, Class<? extends Converter> valueConverter) throws InterruptedException, IOException {
    try {
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS " + ksName + " WITH replication = {'class':'SimpleStrategy','replication_factor':'2'};");
            cqlSession.execute("CREATE TABLE IF NOT EXISTS " + ksName + ".table2 (a text, b int, c int, PRIMARY KEY(a,b)) WITH cdc=true");
            cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c) VALUES('1',1,1)");
            cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c) VALUES('2',1,1)");
            cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c) VALUES('3',1,1)");
        }
        deployConnector(ksName, "table2", keyConverter, valueConverter);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build()) {
            Map<String, Integer> mutationTable2 = new HashMap<>();
            try (Consumer<GenericRecord> consumer = pulsarClient.newConsumer(org.apache.pulsar.client.api.Schema.AUTO_CONSUME()).topic(String.format(Locale.ROOT, "data-%s.table2", ksName)).subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
                Message<GenericRecord> msg;
                while ((msg = consumer.receive(60, TimeUnit.SECONDS)) != null && mutationTable2.values().stream().mapToInt(i -> i).sum() < 4) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals((Integer) 0, mutationTable2.computeIfAbsent((String) key.getField("a"), k -> 0));
                    assertEquals(1, key.getField("b"));
                    assertEquals(1, value.getField("c"));
                    mutationTable2.compute((String) key.getField("a"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 1, mutationTable2.get("1"));
                assertEquals((Integer) 1, mutationTable2.get("2"));
                assertEquals((Integer) 1, mutationTable2.get("3"));
                // trigger a schema update
                try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
                    cqlSession.execute("CREATE TYPE " + ksName + ".type2 (a2 int, b2 boolean);");
                    cqlSession.execute("ALTER TABLE " + ksName + ".table2 ADD d type2");
                    cqlSession.execute("INSERT INTO " + ksName + ".table2 (a,b,c,d) VALUES('1',1,1,{a2:1,b2:true})");
                }
                while ((msg = consumer.receive(30, TimeUnit.SECONDS)) != null && mutationTable2.values().stream().mapToInt(i -> i).sum() < 5) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals("1", key.getField("a"));
                    assertEquals(1, key.getField("b"));
                    assertEquals(1, value.getField("c"));
                    GenericRecord udtGenericRecord = (GenericRecord) value.getField("d");
                    assertEquals(1, udtGenericRecord.getField("a2"));
                    assertEquals(true, udtGenericRecord.getField("b2"));
                    mutationTable2.compute((String) key.getField("a"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 2, mutationTable2.get("1"));
                assertEquals((Integer) 1, mutationTable2.get("2"));
                assertEquals((Integer) 1, mutationTable2.get("3"));
                // delete a row
                try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
                    cqlSession.execute("DELETE FROM " + ksName + ".table2 WHERE a = '1' AND b = 1");
                }
                while ((msg = consumer.receive(30, TimeUnit.SECONDS)) != null && mutationTable2.values().stream().mapToInt(i -> i).sum() < 6) {
                    GenericObject genericObject = msg.getValue();
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    assertEquals("1", key.getField("a"));
                    assertEquals(1, key.getField("b"));
                    assertNull(value);
                    mutationTable2.compute((String) key.getField("a"), (k, v) -> v + 1);
                    consumer.acknowledge(msg);
                }
                assertEquals((Integer) 3, mutationTable2.get("1"));
                assertEquals((Integer) 1, mutationTable2.get("2"));
                assertEquals((Integer) 1, mutationTable2.get("3"));
            }
        }
    } finally {
        dumpFunctionLogs("cassandra-source-" + ksName + "-table2");
        undeployConnector(ksName, "table2");
    }
}

Example 60 with CqlSession

use of com.datastax.oss.driver.api.core.CqlSession in project cdc-apache-cassandra by datastax.

the class CassandraClient method selectRowAsync.

/**
 * Try to read with downgraded consistency
 * @param pkValues primary key column
 * @param nodeId coordinator node id
 * @param consistencyLevels list of consistency to retry
 * @param preparedStatement CQL prepared statement
 * @param md5Digest mutation MD5 digest
 */
public CompletionStage<Tuple3<Row, ConsistencyLevel, UUID>> selectRowAsync(List<Object> pkValues, UUID nodeId, List<ConsistencyLevel> consistencyLevels, PreparedStatement preparedStatement, String md5Digest) {
    BoundStatement statement = preparedStatement.bind(pkValues.toArray(new Object[pkValues.size()]));
    // set the coordinator node
    Node node = null;
    if (nodeId != null) {
        node = cqlSession.getMetadata().getNodes().get(nodeId);
        if (node != null && node.getState().equals(NodeState.UP)) {
            statement = statement.setNode(node);
        }
    }
    log.debug("Fetching md5Digest={} coordinator={} query={} pk={} ", md5Digest, node, preparedStatement.getQuery(), pkValues);
    return executeWithDowngradeConsistencyRetry(cqlSession, statement, consistencyLevels).thenApply(tuple -> {
        log.debug("Read cl={} coordinator={} pk={}", tuple._2, tuple._1.getExecutionInfo().getCoordinator().getHostId(), pkValues);
        Row row = tuple._1.one();
        return new Tuple3<>(row, tuple._2, tuple._1.getExecutionInfo().getCoordinator().getHostId());
    }).whenComplete((tuple, error) -> {
        if (error != null) {
            log.warn("Failed to retrieve row: {}", error);
        }
    });
}