Examples with InstanceState com.datastax.oss.common.sink.state.InstanceState used on opensource projects

Example 1 with InstanceState

use of com.datastax.oss.common.sink.state.InstanceState in project pulsar-sink by datastax.

the class SimpleEndToEndSimulacronIT method should_not_record_failure_offsets_for_mapping_errors_if_ignore_errors_all.

@Test
void should_not_record_failure_offsets_for_mapping_errors_if_ignore_errors_all() {
    SimulacronUtils.primeTables(simulacron, schema);
    Query good1 = makeQuery(42, "the answer", 153000987000L);
    simulacron.prime(when(good1).then(noRows()));
    Query good2 = makeQuery(43, "another answer", 153000987000L);
    simulacron.prime(when(good2).then(noRows()));
    Map<String, Object> connectorPropertiesIgnoreErrors = new ImmutableMap.Builder<String, Object>().putAll(connectorProperties).put("ignoreErrors", "All").build();
    taskConfigs.add(connectorPropertiesIgnoreErrors);
    Record<GenericRecord> record1 = makeRecord(0, "42", "the answer", 153000987L, 1234);
    Record<GenericRecord> record2 = makeRecord(0, "bad key", "will fail", 153000987L, 1235);
    Record<GenericRecord> record3 = makeRecord(0, "43", "another answer", 153000987L, 1236);
    runTaskWithRecords(record1, record2, record3);
    assertThat(logs.getAllMessagesAsString()).contains("Error decoding/mapping Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={field1=will fail}}").contains("Could not parse 'bad key'");
    InstanceState instanceState = task.getInstanceState();
    assertThat(instanceState.getFailedRecordCounter("mytopic", "ks1.table1")).isEqualTo(1);
    assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1")).isEqualTo(2);
}

Example 2 with InstanceState

use of com.datastax.oss.common.sink.state.InstanceState in project pulsar-sink by datastax.

the class ConnectorSettingsCCMIT method should_load_settings_from_dse_reference_conf.

/**
 * Test for KAF-135
 */
@Test
void should_load_settings_from_dse_reference_conf() {
    // given (connector mapping need to be defined)
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, doublecol=value.double"));
    initConnectorAndTask();
    // when
    InstanceState instanceState = task.getInstanceState();
    // then setting from dse-reference.conf should be defined
    assertThat(instanceState.getSession().getContext().getConfig().getDefaultProfile().getInt(DseDriverOption.CONTINUOUS_PAGING_PAGE_SIZE)).isGreaterThan(0);
}

Example 3 with InstanceState

use of com.datastax.oss.common.sink.state.InstanceState in project pulsar-sink by datastax.

the class SimpleEndToEndSimulacronIT method failure_offsets.

@Test
void failure_offsets() {
    SimulacronUtils.primeTables(simulacron, schema);
    Query good1 = makeQuery(42, "the answer", 153000987000L);
    simulacron.prime(when(good1).then(noRows()));
    Query bad1 = makeQuery(32, "fail", 153000987000L);
    simulacron.prime(when(bad1).then(serverError("bad thing")).delay(500, TimeUnit.MILLISECONDS));
    Query good2 = makeQuery(22, "success", 153000987000L);
    simulacron.prime(when(good2).then(noRows()));
    Query bad2 = makeQuery(12, "fail2", 153000987000L);
    simulacron.prime(when(bad2).then(serverError("bad thing")));
    taskConfigs.add(connectorProperties);
    Record<GenericRecord> record1 = makeRecord(0, "42", "the answer", 153000987L, 1234);
    Record<GenericRecord> record2 = makeRecord(0, "32", "fail", 153000987L, 1235);
    Record<GenericRecord> record3 = makeRecord(0, "22", "success", 153000987L, 1236);
    Record<GenericRecord> record4 = makeRecord(0, "12", "fail2", 153000987L, 1237);
    // Make a bad record in a different partition.
    Record<GenericRecord> record5 = makeRecord(1, "bad key", "fail3", 153000987L, 1238);
    // bad record in the wrong topic. THis is probably not realistic but allows us to test the outer
    // try-catch block in mapAndQueueRecord().
    Record<GenericRecord> record6 = new PulsarRecordImpl("persistent://tenant/namespace/wrongtopic", "", new GenericRecordImpl(), recordType);
    runTaskWithRecords(record1, record2, record3, record4, record5, record6);
    assertThat(logs.getAllMessagesAsString()).contains("Error inserting/updating row for Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={field1=fail2}}").contains("Error decoding/mapping Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={field1=fail3}}").contains("Connector has no configuration for record topic 'wrongtopic'").contains("Could not parse 'bad key'").contains("statement: INSERT INTO ks1.table1(a,b) VALUES (:a,:b) USING TIMESTAMP :" + SinkUtil.TIMESTAMP_VARNAME);
    InstanceState instanceState = task.getInstanceState();
    assertThat(instanceState.getFailedRecordCounter("mytopic", "ks1.table1")).isEqualTo(3);
    assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1")).isEqualTo(4);
    assertThat(instanceState.getFailedWithUnknownTopicCounter()).isEqualTo(1);
}

Example 4 with InstanceState

use of com.datastax.oss.common.sink.state.InstanceState in project pulsar-sink by datastax.

the class SimpleEndToEndSimulacronIT method batch_requests.

// 
// @ParameterizedTest
// @CsvSource({"All", "Driver"})
// void should_not_record_failure_offsets_for_driver_errors_if_ignore_errors_all_or_driver(
// String ignoreErrors) {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// Query bad1 = makeQuery(32, "fail", 153000987000L);
// simulacron.prime(when(bad1).then(serverError("bad thing")).delay(500,
// TimeUnit.MILLISECONDS));
// 
// Query good2 = makeQuery(22, "success", 153000987000L);
// simulacron.prime(when(good2).then(noRows()));
// 
// Query bad2 = makeQuery(12, "fail2", 153000987000L);
// simulacron.prime(when(bad2).then(serverError("bad thing")));
// 
// Query bad3 = makeQuery(2, "fail3", 153000987000L);
// simulacron.prime(when(bad3).then(serverError("bad thing")));
// 
// Map<String, String> connectorPropertiesIgnoreErrors =
// new ImmutableMap.Builder<String, String>()
// .putAll(connectorProperties)
// .put("ignoreErrors", ignoreErrors)
// .build();
// 
// conn.start(connectorPropertiesIgnoreErrors);
// 
// Record<GenericRecord> record1 = makeRecord(42, "the answer", 153000987L, 1234);
// Record<GenericRecord> record2 = makeRecord(32, "fail", 153000987L, 1235);
// Record<GenericRecord> record3 = makeRecord(22, "success", 153000987L, 1236);
// Record<GenericRecord> record4 = makeRecord(12, "fail2", 153000987L, 1237);
// 
// // Make a bad record in a different partition.
// Record<GenericRecord> record5 = makeRecord(1, 2, "fail3", 153000987L, 1238);
// runTaskWithRecords(record1, record2, record3, record4, record5);
// 
// // Verify that we get an error offset for the first record that failed in partition 0 (1235)
// // even though its failure was discovered after 1237. Also, 1238 belongs to a different
// // partition, so it should be included.
// Map<TopicPartition, OffsetAndMetadata> currentOffsets = new HashMap<>();
// task.preCommit(currentOffsets);
// assertThat(currentOffsets).isEmpty();
// 
// assertThat(logs.getAllMessagesAsString())
// .contains("Error inserting/updating row for Pulsar record
// Record<GenericRecord>{kafkaOffset=1237")
// .contains("Error inserting/updating row for Pulsar record
// Record<GenericRecord>{kafkaOffset=1238")
// .contains(
// "statement: INSERT INTO ks1.table1(a,b) VALUES (:a,:b) USING TIMESTAMP
// SinkUtil.TIMESTAMP_VARNAME");
// InstanceState instanceState = task.getInstanceState();
// assertThat(instanceState.getFailedRecordCounter("mytopic", "ks1.table1").getCount())
// .isEqualTo(3);
// assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1").getCount()).isEqualTo(5);
// }
// 
// @Test
// void success_offset() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// Query good2 = makeQuery(22, "success", 153000987000L);
// simulacron.prime(when(good2).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> record1 = makeRecord(42, "the answer", 153000987L, 1234);
// Record<GenericRecord> record2 = makeRecord(22, "success", 153000987L, 1235);
// runTaskWithRecords(record1, record2);
// 
// Map<TopicPartition, OffsetAndMetadata> currentOffsets = new HashMap<>();
// task.preCommit(currentOffsets);
// assertThat(currentOffsets).isEmpty();
// 
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// assertThat(queryList.size()).isEqualTo(2);
// assertThat(queryList.get(0).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(1).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// }
// 
// /** Test for KAF-72 */
// @Test
// void should_record_counters_per_topic_ks_table() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1topic1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1topic1).then(noRows()));
// 
// Query good2topic1 = makeQuery(22, "success", 153000987000L);
// simulacron.prime(when(good2topic1).then(noRows()));
// 
// Query good1topic2 = makeTtlQuery(22, "success", 153000987000L, 22L);
// simulacron.prime(when(good1topic2).then(noRows()));
// 
// Query good2topic2 = makeTtlQuery(33, "success_2", 153000987000L, 33L);
// simulacron.prime(when(good2topic2).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> record1topic1 = makeRecord(42, "the answer", 153000987L, 1234);
// Record<GenericRecord> record2topic1 = makeRecord(22, "success", 153000987L, 1235);
// Record<GenericRecord> record1topic2 = makeTtlRecord(22, "success", 153000987L, 1235);
// Record<GenericRecord> record2topic2 = makeTtlRecord(33, "success_2", 153000987L, 1235);
// 
// runTaskWithRecords(record1topic1, record2topic1, record1topic2, record2topic2);
// 
// Map<TopicPartition, OffsetAndMetadata> currentOffsets = new HashMap<>();
// task.preCommit(currentOffsets);
// assertThat(currentOffsets).isEmpty();
// 
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// assertThat(queryList.size()).isEqualTo(4);
// assertThat(queryList.get(0).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(1).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(2).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(3).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// 
// InstanceState instanceState = task.getInstanceState();
// assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1").getCount()).isEqualTo(2);
// assertThat(instanceState.getRecordCounter("mytopic_with_ttl",
// "ks1.table1_with_ttl").getCount())
// .isEqualTo(2);
// }
// 
// @Test
// void consistency_level() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// Query good2 =
// new Query(
// "INSERT INTO ks1.table2(a,b) VALUES (:a,:b) USING TIMESTAMP
// :SinkUtil.TIMESTAMP_VARNAME",
// Collections.emptyList(),
// makeParams(22, "success", 153000987000L),
// PARAM_TYPES);
// simulacron.prime(when(good2).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> record1 = makeRecord(42, "the answer", 153000987L, 1234);
// 
// // Put the second record in "yourtopic", which has QUORUM CL.
// Record<GenericRecord> record2 =
// new Record<GenericRecord>(
// "yourtopic",
// 0,
// null,
// 22,
// null,
// "success",
// 1235L,
// 153000987L,
// TimestampType.CREATE_TIME);
// runTaskWithRecords(record1, record2);
// 
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// assertThat(queryList.size()).isEqualTo(2);
// 
// for (QueryLog queryLog : queryList) {
// if (queryLog.getQuery().contains("table1")) {
// assertThat(queryLog.getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// } else if (queryLog.getQuery().contains("table2")) {
// assertThat(queryLog.getConsistency()).isEqualTo(ConsistencyLevel.QUORUM);
// } else {
// fail("%s is not for table1 nor table2!", queryLog.toString());
// }
// }
// }
// 
// @Test
// void undefined_topic() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> goodRecord = makeRecord(42, "the answer", 153000987L, 1234);
// 
// Record<GenericRecord> badRecord = new Record<GenericRecord>("unknown", 0, null, 42L, null,
// 42, 1234L);
// runTaskWithRecords(goodRecord, badRecord);
// assertThat(logs.getAllMessagesAsString())
// .contains("Error decoding/mapping Kafka record Record<GenericRecord>{kafkaOffset=1234")
// .contains(
// "Connector has no configuration for record topic 'unknown'. Please update the
// configuration and restart.");
// 
// // Verify that the insert for good1 was issued.
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// byte[] secondParam = new byte[10];
// ((Execute) queryList.get(0).getFrame().message)
// .options
// .positionalValues
// .get(1)
// .get(secondParam);
// assertThat(new String(secondParam, StandardCharsets.UTF_8)).isEqualTo("the answer");
// }
// 
@Test
void batch_requests() {
    // Insert 5 records: 2 from mytopic, 3 from yourtopic. Verify that they batch properly and
    // with the correct CLs.
    // Even though we will not be executing simple statements in this test, we must specify
    // that we will so that Simulacron handles preparing our statement properly.
    SimulacronUtils.primeTables(simulacron, schema);
    Query good1 = makeQuery(42, "the answer", 153000987000L);
    simulacron.prime(when(good1).then(noRows()));
    Query good2 = new Query("INSERT INTO ks1.table2(a,b) VALUES (:a,:b) USING TIMESTAMP :" + SinkUtil.TIMESTAMP_VARNAME, Collections.emptyList(), makeParams(42, "topic2 success1", 153000987000L), PARAM_TYPES);
    simulacron.prime(when(good2).then(noRows()));
    taskConfigs.add(connectorProperties);
    Record<GenericRecord> goodRecord1 = makeRecord(42, "the answer", 153000987L, 1234);
    Record<GenericRecord> goodRecord2 = makeRecord(42, "the second answer", 153000987L, 1234);
    Record<GenericRecord> goodRecord3 = new PulsarRecordImpl("persistent://tenant/namespace/yourtopic", "42", new GenericRecordImpl().put("field1", "topic2 success1"), recordType, 153000987L);
    Record<GenericRecord> goodRecord4 = new PulsarRecordImpl("persistent://tenant/namespace/yourtopic", "42", new GenericRecordImpl().put("field1", "topic2 success2"), recordType, 153000987L);
    Record<GenericRecord> goodRecord5 = new PulsarRecordImpl("persistent://tenant/namespace/yourtopic", "42", new GenericRecordImpl().put("field1", "topic2 success3"), recordType, 153000987L);
    ;
    // The order of records shouldn't matter here, but we try to mix things up to emulate
    // a real workload.
    runTaskWithRecords(goodRecord1, goodRecord3, goodRecord2, goodRecord4, goodRecord5);
    // Verify that we issued two batch requests, one at LOCAL_ONE (for table1/mytopic) and
    // one at QUORUM (for table2/yourtopic). There's seem pretty gnarly unwrapping of request
    // info. We distinguish one batch from the other based on the number of statements in the
    // batch.
    List<QueryLog> queryList = simulacron.node(0).getLogs().getQueryLogs().stream().filter(q -> q.getType().equals("BATCH")).collect(Collectors.toList());
    Map<ConsistencyLevel, Integer> queryInfo = queryList.stream().map(queryLog -> (Batch) queryLog.getFrame().message).collect(Collectors.toMap(message -> ConsistencyLevel.fromCode(message.consistency), message -> message.values.size()));
    assertThat(queryInfo).containsOnly(entry(ConsistencyLevel.LOCAL_ONE, 2), entry(ConsistencyLevel.QUORUM, 3));
    InstanceState instanceState = task.getInstanceState();
    // verify that was one batch with 2 statements for mytopic
    verifyOneBatchWithNStatements(instanceState.getBatchSizeHistogramSummary("mytopic", "ks1.table1"), 2);
    // verify that was one batch with 3 statements for yourtopic
    verifyOneBatchWithNStatements(instanceState.getBatchSizeHistogramSummary("yourtopic", "ks1.table2"), 3);
    // verify batchSizeInBytes updates for mytopic
    verifyBatchSizeInBytesUpdate(instanceState.getBatchSizeInBytesHistogramSummary("mytopic", "ks1.table1"), 2, false);
    // verify batchSizeInBytes updates for yourtopic
    verifyBatchSizeInBytesUpdate(instanceState.getBatchSizeInBytesHistogramSummary("yourtopic", "ks1.table2"), 3, true);
}