Examples with GenericRecordImpl com.datastax.oss.sink.pulsar.GenericRecordImpl used on opensource projects

Example 46 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class SimpleEndToEndSimulacronIT method failure_offsets.

@Test
void failure_offsets() {
    SimulacronUtils.primeTables(simulacron, schema);
    Query good1 = makeQuery(42, "the answer", 153000987000L);
    simulacron.prime(when(good1).then(noRows()));
    Query bad1 = makeQuery(32, "fail", 153000987000L);
    simulacron.prime(when(bad1).then(serverError("bad thing")).delay(500, TimeUnit.MILLISECONDS));
    Query good2 = makeQuery(22, "success", 153000987000L);
    simulacron.prime(when(good2).then(noRows()));
    Query bad2 = makeQuery(12, "fail2", 153000987000L);
    simulacron.prime(when(bad2).then(serverError("bad thing")));
    taskConfigs.add(connectorProperties);
    Record<GenericRecord> record1 = makeRecord(0, "42", "the answer", 153000987L, 1234);
    Record<GenericRecord> record2 = makeRecord(0, "32", "fail", 153000987L, 1235);
    Record<GenericRecord> record3 = makeRecord(0, "22", "success", 153000987L, 1236);
    Record<GenericRecord> record4 = makeRecord(0, "12", "fail2", 153000987L, 1237);
    // Make a bad record in a different partition.
    Record<GenericRecord> record5 = makeRecord(1, "bad key", "fail3", 153000987L, 1238);
    // bad record in the wrong topic. THis is probably not realistic but allows us to test the outer
    // try-catch block in mapAndQueueRecord().
    Record<GenericRecord> record6 = new PulsarRecordImpl("persistent://tenant/namespace/wrongtopic", "", new GenericRecordImpl(), recordType);
    runTaskWithRecords(record1, record2, record3, record4, record5, record6);
    assertThat(logs.getAllMessagesAsString()).contains("Error inserting/updating row for Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={field1=fail2}}").contains("Error decoding/mapping Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={field1=fail3}}").contains("Connector has no configuration for record topic 'wrongtopic'").contains("Could not parse 'bad key'").contains("statement: INSERT INTO ks1.table1(a,b) VALUES (:a,:b) USING TIMESTAMP :" + SinkUtil.TIMESTAMP_VARNAME);
    InstanceState instanceState = task.getInstanceState();
    assertThat(instanceState.getFailedRecordCounter("mytopic", "ks1.table1")).isEqualTo(3);
    assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1")).isEqualTo(4);
    assertThat(instanceState.getFailedWithUnknownTopicCounter()).isEqualTo(1);
}

Example 47 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class SimpleEndToEndSimulacronIT method batch_requests.

// 
// @ParameterizedTest
// @CsvSource({"All", "Driver"})
// void should_not_record_failure_offsets_for_driver_errors_if_ignore_errors_all_or_driver(
// String ignoreErrors) {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// Query bad1 = makeQuery(32, "fail", 153000987000L);
// simulacron.prime(when(bad1).then(serverError("bad thing")).delay(500,
// TimeUnit.MILLISECONDS));
// 
// Query good2 = makeQuery(22, "success", 153000987000L);
// simulacron.prime(when(good2).then(noRows()));
// 
// Query bad2 = makeQuery(12, "fail2", 153000987000L);
// simulacron.prime(when(bad2).then(serverError("bad thing")));
// 
// Query bad3 = makeQuery(2, "fail3", 153000987000L);
// simulacron.prime(when(bad3).then(serverError("bad thing")));
// 
// Map<String, String> connectorPropertiesIgnoreErrors =
// new ImmutableMap.Builder<String, String>()
// .putAll(connectorProperties)
// .put("ignoreErrors", ignoreErrors)
// .build();
// 
// conn.start(connectorPropertiesIgnoreErrors);
// 
// Record<GenericRecord> record1 = makeRecord(42, "the answer", 153000987L, 1234);
// Record<GenericRecord> record2 = makeRecord(32, "fail", 153000987L, 1235);
// Record<GenericRecord> record3 = makeRecord(22, "success", 153000987L, 1236);
// Record<GenericRecord> record4 = makeRecord(12, "fail2", 153000987L, 1237);
// 
// // Make a bad record in a different partition.
// Record<GenericRecord> record5 = makeRecord(1, 2, "fail3", 153000987L, 1238);
// runTaskWithRecords(record1, record2, record3, record4, record5);
// 
// // Verify that we get an error offset for the first record that failed in partition 0 (1235)
// // even though its failure was discovered after 1237. Also, 1238 belongs to a different
// // partition, so it should be included.
// Map<TopicPartition, OffsetAndMetadata> currentOffsets = new HashMap<>();
// task.preCommit(currentOffsets);
// assertThat(currentOffsets).isEmpty();
// 
// assertThat(logs.getAllMessagesAsString())
// .contains("Error inserting/updating row for Pulsar record
// Record<GenericRecord>{kafkaOffset=1237")
// .contains("Error inserting/updating row for Pulsar record
// Record<GenericRecord>{kafkaOffset=1238")
// .contains(
// "statement: INSERT INTO ks1.table1(a,b) VALUES (:a,:b) USING TIMESTAMP
// SinkUtil.TIMESTAMP_VARNAME");
// InstanceState instanceState = task.getInstanceState();
// assertThat(instanceState.getFailedRecordCounter("mytopic", "ks1.table1").getCount())
// .isEqualTo(3);
// assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1").getCount()).isEqualTo(5);
// }
// 
// @Test
// void success_offset() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// Query good2 = makeQuery(22, "success", 153000987000L);
// simulacron.prime(when(good2).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> record1 = makeRecord(42, "the answer", 153000987L, 1234);
// Record<GenericRecord> record2 = makeRecord(22, "success", 153000987L, 1235);
// runTaskWithRecords(record1, record2);
// 
// Map<TopicPartition, OffsetAndMetadata> currentOffsets = new HashMap<>();
// task.preCommit(currentOffsets);
// assertThat(currentOffsets).isEmpty();
// 
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// assertThat(queryList.size()).isEqualTo(2);
// assertThat(queryList.get(0).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(1).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// }
// 
// /** Test for KAF-72 */
// @Test
// void should_record_counters_per_topic_ks_table() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1topic1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1topic1).then(noRows()));
// 
// Query good2topic1 = makeQuery(22, "success", 153000987000L);
// simulacron.prime(when(good2topic1).then(noRows()));
// 
// Query good1topic2 = makeTtlQuery(22, "success", 153000987000L, 22L);
// simulacron.prime(when(good1topic2).then(noRows()));
// 
// Query good2topic2 = makeTtlQuery(33, "success_2", 153000987000L, 33L);
// simulacron.prime(when(good2topic2).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> record1topic1 = makeRecord(42, "the answer", 153000987L, 1234);
// Record<GenericRecord> record2topic1 = makeRecord(22, "success", 153000987L, 1235);
// Record<GenericRecord> record1topic2 = makeTtlRecord(22, "success", 153000987L, 1235);
// Record<GenericRecord> record2topic2 = makeTtlRecord(33, "success_2", 153000987L, 1235);
// 
// runTaskWithRecords(record1topic1, record2topic1, record1topic2, record2topic2);
// 
// Map<TopicPartition, OffsetAndMetadata> currentOffsets = new HashMap<>();
// task.preCommit(currentOffsets);
// assertThat(currentOffsets).isEmpty();
// 
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// assertThat(queryList.size()).isEqualTo(4);
// assertThat(queryList.get(0).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(1).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(2).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// assertThat(queryList.get(3).getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// 
// InstanceState instanceState = task.getInstanceState();
// assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1").getCount()).isEqualTo(2);
// assertThat(instanceState.getRecordCounter("mytopic_with_ttl",
// "ks1.table1_with_ttl").getCount())
// .isEqualTo(2);
// }
// 
// @Test
// void consistency_level() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// Query good2 =
// new Query(
// "INSERT INTO ks1.table2(a,b) VALUES (:a,:b) USING TIMESTAMP
// :SinkUtil.TIMESTAMP_VARNAME",
// Collections.emptyList(),
// makeParams(22, "success", 153000987000L),
// PARAM_TYPES);
// simulacron.prime(when(good2).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> record1 = makeRecord(42, "the answer", 153000987L, 1234);
// 
// // Put the second record in "yourtopic", which has QUORUM CL.
// Record<GenericRecord> record2 =
// new Record<GenericRecord>(
// "yourtopic",
// 0,
// null,
// 22,
// null,
// "success",
// 1235L,
// 153000987L,
// TimestampType.CREATE_TIME);
// runTaskWithRecords(record1, record2);
// 
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// assertThat(queryList.size()).isEqualTo(2);
// 
// for (QueryLog queryLog : queryList) {
// if (queryLog.getQuery().contains("table1")) {
// assertThat(queryLog.getConsistency()).isEqualTo(ConsistencyLevel.LOCAL_ONE);
// } else if (queryLog.getQuery().contains("table2")) {
// assertThat(queryLog.getConsistency()).isEqualTo(ConsistencyLevel.QUORUM);
// } else {
// fail("%s is not for table1 nor table2!", queryLog.toString());
// }
// }
// }
// 
// @Test
// void undefined_topic() {
// SimulacronUtils.primeTables(simulacron, schema);
// 
// Query good1 = makeQuery(42, "the answer", 153000987000L);
// simulacron.prime(when(good1).then(noRows()));
// 
// conn.start(connectorProperties);
// 
// Record<GenericRecord> goodRecord = makeRecord(42, "the answer", 153000987L, 1234);
// 
// Record<GenericRecord> badRecord = new Record<GenericRecord>("unknown", 0, null, 42L, null,
// 42, 1234L);
// runTaskWithRecords(goodRecord, badRecord);
// assertThat(logs.getAllMessagesAsString())
// .contains("Error decoding/mapping Kafka record Record<GenericRecord>{kafkaOffset=1234")
// .contains(
// "Connector has no configuration for record topic 'unknown'. Please update the
// configuration and restart.");
// 
// // Verify that the insert for good1 was issued.
// List<QueryLog> queryList =
// simulacron
// .node(0)
// .getLogs()
// .getQueryLogs()
// .stream()
// .filter(q -> q.getType().equals("EXECUTE"))
// .collect(Collectors.toList());
// byte[] secondParam = new byte[10];
// ((Execute) queryList.get(0).getFrame().message)
// .options
// .positionalValues
// .get(1)
// .get(secondParam);
// assertThat(new String(secondParam, StandardCharsets.UTF_8)).isEqualTo("the answer");
// }
// 
@Test
void batch_requests() {
    // Insert 5 records: 2 from mytopic, 3 from yourtopic. Verify that they batch properly and
    // with the correct CLs.
    // Even though we will not be executing simple statements in this test, we must specify
    // that we will so that Simulacron handles preparing our statement properly.
    SimulacronUtils.primeTables(simulacron, schema);
    Query good1 = makeQuery(42, "the answer", 153000987000L);
    simulacron.prime(when(good1).then(noRows()));
    Query good2 = new Query("INSERT INTO ks1.table2(a,b) VALUES (:a,:b) USING TIMESTAMP :" + SinkUtil.TIMESTAMP_VARNAME, Collections.emptyList(), makeParams(42, "topic2 success1", 153000987000L), PARAM_TYPES);
    simulacron.prime(when(good2).then(noRows()));
    taskConfigs.add(connectorProperties);
    Record<GenericRecord> goodRecord1 = makeRecord(42, "the answer", 153000987L, 1234);
    Record<GenericRecord> goodRecord2 = makeRecord(42, "the second answer", 153000987L, 1234);
    Record<GenericRecord> goodRecord3 = new PulsarRecordImpl("persistent://tenant/namespace/yourtopic", "42", new GenericRecordImpl().put("field1", "topic2 success1"), recordType, 153000987L);
    Record<GenericRecord> goodRecord4 = new PulsarRecordImpl("persistent://tenant/namespace/yourtopic", "42", new GenericRecordImpl().put("field1", "topic2 success2"), recordType, 153000987L);
    Record<GenericRecord> goodRecord5 = new PulsarRecordImpl("persistent://tenant/namespace/yourtopic", "42", new GenericRecordImpl().put("field1", "topic2 success3"), recordType, 153000987L);
    ;
    // The order of records shouldn't matter here, but we try to mix things up to emulate
    // a real workload.
    runTaskWithRecords(goodRecord1, goodRecord3, goodRecord2, goodRecord4, goodRecord5);
    // Verify that we issued two batch requests, one at LOCAL_ONE (for table1/mytopic) and
    // one at QUORUM (for table2/yourtopic). There's seem pretty gnarly unwrapping of request
    // info. We distinguish one batch from the other based on the number of statements in the
    // batch.
    List<QueryLog> queryList = simulacron.node(0).getLogs().getQueryLogs().stream().filter(q -> q.getType().equals("BATCH")).collect(Collectors.toList());
    Map<ConsistencyLevel, Integer> queryInfo = queryList.stream().map(queryLog -> (Batch) queryLog.getFrame().message).collect(Collectors.toMap(message -> ConsistencyLevel.fromCode(message.consistency), message -> message.values.size()));
    assertThat(queryInfo).containsOnly(entry(ConsistencyLevel.LOCAL_ONE, 2), entry(ConsistencyLevel.QUORUM, 3));
    InstanceState instanceState = task.getInstanceState();
    // verify that was one batch with 2 statements for mytopic
    verifyOneBatchWithNStatements(instanceState.getBatchSizeHistogramSummary("mytopic", "ks1.table1"), 2);
    // verify that was one batch with 3 statements for yourtopic
    verifyOneBatchWithNStatements(instanceState.getBatchSizeHistogramSummary("yourtopic", "ks1.table2"), 3);
    // verify batchSizeInBytes updates for mytopic
    verifyBatchSizeInBytesUpdate(instanceState.getBatchSizeInBytesHistogramSummary("mytopic", "ks1.table1"), 2, false);
    // verify batchSizeInBytes updates for yourtopic
    verifyBatchSizeInBytesUpdate(instanceState.getBatchSizeInBytesHistogramSummary("yourtopic", "ks1.table2"), 3, true);
}

Example 48 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class SimpleEndToEndSimulacronIT method fail_delete.

@Test
void fail_delete() {
    SimulacronUtils.primeTables(simulacron, schema);
    Query bad1 = new Query("DELETE FROM ks1.mycounter WHERE a = :a AND b = :b", Collections.emptyList(), ImmutableMap.<String, Object>builder().put("a", 37).put("b", "delete").build(), ImmutableMap.<String, String>builder().put("a", "int").put("b", "varchar").build());
    simulacron.prime(when(bad1).then(serverError("bad thing")));
    Map<String, Object> connProps = new HashMap<>();
    connProps.put("name", INSTANCE_NAME);
    connProps.put("contactPoints", hostname);
    connProps.put("port", port);
    connProps.put("loadBalancing.localDc", "dc1");
    connProps.put("topic.mytopic.ks1.mycounter.mapping", "a=value.bigint, b=value.text, c=value.int");
    taskConfigs.add(connProps);
    RecordSchemaBuilder builder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("MyBean");
    builder.field("bigint").type(SchemaType.INT64);
    builder.field("text").type(SchemaType.STRING);
    builder.field("int").type(SchemaType.INT32);
    Schema<GenericRecord> schema = org.apache.pulsar.client.api.Schema.generic(builder.build(SchemaType.AVRO));
    GenericRecord value = new GenericRecordImpl().put("bigint", 37L).put("text", "delete");
    Record<GenericRecord> record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, value, schema);
    runTaskWithRecords(record);
    // The log may need a little time to be updated with our error message.
    try {
        MILLISECONDS.sleep(500);
    } catch (InterruptedException e) {
    // swallow
    }
    assertThat(logs.getAllMessagesAsString()).contains("Error inserting/updating row for Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={bigint=37, text=delete}}").contains("statement: DELETE FROM ks1.mycounter WHERE a = :a AND b = :b");
}

Example 49 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class StructToUDTCodecTest method should_not_convert_from_invalid_external.

@Test
void should_not_convert_from_invalid_external() {
    RecordSchemaBuilder builder = SchemaBuilder.record("MyBean");
    builder.field("a1").type(SchemaType.INT32);
    Schema schemaOther = Schema.generic(builder.build(SchemaType.AVRO));
    Record<GenericRecord> other = new PulsarRecordImpl(null, null, new GenericRecordImpl().put("a1", 32), schema);
    RecordSchemaBuilder builder2 = SchemaBuilder.record("MyBean");
    builder2.field("a1").type(SchemaType.INT32);
    builder2.field("a2").type(SchemaType.INT32);
    Schema schemaOther2 = Schema.generic(builder2.build(SchemaType.AVRO));
    Record<GenericRecord> other2 = new PulsarRecordImpl(null, null, new GenericRecordImpl().put("a1", 32).put("a2", 40), schemaOther2);
    assertThat(udtCodec1).cannotConvertFromExternal(PulsarStruct.ofRecord(other, registry)).cannotConvertFromExternal(PulsarStruct.ofRecord(other2, registry));
}

Example 50 with GenericRecordImpl

use of com.datastax.oss.sink.pulsar.GenericRecordImpl in project pulsar-sink by datastax.

the class PlaintextAuthCCMIT method should_error_that_password_or_username_is_incorrect.

@ParameterizedTest(name = "[{index}] extras={0}")
@MethodSource("incorrectCredentialsProvider")
void should_error_that_password_or_username_is_incorrect(Map<String, String> extras) {
    taskConfigs.add(makeConnectorProperties(extras));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl(), recordType);
    assertThatThrownBy(() -> runTaskWithRecords(record)).isInstanceOf(AllNodesFailedException.class).hasMessageContaining("and/or password are incorrect");
}