Examples with PulsarRecordImpl com.datastax.oss.sink.pulsar.PulsarRecordImpl used on opensource projects

Example 61 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class NowFunctionCCMIT method should_insert_value_using_now_function_avro.

@Test
void should_insert_value_using_now_function_avro() {
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, loaded_at=now(), loaded_at2=now()"));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234567), recordType);
    runTaskWithRecords(record);
    // Verify that the record was inserted properly in the database.
    List<Row> results = session.execute("SELECT bigintcol, loaded_at, loaded_at2 FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    UUID loadedAt = row.get("loaded_at", TypeCodecs.TIMEUUID);
    UUID loadedAt2 = row.get("loaded_at2", TypeCodecs.TIMEUUID);
    assertThat(loadedAt).isNotEqualTo(loadedAt2);
}

Example 62 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class NowFunctionCCMIT method should_insert_value_using_now_function_json.

@Test
void should_insert_value_using_now_function_json() {
    // given
    taskConfigs.add(makeConnectorProperties("bigintcol=value.bigint, loaded_at=now()"));
    // when
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, new GenericRecordImpl().put("bigint", 1234567), recordTypeJson);
    runTaskWithRecords(record);
    // then
    List<Row> results = session.execute("SELECT bigintcol, loaded_at FROM types").all();
    assertThat(results.size()).isEqualTo(1);
    Row row = results.get(0);
    assertThat(row.getLong("bigintcol")).isEqualTo(1234567L);
    assertThat(row.get("loaded_at", TypeCodecs.TIMEUUID)).isLessThanOrEqualTo(Uuids.timeBased());
}

Example 63 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class HeadersCCMIT method should_fail_when_using_header_without_specific_field_in_a_mapping.

@Test
void should_fail_when_using_header_without_specific_field_in_a_mapping() {
    taskConfigs.add(makeConnectorProperties("bigintcol=key, udtcol=header"));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, 1234L, Schema.INT64).setProperty("myproperty", "value");
    assertThatThrownBy(() -> runTaskWithRecords(record)).isInstanceOf(ConfigException.class).hasMessageContaining("Invalid field name 'header': field names in mapping must be 'key', 'value', or start with 'key.' or 'value.' or 'header.', or be one of supported functions: '[now()]'");
}

Example 64 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class HeadersCCMIT method should_delete_when_header_values_are_null.

@Test
void should_delete_when_header_values_are_null() {
    // First insert a row...
    session.execute("INSERT INTO pk_value (my_pk, my_value) VALUES (1234567, true)");
    List<Row> results = session.execute("SELECT * FROM pk_value").all();
    assertThat(results.size()).isEqualTo(1);
    taskConfigs.add(makeConnectorProperties("my_pk=header.my_pk, my_value=header.my_value", "pk_value", null));
    PulsarRecordImpl record = new PulsarRecordImpl("persistent://tenant/namespace/mytopic", null, 1L, Schema.INT64).setProperty("my_pk", 1234567L + "").setProperty("my_value", null);
    runTaskWithRecords(record);
    // Verify that the record was deleted from the database.
    results = session.execute("SELECT * FROM pk_value").all();
    assertThat(results.size()).isEqualTo(0);
}

Example 65 with PulsarRecordImpl

use of com.datastax.oss.sink.pulsar.PulsarRecordImpl in project pulsar-sink by datastax.

the class SimpleEndToEndSimulacronIT method failure_offsets.

@Test
void failure_offsets() {
    SimulacronUtils.primeTables(simulacron, schema);
    Query good1 = makeQuery(42, "the answer", 153000987000L);
    simulacron.prime(when(good1).then(noRows()));
    Query bad1 = makeQuery(32, "fail", 153000987000L);
    simulacron.prime(when(bad1).then(serverError("bad thing")).delay(500, TimeUnit.MILLISECONDS));
    Query good2 = makeQuery(22, "success", 153000987000L);
    simulacron.prime(when(good2).then(noRows()));
    Query bad2 = makeQuery(12, "fail2", 153000987000L);
    simulacron.prime(when(bad2).then(serverError("bad thing")));
    taskConfigs.add(connectorProperties);
    Record<GenericRecord> record1 = makeRecord(0, "42", "the answer", 153000987L, 1234);
    Record<GenericRecord> record2 = makeRecord(0, "32", "fail", 153000987L, 1235);
    Record<GenericRecord> record3 = makeRecord(0, "22", "success", 153000987L, 1236);
    Record<GenericRecord> record4 = makeRecord(0, "12", "fail2", 153000987L, 1237);
    // Make a bad record in a different partition.
    Record<GenericRecord> record5 = makeRecord(1, "bad key", "fail3", 153000987L, 1238);
    // bad record in the wrong topic. THis is probably not realistic but allows us to test the outer
    // try-catch block in mapAndQueueRecord().
    Record<GenericRecord> record6 = new PulsarRecordImpl("persistent://tenant/namespace/wrongtopic", "", new GenericRecordImpl(), recordType);
    runTaskWithRecords(record1, record2, record3, record4, record5, record6);
    assertThat(logs.getAllMessagesAsString()).contains("Error inserting/updating row for Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={field1=fail2}}").contains("Error decoding/mapping Pulsar record PulsarSinkRecord{PulsarRecordImpl{topic=persistent://tenant/namespace/mytopic, value=GenericRecordImpl{values={field1=fail3}}").contains("Connector has no configuration for record topic 'wrongtopic'").contains("Could not parse 'bad key'").contains("statement: INSERT INTO ks1.table1(a,b) VALUES (:a,:b) USING TIMESTAMP :" + SinkUtil.TIMESTAMP_VARNAME);
    InstanceState instanceState = task.getInstanceState();
    assertThat(instanceState.getFailedRecordCounter("mytopic", "ks1.table1")).isEqualTo(3);
    assertThat(instanceState.getRecordCounter("mytopic", "ks1.table1")).isEqualTo(4);
    assertThat(instanceState.getFailedWithUnknownTopicCounter()).isEqualTo(1);
}