Examples with Record com.datastax.oss.dsbulk.connectors.api.Record used on opensource projects

Example 36 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class LogManagerTest method should_stop_when_max_result_mapping_errors_reached.

@Test
void should_stop_when_max_result_mapping_errors_reached() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(2), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Flux<Record> stmts = Flux.just(rowRecord1, rowRecord2, rowRecord3);
    try {
        stmts.transform(logManager.newUnmappableRecordsHandler()).blockLast();
        fail("Expecting TooManyErrorsException to be thrown");
    } catch (TooManyErrorsException e) {
        assertThat(e).hasMessage("Too many errors, the maximum allowed is 2.");
        assertThat(((AbsoluteErrorThreshold) e.getThreshold()).getMaxErrors()).isEqualTo(2);
    }
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("mapping-errors.log");
    assertThat(errors.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).doesNotContain("Source: ").doesNotContain("Resource: ").doesNotContain("Position: ").contains("SELECT 1").containsOnlyOnce("c1: 1").containsOnlyOnce("java.lang.RuntimeException: error 1").contains("SELECT 2").containsOnlyOnce("c1: 2").containsOnlyOnce("java.lang.RuntimeException: error 2").doesNotContain("c3: 3");
}

Example 37 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class LogManagerTest method should_handle_unmappable_statements_without_source.

@Test
void should_handle_unmappable_statements_without_source() throws Exception {
    Path outputDir = Files.createTempDirectory("test");
    LogManager logManager = new LogManager(session, outputDir, ErrorThreshold.forAbsoluteValue(1), ErrorThreshold.forAbsoluteValue(0), true, statementFormatter, EXTENDED, rowFormatter);
    logManager.init();
    Record record = DefaultRecord.indexed(null, resource1, 1, "foo", " bar");
    UnmappableStatement stmt = new UnmappableStatement(record, new RuntimeException("error 1"));
    Flux<BatchableStatement<?>> stmts = Flux.just(stmt);
    stmts.transform(logManager.newUnmappableStatementsHandler()).blockLast();
    logManager.close();
    Path errors = logManager.getOperationDirectory().resolve("mapping-errors.log");
    Path positions = logManager.getOperationDirectory().resolve("positions.txt");
    assertThat(errors.toFile()).exists();
    assertThat(positions.toFile()).exists();
    assertThat(FileUtils.listAllFilesInDirectory(logManager.getOperationDirectory())).containsOnly(errors, positions);
    List<String> lines = Files.readAllLines(errors, UTF_8);
    String content = String.join("\n", lines);
    assertThat(content).doesNotContain("Source: ").contains("Resource: " + resource1).contains("Position: 1").contains("java.lang.RuntimeException: error 1");
}

Example 38 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class DefaultReadResultMapperTest method should_map_result_to_mapped_record_when_mapping_succeeds.

@ParameterizedTest
@ValueSource(booleans = { true, false })
void should_map_result_to_mapped_record_when_mapping_succeeds(boolean retainRecordSources) {
    DefaultReadResultMapper mapper = new DefaultReadResultMapper(mapping, recordMetadata, RESOURCE, retainRecordSources);
    Record record = mapper.map(result);
    Assertions.assertThat(record.fields()).containsOnly(F0, F1, F2);
    assertThat(record.getFieldValue(F0)).isEqualTo(42);
    assertThat(record.getFieldValue(F1)).isEqualTo("foo");
    assertThat(record.getFieldValue(F2)).isEqualTo("bar");
    if (retainRecordSources) {
        assertThat(record.getSource()).isSameAs(result);
    } else {
        assertThat(record.getSource()).isNull();
    }
    assertThat(record.getResource()).isEqualTo(URI.create("cql://ks1/table1"));
}

Example 39 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class JsonConnectorTest method should_read_from_stdin_with_special_encoding.

@Test
void should_read_from_stdin_with_special_encoding() throws Exception {
    InputStream stdin = System.in;
    try {
        String line = "{ \"fóô\" : \"bàr\", \"qïx\" : null }\n";
        InputStream is = new ByteArrayInputStream(line.getBytes(ISO_8859_1));
        System.setIn(is);
        JsonConnector connector = new JsonConnector();
        Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.json", "encoding", "ISO-8859-1");
        connector.configure(settings, true, true);
        connector.init();
        assertThat(connector.readConcurrency()).isOne();
        assertThat(ReflectionUtils.invokeMethod("isDataSizeSamplingAvailable", connector, Boolean.TYPE)).isFalse();
        List<Record> actual = Flux.merge(connector.read()).collectList().block();
        assertThat(actual).hasSize(1);
        assertThat(actual.get(0).getSource()).isEqualTo(objectMapper.readTree(line));
        assertThat(actual.get(0).getResource()).isEqualTo(URI.create("std:/"));
        assertThat(actual.get(0).getPosition()).isEqualTo(1L);
        assertThat(actual.get(0).getFieldValue(new DefaultMappedField("fóô"))).isEqualTo(factory.textNode("bàr"));
        assertThat(actual.get(0).getFieldValue(new DefaultMappedField("qïx"))).isEqualTo(factory.nullNode());
        connector.close();
    } finally {
        System.setIn(stdin);
    }
}

Example 40 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class JsonConnectorTest method should_write_multiple_files.

@Test
void should_write_multiple_files() throws Exception {
    JsonConnector connector = new JsonConnector();
    Path out = Files.createTempDirectory("test");
    try {
        int maxConcurrentFiles = 4;
        Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.json", "url", quoteJson(out), "escape", "\"\\\"\"", "maxConcurrentFiles", maxConcurrentFiles);
        connector.configure(settings, false, true);
        connector.init();
        assertThat(connector.writeConcurrency()).isEqualTo(maxConcurrentFiles);
        // repeat the records 1000 times to fully exercise multiple file writing
        Scheduler scheduler = Schedulers.newParallel("workflow");
        Function<Publisher<Record>, Publisher<Record>> write = connector.write();
        Flux.range(0, 1000).flatMap(i -> Flux.fromIterable(createRecords(false, resource)).transform(write).subscribeOn(scheduler), maxConcurrentFiles).blockLast();
        connector.close();
        List<String> actual = FileUtils.readAllLinesInDirectoryAsStream(out).sorted().distinct().collect(Collectors.toList());
        assertThat(actual).containsOnly("{\"Year\":1997,\"Make\":\"Ford\",\"Model\":\"E350\",\"Description\":\"ac, abs, moon\",\"Price\":3000.0}", "{\"Year\":1999,\"Make\":\"Chevy\",\"Model\":\"Venture \\\"Extended Edition\\\"\",\"Description\":null,\"Price\":4900.0}", "{\"Year\":1996,\"Make\":\"Jeep\",\"Model\":\"Grand Cherokee\",\"Description\":\"MUST SELL!\\nair, moon roof, loaded\",\"Price\":4799.0}", "{\"Year\":1999,\"Make\":\"Chevy\",\"Model\":\"Venture \\\"Extended Edition, Very Large\\\"\",\"Description\":null,\"Price\":5000.0}", "{\"Year\":null,\"Make\":null,\"Model\":\"Venture \\\"Extended Edition\\\"\",\"Description\":null,\"Price\":4900.0}");
    } finally {
        deleteDirectory(out);
    }
}