Examples with Record com.datastax.oss.dsbulk.connectors.api.Record used on opensource projects

Example 11 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class CSVConnectorTest method should_write_to_stdout_with_special_encoding.

@Test
void should_write_to_stdout_with_special_encoding() throws Exception {
    PrintStream stdout = System.out;
    try {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        PrintStream out = new PrintStream(baos);
        System.setOut(out);
        CSVConnector connector = new CSVConnector();
        Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.csv", "header", false, "encoding", "ISO-8859-1");
        connector.configure(settings, false, true);
        connector.init();
        assertThat(connector.writeConcurrency()).isOne();
        assertThat(ReflectionUtils.invokeMethod("isDataSizeSamplingAvailable", connector, Boolean.TYPE)).isFalse();
        Flux.<Record>just(DefaultRecord.indexed("source", resource, IRRELEVANT_POSITION, "fóô", "bàr", "qïx")).transform(connector.write()).blockLast();
        connector.close();
        assertThat(new String(baos.toByteArray(), ISO_8859_1)).isEqualTo("fóô,bàr,qïx" + System.lineSeparator());
    } finally {
        System.setOut(stdout);
    }
}

Example 12 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class CSVConnectorTest method should_honor_ignoreLeadingWhitespacesInQuotes_and_ignoreTrailingWhitespacesInQuotes.

@Test
void should_honor_ignoreLeadingWhitespacesInQuotes_and_ignoreTrailingWhitespacesInQuotes() throws Exception {
    Path file = Files.createTempFile("test", ".csv");
    Files.write(file, Collections.singleton("\" foo \""));
    CSVConnector connector = new CSVConnector();
    Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.csv", "url", StringUtils.quoteJson(file), "ignoreLeadingWhitespacesInQuotes", false, "ignoreTrailingWhitespacesInQuotes", false, "header", false);
    connector.configure(settings, true, true);
    connector.init();
    List<Record> records = Flux.merge(connector.read()).collectList().block();
    assertThat(records).hasSize(1);
    assertThat(records.get(0).getFieldValue(new DefaultIndexedField(0))).isEqualTo(" foo ");
    connector.close();
}

Example 13 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class CSVConnectorTest method should_write_to_stdout_with_special_newline.

@Test
void should_write_to_stdout_with_special_newline() throws Exception {
    PrintStream stdout = System.out;
    try {
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        PrintStream out = new PrintStream(baos);
        System.setOut(out);
        CSVConnector connector = new CSVConnector();
        Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.csv", "header", false, "newline", "\"\\r\\n\"");
        connector.configure(settings, false, true);
        connector.init();
        assertThat(connector.writeConcurrency()).isOne();
        assertThat(ReflectionUtils.invokeMethod("isDataSizeSamplingAvailable", connector, Boolean.TYPE)).isFalse();
        Flux.<Record>just(DefaultRecord.indexed("source", resource, IRRELEVANT_POSITION, "abc", "de\nf", "ghk")).transform(connector.write()).blockLast();
        connector.close();
        assertThat(new String(baos.toByteArray(), UTF_8)).isEqualTo("abc,\"de\nf\",ghk\r\n");
    } finally {
        System.setOut(stdout);
    }
}

Example 14 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class CSVConnectorTest method should_honor_emptyValue_when_reading.

@ParameterizedTest
@MethodSource
void should_honor_emptyValue_when_reading(String quote, String emptyValue, String expected) throws Exception {
    Path file = Files.createTempFile("test", ".csv");
    Files.write(file, Collections.singleton(Strings.repeat(quote, 2)));
    CSVConnector connector = new CSVConnector();
    Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.csv", "url", StringUtils.quoteJson(file), "quote", StringUtils.quoteJson(quote), "emptyValue", StringUtils.quoteJson(emptyValue), "header", false);
    connector.configure(settings, true, true);
    connector.init();
    List<Record> records = Flux.merge(connector.read()).collectList().block();
    assertThat(records).hasSize(1);
    assertThat(records.get(0).getFieldValue(new DefaultIndexedField(0))).isEqualTo(expected);
    connector.close();
}

Example 15 with Record

use of com.datastax.oss.dsbulk.connectors.api.Record in project dsbulk by datastax.

the class CSVConnectorTest method should_read_single_file.

@ParameterizedTest(name = "[{index}] read {0} with compression {1} (sources: {2})")
@MethodSource
@DisplayName("Should read single file with given compression")
void should_read_single_file(String fileName, String compression, boolean retainRecordSources) throws Exception {
    CSVConnector connector = new CSVConnector();
    Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.csv", "url", url("/" + fileName), "normalizeLineEndingsInQuotes", true, "escape", "\"\\\"\"", "comment", "\"#\"", "compression", StringUtils.quoteJson(compression));
    connector.configure(settings, true, retainRecordSources);
    connector.init();
    assertThat(connector.readConcurrency()).isOne();
    List<Record> actual = Flux.merge(connector.read()).collectList().block();
    assertRecords(actual, retainRecordSources);
    connector.close();
}