Examples with DefaultMappedField com.datastax.oss.dsbulk.connectors.api.DefaultMappedField used on opensource projects

Example 1 with DefaultMappedField

use of com.datastax.oss.dsbulk.connectors.api.DefaultMappedField in project dsbulk by datastax.

the class DefaultMappingTest method should_create_mapping.

@Test
void should_create_mapping() {
    ImmutableSetMultimap<Field, CQLWord> fieldsToVariables = ImmutableSetMultimap.<Field, CQLWord>builder().put(new DefaultMappedField("f1"), CQLWord.fromInternal("c1")).put(new DefaultMappedField("f2"), CQLWord.fromInternal("c2")).build();
    when(codecFactory.createConvertingCodec(DataTypes.TEXT, GenericType.STRING, true)).thenAnswer(invocation -> textCodec);
    when(codecFactory.createConvertingCodec(DataTypes.BOOLEAN, GenericType.STRING, true)).thenAnswer(invocation -> booleanCodec);
    DefaultMapping mapping = new DefaultMapping(fieldsToVariables, codecFactory, ImmutableSet.of());
    assertThat(mapping.fieldToVariables(new MappedMappingField("f1"))).containsExactly(CQLWord.fromInternal("c1"));
    assertThat(mapping.fieldToVariables(new MappedMappingField("f2"))).containsExactly(CQLWord.fromInternal("c2"));
    assertThat(mapping.fieldToVariables(new MappedMappingField("nonexistent"))).isEmpty();
    assertThat(mapping.variableToFields(CQLWord.fromInternal("c1"))).containsExactly(new MappedMappingField("f1"));
    assertThat(mapping.variableToFields(CQLWord.fromInternal("c2"))).containsExactly(new MappedMappingField("f2"));
    assertThat(mapping.variableToFields(CQLWord.fromInternal("nonexistent"))).isEmpty();
    assertThat(mapping.codec(CQLWord.fromInternal("f1"), DataTypes.TEXT, GenericType.STRING)).isSameAs(textCodec);
    assertThat(mapping.codec(CQLWord.fromInternal("f2"), DataTypes.BOOLEAN, GenericType.STRING)).isSameAs(booleanCodec);
}

Example 2 with DefaultMappedField

use of com.datastax.oss.dsbulk.connectors.api.DefaultMappedField in project dsbulk by datastax.

the class CSVConnectorTest method should_honor_emptyValue_when_writing.

@ParameterizedTest
@MethodSource
void should_honor_emptyValue_when_writing(String quote, String emptyValue, String expected) throws Exception {
    Path out = Files.createTempDirectory("test");
    CSVConnector connector = new CSVConnector();
    Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.csv", "url", StringUtils.quoteJson(out), "quote", StringUtils.quoteJson(quote), "emptyValue", StringUtils.quoteJson(emptyValue), "header", false);
    connector.configure(settings, false, true);
    connector.init();
    Flux.<Record>just(DefaultRecord.mapped("source", resource, IRRELEVANT_POSITION, new Field[] { new DefaultMappedField("field1"), new DefaultMappedField("field2") }, "", "field2")).transform(connector.write()).blockLast();
    connector.close();
    List<String> actual = Files.readAllLines(out.resolve("output-000001.csv"));
    assertThat(actual).hasSize(1).containsExactly(expected);
}

Example 3 with DefaultMappedField

use of com.datastax.oss.dsbulk.connectors.api.DefaultMappedField in project dsbulk by datastax.

the class SearchEndToEndCCMIT method normal_unload_of_search_enabled_table.

/**
 * Test for DAT-365: regular unload of a search-enabled table should not contain the solr_query
 * column.
 */
@Test
void normal_unload_of_search_enabled_table() {
    session.execute("CREATE TABLE IF NOT EXISTS test_search2 (pk int, cc int, v varchar, PRIMARY KEY (pk, cc))");
    session.execute("CREATE SEARCH INDEX IF NOT EXISTS ON test_search2 WITH COLUMNS v { indexed:true };");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 0, 'foo')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 1, 'bar')");
    session.execute("INSERT INTO test_search2 (pk, cc, v) VALUES (0, 2, 'qix')");
    // Wait until index is built
    await().atMost(ONE_MINUTE).until(() -> !session.execute("SELECT v FROM test_search2 WHERE solr_query = '{\"q\": \"v:foo\"}'").all().isEmpty());
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.table");
    args.add("test_search2");
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(records).hasSize(3).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("foo");
    }, Index.atIndex(0)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("1");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("bar");
    }, Index.atIndex(1)).satisfies(record -> {
        assertThat(record.fields()).hasSize(3);
        assertThat(record.getFieldValue(new DefaultMappedField("pk"))).isEqualTo("0");
        assertThat(record.getFieldValue(new DefaultMappedField("cc"))).isEqualTo("2");
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("qix");
    }, Index.atIndex(2));
}

Example 4 with DefaultMappedField

use of com.datastax.oss.dsbulk.connectors.api.DefaultMappedField in project dsbulk by datastax.

the class SearchEndToEndCCMIT method full_unload_search_solr_query.

/**
 * Test for DAT-309: unload of a Solr query
 */
@Test
void full_unload_search_solr_query() {
    session.execute("CREATE TABLE IF NOT EXISTS test_search (pk int, cc int, v varchar, PRIMARY KEY (pk, cc))");
    session.execute("CREATE SEARCH INDEX IF NOT EXISTS ON test_search WITH COLUMNS v { indexed:true };");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 0, 'foo')");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 1, 'bar')");
    session.execute("INSERT INTO test_search (pk, cc, v) VALUES (0, 2, 'qix')");
    String query = "SELECT v FROM test_search WHERE solr_query = '{\"q\": \"v:foo\"}'";
    // Wait until index is built
    await().atMost(ONE_MINUTE).until(() -> !session.execute(query).all().isEmpty());
    List<String> args = new ArrayList<>();
    args.add("unload");
    args.add("--connector.name");
    args.add("mock");
    args.add("--executor.continuousPaging.enabled");
    args.add("true");
    args.add("--schema.keyspace");
    args.add(session.getKeyspace().map(CqlIdentifier::asInternal).orElseThrow(IllegalStateException::new));
    args.add("--schema.query");
    args.add(StringUtils.quoteJson(query));
    ExitStatus status = new DataStaxBulkLoader(addCommonSettings(args)).run();
    assertStatus(status, STATUS_OK);
    assertThat(logs).hasMessageContaining("completed successfully").hasMessageContaining("Continuous paging is enabled but is not compatible with search queries; disabling");
    assertThat(stderr.getStreamAsStringPlain()).contains("completed successfully").contains("Continuous paging is enabled but is not compatible with search queries; disabling");
    assertThat(records).singleElement().satisfies(record -> {
        assertThat(record.fields()).hasSize(1);
        assertThat(record.getFieldValue(new DefaultMappedField("v"))).isEqualTo("foo");
    });
}

Example 5 with DefaultMappedField

use of com.datastax.oss.dsbulk.connectors.api.DefaultMappedField in project dsbulk by datastax.

the class RecordUtils method mappedJson.

public static Record mappedJson(String... tokens) {
    int counter = COUNTER.incrementAndGet();
    DefaultRecord record = DefaultRecord.indexed("source" + counter, URI.create("file://file" + counter + ".json"), counter - 1);
    for (int i = 0; i < tokens.length; i += 2) {
        record.put(new DefaultMappedField(tokens[i]), JsonNodeFactory.instance.textNode(tokens[i + 1]));
    }
    return record;
}