Examples with Field com.datastax.oss.dsbulk.connectors.api.Field used on opensource projects

Example 1 with Field

use of com.datastax.oss.dsbulk.connectors.api.Field in project dsbulk by datastax.

the class DefaultMappingTest method should_create_mapping.

@Test
void should_create_mapping() {
    ImmutableSetMultimap<Field, CQLWord> fieldsToVariables = ImmutableSetMultimap.<Field, CQLWord>builder().put(new DefaultMappedField("f1"), CQLWord.fromInternal("c1")).put(new DefaultMappedField("f2"), CQLWord.fromInternal("c2")).build();
    when(codecFactory.createConvertingCodec(DataTypes.TEXT, GenericType.STRING, true)).thenAnswer(invocation -> textCodec);
    when(codecFactory.createConvertingCodec(DataTypes.BOOLEAN, GenericType.STRING, true)).thenAnswer(invocation -> booleanCodec);
    DefaultMapping mapping = new DefaultMapping(fieldsToVariables, codecFactory, ImmutableSet.of());
    assertThat(mapping.fieldToVariables(new MappedMappingField("f1"))).containsExactly(CQLWord.fromInternal("c1"));
    assertThat(mapping.fieldToVariables(new MappedMappingField("f2"))).containsExactly(CQLWord.fromInternal("c2"));
    assertThat(mapping.fieldToVariables(new MappedMappingField("nonexistent"))).isEmpty();
    assertThat(mapping.variableToFields(CQLWord.fromInternal("c1"))).containsExactly(new MappedMappingField("f1"));
    assertThat(mapping.variableToFields(CQLWord.fromInternal("c2"))).containsExactly(new MappedMappingField("f2"));
    assertThat(mapping.variableToFields(CQLWord.fromInternal("nonexistent"))).isEmpty();
    assertThat(mapping.codec(CQLWord.fromInternal("f1"), DataTypes.TEXT, GenericType.STRING)).isSameAs(textCodec);
    assertThat(mapping.codec(CQLWord.fromInternal("f2"), DataTypes.BOOLEAN, GenericType.STRING)).isSameAs(booleanCodec);
}

Example 2 with Field

use of com.datastax.oss.dsbulk.connectors.api.Field in project dsbulk by datastax.

the class CSVConnectorTest method should_honor_emptyValue_when_writing.

@ParameterizedTest
@MethodSource
void should_honor_emptyValue_when_writing(String quote, String emptyValue, String expected) throws Exception {
    Path out = Files.createTempDirectory("test");
    CSVConnector connector = new CSVConnector();
    Config settings = TestConfigUtils.createTestConfig("dsbulk.connector.csv", "url", StringUtils.quoteJson(out), "quote", StringUtils.quoteJson(quote), "emptyValue", StringUtils.quoteJson(emptyValue), "header", false);
    connector.configure(settings, false, true);
    connector.init();
    Flux.<Record>just(DefaultRecord.mapped("source", resource, IRRELEVANT_POSITION, new Field[] { new DefaultMappedField("field1"), new DefaultMappedField("field2") }, "", "field2")).transform(connector.write()).blockLast();
    connector.close();
    List<String> actual = Files.readAllLines(out.resolve("output-000001.csv"));
    assertThat(actual).hasSize(1).containsExactly(expected);
}

Example 3 with Field

use of com.datastax.oss.dsbulk.connectors.api.Field in project dsbulk by datastax.

the class SchemaSettingsTest method assertMapping.

private static void assertMapping(DefaultMapping mapping, Object... fieldsAndVars) {
    ImmutableSetMultimap.Builder<Object, Object> expected = ImmutableSetMultimap.builder();
    for (int i = 0; i < fieldsAndVars.length; i += 2) {
        String first = fieldsAndVars[i] instanceof String ? (String) fieldsAndVars[i] : ((CqlIdentifier) fieldsAndVars[i]).asInternal();
        CQLWord second = fieldsAndVars[i + 1] instanceof String ? CQLWord.fromInternal((String) fieldsAndVars[i + 1]) : CQLWord.fromCqlIdentifier((CqlIdentifier) fieldsAndVars[i + 1]);
        if (CharMatcher.inRange('0', '9').matchesAllOf(first)) {
            expected.put(new DefaultIndexedField(Integer.parseInt(first)), second);
        } else {
            expected.put(new DefaultMappedField(first), second);
        }
    }
    @SuppressWarnings("unchecked") SetMultimap<Field, CQLWord> fieldsToVariables = (SetMultimap<Field, CQLWord>) getInternalState(mapping, "fieldsToVariables");
    assertThat(fieldsToVariables).isEqualTo(expected.build());
}

Example 4 with Field

use of com.datastax.oss.dsbulk.connectors.api.Field in project dsbulk by datastax.

the class DefaultReadResultMapper method map.

@NonNull
@Override
public Record map(@NonNull ReadResult result) {
    Object source = retainRecordSources ? result : null;
    try {
        Row row = result.getRow().orElseThrow(IllegalStateException::new);
        ColumnDefinitions columnDefinitions = row.getColumnDefinitions();
        DefaultRecord record = new DefaultRecord(source, resource, -1);
        for (ColumnDefinition def : columnDefinitions) {
            CQLWord variable = CQLWord.fromInternal(def.getName().asInternal());
            CqlIdentifier name = variable.asIdentifier();
            DataType cqlType = def.getType();
            Set<Field> fields = mapping.variableToFields(variable);
            for (Field field : fields) {
                GenericType<?> fieldType = null;
                try {
                    fieldType = recordMetadata.getFieldType(field, cqlType);
                    TypeCodec<?> codec = mapping.codec(variable, cqlType, fieldType);
                    Object value = row.get(name, codec);
                    record.setFieldValue(field, value);
                } catch (Exception e) {
                    String msg = String.format("Could not deserialize column %s of type %s as %s", name.asCql(true), cqlType, fieldType);
                    throw new IllegalArgumentException(msg, e);
                }
            }
        }
        return record;
    } catch (Exception e) {
        return new DefaultErrorRecord(source, resource, -1, e);
    }
}

Example 5 with Field

use of com.datastax.oss.dsbulk.connectors.api.Field in project dsbulk by datastax.

the class DefaultRecordMapper method bindStatement.

private MappedBoundStatement bindStatement(Record record, PreparedStatement insertStatement) {
    BoundStatementBuilder builder = boundStatementBuilderFactory.apply(insertStatement);
    ColumnDefinitions variableDefinitions = insertStatement.getVariableDefinitions();
    for (Field field : record.fields()) {
        Set<CQLWord> variables = mapping.fieldToVariables(field);
        for (CQLWord variable : variables) {
            CqlIdentifier name = variable.asIdentifier();
            if (size == 1 || variableDefinitions.contains(name)) {
                DataType cqlType = variableDefinitions.get(name).getType();
                GenericType<?> fieldType = recordMetadata.getFieldType(field, cqlType);
                Object raw = record.getFieldValue(field);
                builder = bindColumn(builder, field, variable, raw, cqlType, fieldType);
            }
        }
    }
    ensurePrimaryKeySet(builder);
    if (protocolVersion.getCode() < DefaultProtocolVersion.V4.getCode()) {
        ensureAllVariablesSet(builder, insertStatement);
    }
    BoundStatement bs = builder.build();
    return new MappedBoundStatement(record, bs);
}