Examples with ALLOW_COMMENTS - org.apache.flink.formats.csv.CsvFormatOptions.ALLOW

Example 1 with ALLOW_COMMENTS

use of org.apache.flink.formats.csv.CsvFormatOptions.ALLOW_COMMENTS in project flink by apache.

the class CsvFileFormatFactory method buildCsvSchema.

private static CsvSchema buildCsvSchema(RowType rowType, ReadableConfig options) {
    final CsvSchema csvSchema = CsvRowSchemaConverter.convert(rowType);
    final CsvSchema.Builder csvBuilder = csvSchema.rebuild();
    // format properties
    options.getOptional(FIELD_DELIMITER).map(s -> StringEscapeUtils.unescapeJava(s).charAt(0)).ifPresent(csvBuilder::setColumnSeparator);
    if (options.get(DISABLE_QUOTE_CHARACTER)) {
        csvBuilder.disableQuoteChar();
    } else {
        options.getOptional(QUOTE_CHARACTER).map(s -> s.charAt(0)).ifPresent(csvBuilder::setQuoteChar);
    }
    options.getOptional(ALLOW_COMMENTS).ifPresent(csvBuilder::setAllowComments);
    options.getOptional(ARRAY_ELEMENT_DELIMITER).ifPresent(csvBuilder::setArrayElementSeparator);
    options.getOptional(ESCAPE_CHARACTER).map(s -> s.charAt(0)).ifPresent(csvBuilder::setEscapeChar);
    options.getOptional(NULL_LITERAL).ifPresent(csvBuilder::setNullValue);
    return csvBuilder.build();
}

Also used : DynamicTableFactory(org.apache.flink.table.factories.DynamicTableFactory) DataType(org.apache.flink.table.types.DataType) EncodingFormat(org.apache.flink.table.connector.format.EncodingFormat) ChangelogMode(org.apache.flink.table.connector.ChangelogMode) FIELD_DELIMITER(org.apache.flink.formats.csv.CsvFormatOptions.FIELD_DELIMITER) BulkWriterFormatFactory(org.apache.flink.connector.file.table.factories.BulkWriterFormatFactory) CsvSchema(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema) Context(org.apache.flink.table.connector.source.DynamicTableSource.Context) JsonNode(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.JsonNode) RowType(org.apache.flink.table.types.logical.RowType) ALLOW_COMMENTS(org.apache.flink.formats.csv.CsvFormatOptions.ALLOW_COMMENTS) Factory(org.apache.flink.api.common.serialization.BulkWriter.Factory) ReadableConfig(org.apache.flink.configuration.ReadableConfig) FileSourceSplit(org.apache.flink.connector.file.src.FileSourceSplit) IGNORE_PARSE_ERRORS(org.apache.flink.formats.csv.CsvFormatOptions.IGNORE_PARSE_ERRORS) QUOTE_CHARACTER(org.apache.flink.formats.csv.CsvFormatOptions.QUOTE_CHARACTER) RowDataToCsvConverter(org.apache.flink.formats.csv.RowDataToCsvConverters.RowDataToCsvConverter) ESCAPE_CHARACTER(org.apache.flink.formats.csv.CsvFormatOptions.ESCAPE_CHARACTER) StreamFormatAdapter(org.apache.flink.connector.file.src.impl.StreamFormatAdapter) ConfigOption(org.apache.flink.configuration.ConfigOption) StringEscapeUtils(org.apache.commons.lang3.StringEscapeUtils) Preconditions.checkNotNull(org.apache.flink.util.Preconditions.checkNotNull) BulkDecodingFormat(org.apache.flink.connector.file.table.format.BulkDecodingFormat) Projection(org.apache.flink.table.connector.Projection) BulkReaderFormatFactory(org.apache.flink.connector.file.table.factories.BulkReaderFormatFactory) RowData(org.apache.flink.table.data.RowData) DynamicTableSink(org.apache.flink.table.connector.sink.DynamicTableSink) BulkWriter(org.apache.flink.api.common.serialization.BulkWriter) ObjectNode(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode) Set(java.util.Set) ProjectableDecodingFormat(org.apache.flink.table.connector.format.ProjectableDecodingFormat) DISABLE_QUOTE_CHARACTER(org.apache.flink.formats.csv.CsvFormatOptions.DISABLE_QUOTE_CHARACTER) ARRAY_ELEMENT_DELIMITER(org.apache.flink.formats.csv.CsvFormatOptions.ARRAY_ELEMENT_DELIMITER) Converter(org.apache.flink.formats.common.Converter) NULL_LITERAL(org.apache.flink.formats.csv.CsvFormatOptions.NULL_LITERAL) Internal(org.apache.flink.annotation.Internal) BulkFormat(org.apache.flink.connector.file.src.reader.BulkFormat) Collections(java.util.Collections) CsvMapper(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvMapper) CsvSchema(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.dataformat.csv.CsvSchema)

Example 2 with ALLOW_COMMENTS

use of org.apache.flink.formats.csv.CsvFormatOptions.ALLOW_COMMENTS in project flink-mirror by flink-ci.

the class CsvFileFormatFactory method buildCsvSchema.

private static CsvSchema buildCsvSchema(RowType rowType, ReadableConfig options) {
    final CsvSchema csvSchema = CsvRowSchemaConverter.convert(rowType);
    final CsvSchema.Builder csvBuilder = csvSchema.rebuild();
    // format properties
    options.getOptional(FIELD_DELIMITER).map(s -> StringEscapeUtils.unescapeJava(s).charAt(0)).ifPresent(csvBuilder::setColumnSeparator);
    if (options.get(DISABLE_QUOTE_CHARACTER)) {
        csvBuilder.disableQuoteChar();
    } else {
        options.getOptional(QUOTE_CHARACTER).map(s -> s.charAt(0)).ifPresent(csvBuilder::setQuoteChar);
    }
    options.getOptional(ALLOW_COMMENTS).ifPresent(csvBuilder::setAllowComments);
    options.getOptional(ARRAY_ELEMENT_DELIMITER).ifPresent(csvBuilder::setArrayElementSeparator);
    options.getOptional(ESCAPE_CHARACTER).map(s -> s.charAt(0)).ifPresent(csvBuilder::setEscapeChar);
    options.getOptional(NULL_LITERAL).ifPresent(csvBuilder::setNullValue);
    return csvBuilder.build();
}

Example 3 with ALLOW_COMMENTS

use of org.apache.flink.formats.csv.CsvFormatOptions.ALLOW_COMMENTS in project flink by splunk.

the class CsvFileFormatFactory method buildCsvSchema.

private static CsvSchema buildCsvSchema(RowType rowType, ReadableConfig options) {
    final CsvSchema csvSchema = CsvRowSchemaConverter.convert(rowType);
    final CsvSchema.Builder csvBuilder = csvSchema.rebuild();
    // format properties
    options.getOptional(FIELD_DELIMITER).map(s -> StringEscapeUtils.unescapeJava(s).charAt(0)).ifPresent(csvBuilder::setColumnSeparator);
    if (options.get(DISABLE_QUOTE_CHARACTER)) {
        csvBuilder.disableQuoteChar();
    } else {
        options.getOptional(QUOTE_CHARACTER).map(s -> s.charAt(0)).ifPresent(csvBuilder::setQuoteChar);
    }
    options.getOptional(ALLOW_COMMENTS).ifPresent(csvBuilder::setAllowComments);
    options.getOptional(ARRAY_ELEMENT_DELIMITER).ifPresent(csvBuilder::setArrayElementSeparator);
    options.getOptional(ESCAPE_CHARACTER).map(s -> s.charAt(0)).ifPresent(csvBuilder::setEscapeChar);
    options.getOptional(NULL_LITERAL).ifPresent(csvBuilder::setNullValue);
    return csvBuilder.build();
}

Aggregations

Collections (java.util.Collections)3 Set (java.util.Set)3 StringEscapeUtils (org.apache.commons.lang3.StringEscapeUtils)3 Internal (org.apache.flink.annotation.Internal)3 BulkWriter (org.apache.flink.api.common.serialization.BulkWriter)3 Factory (org.apache.flink.api.common.serialization.BulkWriter.Factory)3 ConfigOption (org.apache.flink.configuration.ConfigOption)3 ReadableConfig (org.apache.flink.configuration.ReadableConfig)3 FileSourceSplit (org.apache.flink.connector.file.src.FileSourceSplit)3 StreamFormatAdapter (org.apache.flink.connector.file.src.impl.StreamFormatAdapter)3 BulkFormat (org.apache.flink.connector.file.src.reader.BulkFormat)3 BulkReaderFormatFactory (org.apache.flink.connector.file.table.factories.BulkReaderFormatFactory)3 BulkWriterFormatFactory (org.apache.flink.connector.file.table.factories.BulkWriterFormatFactory)3 BulkDecodingFormat (org.apache.flink.connector.file.table.format.BulkDecodingFormat)3 Converter (org.apache.flink.formats.common.Converter)3 ALLOW_COMMENTS (org.apache.flink.formats.csv.CsvFormatOptions.ALLOW_COMMENTS)3 ARRAY_ELEMENT_DELIMITER (org.apache.flink.formats.csv.CsvFormatOptions.ARRAY_ELEMENT_DELIMITER)3 DISABLE_QUOTE_CHARACTER (org.apache.flink.formats.csv.CsvFormatOptions.DISABLE_QUOTE_CHARACTER)3 ESCAPE_CHARACTER (org.apache.flink.formats.csv.CsvFormatOptions.ESCAPE_CHARACTER)3 FIELD_DELIMITER (org.apache.flink.formats.csv.CsvFormatOptions.FIELD_DELIMITER)3