Examples with CsvParserSettings com.univocity.parsers.csv.CsvParserSettings used on opensource projects

Example 16 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project Mycat_plus by coderczp.

the class ServerLoadDataInfileHandler method parseFileByLine.

private void parseFileByLine(String file, String encode, String split) {
    List<SQLExpr> columns = statement.getColumns();
    CsvParserSettings settings = new CsvParserSettings();
    settings.setMaxColumns(65535);
    settings.setMaxCharsPerColumn(65535);
    settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
    settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
    if (loadData.getEnclose() != null) {
        settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
    }
    if (loadData.getEscape() != null) {
        settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
    }
    settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
    /*
         *  fix #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
         *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
         *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
         */
    settings.trimValues(false);
    CsvParser parser = new CsvParser(settings);
    InputStreamReader reader = null;
    FileInputStream fileInputStream = null;
    try {
        fileInputStream = new FileInputStream(file);
        reader = new InputStreamReader(fileInputStream, encode);
        parser.beginParsing(reader);
        String[] row = null;
        while ((row = parser.parseNext()) != null) {
            parseOneLine(columns, tableName, row, true, loadData.getLineTerminatedBy());
        }
    } catch (FileNotFoundException | UnsupportedEncodingException e) {
        throw new RuntimeException(e);
    } finally {
        parser.stopParsing();
        if (fileInputStream != null) {
            try {
                fileInputStream.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
        if (reader != null) {
            try {
                reader.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
}

Example 17 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project lightning by automatictester.

the class JMeterCSVFileReader method getParser.

protected CsvParser getParser() {
    CsvParserSettings parserSettings = new CsvParserSettings();
    parserSettings.setLineSeparatorDetectionEnabled(true);
    parserSettings.setHeaderExtractionEnabled(true);
    parserSettings.selectFields("label", "elapsed", "success", "timeStamp");
    RowListProcessor rowProcessor = new RowListProcessor();
    parserSettings.setProcessor(new ConcurrentRowProcessor(rowProcessor));
    return new CsvParser(parserSettings);
}

Example 18 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project lightning by automatictester.

the class PerfMonDataReader method getParser.

protected CsvParser getParser() {
    CsvParserSettings parserSettings = new CsvParserSettings();
    parserSettings.setLineSeparatorDetectionEnabled(true);
    parserSettings.setHeaderExtractionEnabled(false);
    parserSettings.selectIndexes(TIMESTAMP, VALUE, HOST_AND_METRIC);
    RowListProcessor rowProcessor = new RowListProcessor();
    parserSettings.setProcessor(new ConcurrentRowProcessor(rowProcessor));
    return new CsvParser(parserSettings);
}

Example 19 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project junit5 by junit-team.

the class CsvArgumentsProvider method provideArguments.

@Override
public Stream<? extends Arguments> provideArguments(ExtensionContext context) {
    CsvParserSettings settings = new CsvParserSettings();
    settings.getFormat().setDelimiter(delimiter);
    settings.getFormat().setLineSeparator(LINE_SEPARATOR);
    settings.getFormat().setQuote('\'');
    settings.getFormat().setQuoteEscape('\'');
    settings.setEmptyValue("");
    settings.setAutoConfigurationEnabled(false);
    CsvParser csvParser = new CsvParser(settings);
    AtomicLong index = new AtomicLong(0);
    // @formatter:off
    return Arrays.stream(lines).map(line -> Preconditions.notNull(csvParser.parseLine(line + LINE_SEPARATOR), () -> "Line at index " + index.get() + " contains invalid CSV: \"" + line + "\"")).peek(values -> index.incrementAndGet()).map(Arguments::of);
// @formatter:on
}

Example 20 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project junit5 by junit-team.

the class CsvFileArgumentsProvider method accept.

@Override
public void accept(CsvFileSource annotation) {
    resources = annotation.resources();
    charset = Charset.forName(annotation.encoding());
    numLinesToSkip = annotation.numLinesToSkip();
    settings = new CsvParserSettings();
    settings.getFormat().setDelimiter(annotation.delimiter());
    settings.getFormat().setLineSeparator(annotation.lineSeparator());
    settings.getFormat().setQuote('"');
    settings.getFormat().setQuoteEscape('"');
    settings.setEmptyValue("");
    settings.setAutoConfigurationEnabled(false);
}