Examples with CsvParserSettings com.univocity.parsers.csv.CsvParserSettings used on opensource projects

Example 21 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project Mycat-Server by MyCATApache.

the class ServerLoadDataInfileHandler method end.

@Override
public void end(byte packID) {
    isStartLoadData = false;
    this.packID = packID;
    // load in data空包 结束
    saveByteOrToFile(null, true);
    List<SQLExpr> columns = statement.getColumns();
    String tableName = statement.getTableName().getSimpleName();
    if (isHasStoreToFile) {
        parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
    } else {
        String content = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));
        // List<String> lines = Splitter.on(loadData.getLineTerminatedBy()).omitEmptyStrings().splitToList(content);
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if (loadData.getEnclose() != null) {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if (loadData.getEscape() != null) {
            settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
             *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
             *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
             *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
             */
        settings.trimValues(false);
        CsvParser parser = new CsvParser(settings);
        try {
            parser.beginParsing(new StringReader(content));
            String[] row = null;
            while ((row = parser.parseNext()) != null) {
                parseOneLine(columns, tableName, row, false, null);
            }
        } finally {
            parser.stopParsing();
        }
    }
    RouteResultset rrs = buildResultSet(routeResultMap);
    if (rrs != null) {
        flushDataToFile();
        serverConnection.getSession2().execute(rrs, ServerParse.LOAD_DATA_INFILE_SQL);
    }
// sendOk(++packID);
}

Example 22 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project dble by actiontech.

the class ServerLoadDataInfileHandler method end.

@Override
public void end(byte packId) {
    isStartLoadData = false;
    this.packID = packId;
    // empty packet for end
    saveByteOrToFile(null, true);
    List<SQLExpr> columns = statement.getColumns();
    String tableSimpleName = statement.getTableName().getSimpleName();
    if (isHasStoreToFile) {
        parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
    } else {
        String content = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));
        // List<String> lines = Splitter.on(loadData.getLineTerminatedBy()).omitEmptyStrings().splitToList(content);
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if (loadData.getEnclose() != null) {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if (loadData.getEscape() != null) {
            settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
             *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
             *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
             *  FIXME : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
             */
        settings.trimValues(false);
        CsvParser parser = new CsvParser(settings);
        try {
            parser.beginParsing(new StringReader(content));
            String[] row = null;
            int ignoreNumber = 0;
            if (statement.getIgnoreLinesNumber() != null && !"".equals(statement.getIgnoreLinesNumber().toString())) {
                ignoreNumber = Integer.parseInt(statement.getIgnoreLinesNumber().toString());
            }
            while ((row = parser.parseNext()) != null) {
                if (ignoreNumber == 0) {
                    parseOneLine(columns, tableSimpleName, row, true, loadData.getLineTerminatedBy());
                } else {
                    ignoreNumber--;
                }
            }
        } finally {
            parser.stopParsing();
        }
    }
    RouteResultset rrs = buildResultSet(routeResultMap);
    if (rrs != null) {
        flushDataToFile();
        serverConnection.getSession2().execute(rrs);
    }
// sendOk(++packID);
}