Examples with CsvParser com.univocity.parsers.csv.CsvParser used on opensource projects

Example 1 with CsvParser

use of com.univocity.parsers.csv.CsvParser in project Mycat-Server by MyCATApache.

the class ServerLoadDataInfileHandler method parseFileByLine.

private void parseFileByLine(String file, String encode, String split) {
    List<SQLExpr> columns = statement.getColumns();
    CsvParserSettings settings = new CsvParserSettings();
    settings.setMaxColumns(65535);
    settings.setMaxCharsPerColumn(65535);
    settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
    settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
    if (loadData.getEnclose() != null) {
        settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
    }
    if (loadData.getEscape() != null) {
        settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
    }
    settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
    /*
         *  fix #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
         *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
         *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
         */
    settings.trimValues(false);
    CsvParser parser = new CsvParser(settings);
    InputStreamReader reader = null;
    FileInputStream fileInputStream = null;
    try {
        fileInputStream = new FileInputStream(file);
        reader = new InputStreamReader(fileInputStream, encode);
        parser.beginParsing(reader);
        String[] row = null;
        while ((row = parser.parseNext()) != null) {
            parseOneLine(columns, tableName, row, true, loadData.getLineTerminatedBy());
        }
    } catch (FileNotFoundException | UnsupportedEncodingException e) {
        throw new RuntimeException(e);
    } finally {
        parser.stopParsing();
        if (fileInputStream != null) {
            try {
                fileInputStream.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
        if (reader != null) {
            try {
                reader.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
}

Example 2 with CsvParser

use of com.univocity.parsers.csv.CsvParser in project Orchid by JavaEden.

the class CSVParser method parse.

@Override
public JSONElement parse(String extension, String input) {
    List<String[]> allRows;
    if (extension.equalsIgnoreCase("csv")) {
        CsvParserSettings settings = new CsvParserSettings();
        settings.getFormat().setLineSeparator("\n");
        CsvParser parser = new CsvParser(settings);
        allRows = parser.parseAll(org.apache.commons.io.IOUtils.toInputStream(input));
    } else {
        TsvParserSettings settings = new TsvParserSettings();
        settings.getFormat().setLineSeparator("\n");
        TsvParser parser = new TsvParser(settings);
        allRows = parser.parseAll(org.apache.commons.io.IOUtils.toInputStream(input));
    }
    JSONArray array = new JSONArray();
    String[] cols = allRows.get(0);
    for (int i = 1; i < allRows.size(); i++) {
        JSONObject object = new JSONObject();
        for (int j = 0; j < cols.length; j++) {
            object.put(cols[j], allRows.get(i)[j]);
        }
        array.put(object);
    }
    JSONObject object = new JSONObject();
    object.put("list", array);
    return new JSONElement(object);
}

Example 3 with CsvParser

use of com.univocity.parsers.csv.CsvParser in project Mycat-Server by MyCATApache.

the class ServerLoadDataInfileHandler method end.

@Override
public void end(byte packID) {
    isStartLoadData = false;
    this.packID = packID;
    //load in data空包 结束
    saveByteOrToFile(null, true);
    List<SQLExpr> columns = statement.getColumns();
    String tableName = statement.getTableName().getSimpleName();
    if (isHasStoreToFile) {
        parseFileByLine(tempFile, loadData.getCharset(), loadData.getLineTerminatedBy());
    } else {
        String content = new String(tempByteBuffer.toByteArray(), Charset.forName(loadData.getCharset()));
        // List<String> lines = Splitter.on(loadData.getLineTerminatedBy()).omitEmptyStrings().splitToList(content);
        CsvParserSettings settings = new CsvParserSettings();
        settings.setMaxColumns(65535);
        settings.setMaxCharsPerColumn(65535);
        settings.getFormat().setLineSeparator(loadData.getLineTerminatedBy());
        settings.getFormat().setDelimiter(loadData.getFieldTerminatedBy().charAt(0));
        if (loadData.getEnclose() != null) {
            settings.getFormat().setQuote(loadData.getEnclose().charAt(0));
        }
        if (loadData.getEscape() != null) {
            settings.getFormat().setQuoteEscape(loadData.getEscape().charAt(0));
        }
        settings.getFormat().setNormalizedNewline(loadData.getLineTerminatedBy().charAt(0));
        /*
             *  fix bug #1074 : LOAD DATA local INFILE导入的所有Boolean类型全部变成了false
             *  不可见字符将在CsvParser被当成whitespace过滤掉, 使用settings.trimValues(false)来避免被过滤掉
             *  TODO : 设置trimValues(false)之后, 会引起字段值前后的空白字符无法被过滤!
             */
        settings.trimValues(false);
        CsvParser parser = new CsvParser(settings);
        try {
            parser.beginParsing(new StringReader(content));
            String[] row = null;
            while ((row = parser.parseNext()) != null) {
                parseOneLine(columns, tableName, row, false, null);
            }
        } finally {
            parser.stopParsing();
        }
    }
    RouteResultset rrs = buildResultSet(routeResultMap);
    if (rrs != null) {
        flushDataToFile();
        serverConnection.getSession2().execute(rrs, ServerParse.LOAD_DATA_INFILE_SQL);
    }
// sendOk(++packID);
}