Examples with CsvParserSettings com.univocity.parsers.csv.CsvParserSettings used on opensource projects

Example 1 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project Orchid by JavaEden.

the class CSVParser method parse.

@Override
public JSONElement parse(String extension, String input) {
    List<String[]> allRows;
    if (extension.equalsIgnoreCase("csv")) {
        CsvParserSettings settings = new CsvParserSettings();
        settings.getFormat().setLineSeparator("\n");
        CsvParser parser = new CsvParser(settings);
        allRows = parser.parseAll(org.apache.commons.io.IOUtils.toInputStream(input));
    } else {
        TsvParserSettings settings = new TsvParserSettings();
        settings.getFormat().setLineSeparator("\n");
        TsvParser parser = new TsvParser(settings);
        allRows = parser.parseAll(org.apache.commons.io.IOUtils.toInputStream(input));
    }
    JSONArray array = new JSONArray();
    String[] cols = allRows.get(0);
    for (int i = 1; i < allRows.size(); i++) {
        JSONObject object = new JSONObject();
        for (int j = 0; j < cols.length; j++) {
            object.put(cols[j], allRows.get(i)[j]);
        }
        array.put(object);
    }
    JSONObject object = new JSONObject();
    object.put("list", array);
    return new JSONElement(object);
}

Example 2 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project carbondata by apache.

the class CSVStreamParserImp method initialize.

@Override
public void initialize(Configuration configuration, StructType structType) {
    CsvParserSettings settings = CSVInputFormat.extractCsvParserSettings(configuration);
    csvParser = new CsvParser(settings);
}

Example 3 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project mapping-benchmark by arnaudroger.

the class UnivocityCsvParserBenchmark method mapCsv.

@Benchmark
public void mapCsv(Blackhole blackhole, CsvParam csvParam) throws IOException {
    CsvParserSettings settings = new CsvParserSettings();
    // turning off features enabled by default
    settings.setIgnoreLeadingWhitespaces(false);
    settings.setIgnoreTrailingWhitespaces(false);
    settings.setSkipEmptyLines(false);
    settings.setColumnReorderingEnabled(false);
    settings.setReadInputOnSeparateThread(false);
    settings.setRowProcessor(new BeanProcessor<City>(City.class) {

        @Override
        public void beanProcessed(City bean, ParsingContext context) {
            blackhole.consume(bean);
        }
    });
    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(settings);
    try (Reader reader = csvParam.getReader()) {
        parser.parse(reader);
    }
}

Example 4 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project mapping-benchmark by arnaudroger.

the class UnivocityCsvParserBenchmark method parseCsv.

@Benchmark
public void parseCsv(Blackhole blackhole, CsvParam csvParam) throws IOException {
    CsvParserSettings settings = new CsvParserSettings();
    // turning off features enabled by default
    settings.setIgnoreLeadingWhitespaces(false);
    settings.setIgnoreTrailingWhitespaces(false);
    settings.setSkipEmptyLines(false);
    settings.setColumnReorderingEnabled(false);
    settings.setReadInputOnSeparateThread(false);
    settings.setRowProcessor(new AbstractRowProcessor() {

        @Override
        public void rowProcessed(String[] row, ParsingContext context) {
            blackhole.consume(row);
        }
    });
    com.univocity.parsers.csv.CsvParser parser = new com.univocity.parsers.csv.CsvParser(settings);
    try (Reader reader = csvParam.getReader()) {
        parser.parse(reader);
    }
}

Example 5 with CsvParserSettings

use of com.univocity.parsers.csv.CsvParserSettings in project edammap by edamontology.

the class Csv method load.

public static List<InputType> load(String queryPath, QueryType type, int timeout, String userAgent) throws IOException, ParseException {
    List<InputType> inputs = new ArrayList<>();
    BeanListProcessor<? extends InputType> rowProcessor;
    switch(type) {
        case SEQwiki:
            rowProcessor = new BeanListProcessor<SEQwiki>(SEQwiki.class);
            break;
        case msutils:
            rowProcessor = new BeanListProcessor<Msutils>(Msutils.class);
            break;
        case Bioconductor:
            rowProcessor = new BeanListProcessor<Bioconductor>(Bioconductor.class);
            break;
        default:
            rowProcessor = new BeanListProcessor<Generic>(Generic.class);
            break;
    }
    rowProcessor.setStrictHeaderValidationEnabled(false);
    CsvParserSettings settings = new CsvParserSettings();
    settings.setProcessor(rowProcessor);
    settings.setHeaderExtractionEnabled(true);
    settings.setAutoConfigurationEnabled(true);
    // disabling is (slightly) more efficient if your input is small
    settings.setReadInputOnSeparateThread(false);
    settings.setSkipEmptyLines(true);
    settings.trimValues(true);
    settings.setMaxCharsPerColumn(65536);
    settings.getFormat().setDelimiter(',');
    settings.getFormat().setQuote('"');
    settings.getFormat().setQuoteEscape('"');
    settings.getFormat().setCharToEscapeQuoteEscaping('"');
    settings.getFormat().setLineSeparator("\n");
    settings.getFormat().setComment('#');
    try (InputStreamReader reader = new InputStreamReader(Input.newInputStream(queryPath, true, timeout, userAgent), StandardCharsets.UTF_8)) {
        CsvParser parser = new CsvParser(settings);
        parser.parse(reader);
    }
    int i = 0;
    for (InputType inputType : rowProcessor.getBeans()) {
        inputType.check(++i);
        inputs.add(inputType);
    }
    return inputs;
}