Examples with CSVParserBuilder com.opencsv.CSVParserBuilder used on opensource projects

Example 1 with CSVParserBuilder

use of com.opencsv.CSVParserBuilder in project collect by opendatakit.

the class ExternalSQLiteOpenHelper method onCreateNamed.

private void onCreateNamed(SQLiteDatabase db, String tableName) throws Exception {
    Timber.w("Reading data from '%s", dataSetFile.toString());
    onProgress(getLocalizedString(Collect.getInstance(), R.string.ext_import_progress_message, dataSetFile.getName(), ""));
    CSVReader reader = null;
    try {
        reader = new CSVReaderBuilder(new FileReader(dataSetFile)).withCSVParser(new CSVParserBuilder().withSeparator(DELIMITING_CHAR).withQuoteChar(QUOTE_CHAR).withEscapeChar(ESCAPE_CHAR).build()).build();
        String[] headerRow = reader.readNext();
        headerRow[0] = removeByteOrderMark(headerRow[0]);
        if (!ExternalDataUtil.containsAnyData(headerRow)) {
            throw new ExternalDataException(getLocalizedString(Collect.getInstance(), R.string.ext_file_no_data_error));
        }
        List<String> conflictingColumns = ExternalDataUtil.findMatchingColumnsAfterSafeningNames(headerRow);
        if (conflictingColumns != null && !conflictingColumns.isEmpty()) {
            // so the create table query will fail with "duplicate column" error.
            throw new ExternalDataException(getLocalizedString(Collect.getInstance(), R.string.ext_conflicting_columns_error, conflictingColumns));
        }
        Map<String, String> columnNamesCache = new HashMap<>();
        StringBuilder sb = new StringBuilder();
        boolean sortColumnAlreadyPresent = false;
        sb.append("CREATE TABLE IF NOT EXISTS ").append(tableName).append(" ( ");
        for (int i = 0; i < headerRow.length; i++) {
            String columnName = headerRow[i].trim();
            if (columnName.length() == 0) {
                continue;
            }
            if (i != 0) {
                sb.append(", ");
            }
            String safeColumnName = ExternalDataUtil.toSafeColumnName(columnName, columnNamesCache);
            if (safeColumnName.equals(ExternalDataUtil.SORT_COLUMN_NAME)) {
                sortColumnAlreadyPresent = true;
                sb.append(safeColumnName).append(" real ");
            } else {
                sb.append(safeColumnName).append(" text collate nocase ");
            }
        }
        if (!sortColumnAlreadyPresent) {
            sb.append(", ");
            sb.append(ExternalDataUtil.SORT_COLUMN_NAME).append(" real ");
        }
        sb.append(" );");
        String sql = sb.toString();
        Timber.w("Creating database for %s with query: %s", dataSetFile, sql);
        db.execSQL(sql);
        // create the indexes.
        // save the sql for later because inserts will be much faster if we don't have
        // indexes already.
        List<String> createIndexesCommands = new ArrayList<>();
        for (String header : headerRow) {
            if (header.endsWith("_key")) {
                String indexSQL = "CREATE INDEX " + header + "_idx ON " + tableName + " (" + ExternalDataUtil.toSafeColumnName(header, columnNamesCache) + ");";
                createIndexesCommands.add(indexSQL);
                Timber.w("Will create an index on %s later.", header);
            }
        }
        // populate the database
        String[] row = reader.readNext();
        int rowCount = 0;
        while (row != null && !isCancelled()) {
            // SCTO-894 - first we should make sure that this is not an empty line
            if (!ExternalDataUtil.containsAnyData(row)) {
                // yes, that is an empty row, ignore it
                row = reader.readNext();
                continue;
            }
            // we will just fill up the rest with empty strings
            if (row.length < headerRow.length) {
                row = ExternalDataUtil.fillUpNullValues(row, headerRow);
            }
            ContentValues values = new ContentValues();
            if (!sortColumnAlreadyPresent) {
                values.put(ExternalDataUtil.SORT_COLUMN_NAME, rowCount + 1);
            }
            for (int i = 0; i < row.length && i < headerRow.length; i++) {
                String columnName = headerRow[i].trim();
                String columnValue = row[i];
                if (columnName.length() == 0) {
                    continue;
                }
                String safeColumnName = ExternalDataUtil.toSafeColumnName(columnName, columnNamesCache);
                if (safeColumnName.equals(ExternalDataUtil.SORT_COLUMN_NAME)) {
                    try {
                        values.put(safeColumnName, Double.parseDouble(columnValue));
                    } catch (NumberFormatException e) {
                        throw new ExternalDataException(getLocalizedString(Collect.getInstance(), R.string.ext_sortBy_numeric_error, columnValue));
                    }
                } else {
                    values.put(safeColumnName, columnValue);
                }
            }
            db.insertOrThrow(tableName, null, values);
            row = reader.readNext();
            rowCount++;
            if (rowCount % 100 == 0) {
                onProgress(getLocalizedString(Collect.getInstance(), R.string.ext_import_progress_message, dataSetFile.getName(), " (" + rowCount + " records so far)"));
            }
        }
        if (isCancelled()) {
            Timber.w("User canceled reading data from %s", dataSetFile.toString());
            onProgress(getLocalizedString(Collect.getInstance(), R.string.ext_import_cancelled_message));
        } else {
            onProgress(getLocalizedString(Collect.getInstance(), R.string.ext_import_finalizing_message));
            // now create the indexes
            for (String createIndexCommand : createIndexesCommands) {
                Timber.w(createIndexCommand);
                db.execSQL(createIndexCommand);
            }
            Timber.w("Read all data from %s", dataSetFile.toString());
            onProgress(getLocalizedString(Collect.getInstance(), R.string.ext_import_completed_message));
        }
    } finally {
        if (reader != null) {
            try {
                reader.close();
            } catch (IOException e) {
                Timber.e(e);
            }
        }
    }
}

Example 2 with CSVParserBuilder

use of com.opencsv.CSVParserBuilder in project cu-kfs by CU-CommunityApps.

the class CsvBatchInputFileTypeBase method parse.

/**
 * @return parsed object in structure - List<Map<String, String>>
 */
public Object parse(byte[] fileByteContent) throws ParseException {
    // handle null objects and zero byte contents
    String errorMessage = fileByteContent == null ? "an invalid(null) argument was given" : fileByteContent.length == 0 ? "an invalid argument was given, empty input stream" : "";
    if (!errorMessage.isEmpty()) {
        LOG.error(errorMessage);
        throw new IllegalArgumentException(errorMessage);
    }
    List<String> headerList = getCsvHeaderList();
    Object parsedContents;
    validateCSVHeader(fileByteContent, headerList);
    final CSVParser csvParser = new CSVParserBuilder().withSeparator(',').withQuoteChar('"').withEscapeChar('|').build();
    try (final InputStreamReader inputStreamReader = new InputStreamReader(new ByteArrayInputStream(fileByteContent), StandardCharsets.UTF_8);
        final CSVReader csvReader = new CSVReaderBuilder(inputStreamReader).withSkipLines(1).withCSVParser(csvParser).build()) {
        List<String[]> dataList = csvReader.readAll();
        // parse and create List of Maps base on enum value names as map keys
        List<Map<String, String>> dataMapList = new ArrayList<>();
        Map<String, String> rowMap;
        int index;
        for (String[] row : dataList) {
            rowMap = new LinkedHashMap<>();
            // reset index
            index = 0;
            for (String header : headerList) {
                rowMap.put(header, row[index++]);
            }
            dataMapList.add(rowMap);
        }
        parsedContents = dataMapList;
    } catch (IOException ex) {
        LOG.error(ex.getMessage(), ex);
        throw new ParseException(ex.getMessage(), ex);
    }
    return convertParsedObjectToVO(parsedContents);
}

Example 3 with CSVParserBuilder

use of com.opencsv.CSVParserBuilder in project fql by CategoricalData.

the class InstExpCsv method start2.

/*
	 * @Override public boolean equals(Object obj) { return (obj instanceof
	 * InstExpCsv) && super.equals(obj); }
	 */
/**
 * Expects filenames in the map
 */
public static Map<En, List<String[]>> start2(Map<String, String> map, AqlOptions op, Schema<Ty, En, Sym, Fk, Att> sch, boolean omitCheck) throws Exception {
    Character sepChar = (Character) op.getOrDefault(AqlOption.csv_field_delim_char);
    Character quoteChar = (Character) op.getOrDefault(AqlOption.csv_quote_char);
    Character escapeChar = (Character) op.getOrDefault(AqlOption.csv_escape_char);
    final CSVParser parser = new CSVParserBuilder().withSeparator(sepChar).withQuoteChar(quoteChar).withEscapeChar(escapeChar).withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS).build();
    Map<En, List<String[]>> ret = new HashMap<>();
    for (String k : map.keySet()) {
        if (!omitCheck) {
            if (!sch.ens.contains(new En(k))) {
                throw new RuntimeException("Not an entity: " + k);
            }
        }
        File file = new File(map.get(k));
        FileReader fileReader = new FileReader(file);
        final CSVReader reader = new CSVReaderBuilder(fileReader).withCSVParser(parser).withFieldAsNull(CSVReaderNullFieldIndicator.EMPTY_SEPARATORS).build();
        List<String[]> rows = reader.readAll();
        fileReader.close();
        ret.put(new En(k), rows);
    }
    if (!omitCheck) {
        for (En en : sch.ens) {
            if (!ret.containsKey(en)) {
                ret.put(en, new LinkedList<>(Util.singList(Util.union(sch.attsFrom(en).stream().map(Object::toString).collect(Collectors.toList()), sch.fksFrom(en).stream().map(Object::toString).collect(Collectors.toList())).toArray(new String[0]))));
            }
        }
    }
    return ret;
}

Example 4 with CSVParserBuilder

use of com.opencsv.CSVParserBuilder in project metron by apache.

the class CSVConverter method initialize.

/**
 * Initializes the CSVConverter based on the provided config. The config should contain
 * an entry for {@code columns}, and can optionally contain a {@code separator}.
 *
 * @param config The configuration used for setup
 */
public void initialize(Map<String, Object> config) {
    if (config.containsKey(COLUMNS_KEY)) {
        columnMap = getColumnMap(config);
    } else {
        throw new IllegalStateException("CSVExtractor requires " + COLUMNS_KEY + " configuration");
    }
    char separator = ',';
    if (config.containsKey(SEPARATOR_KEY)) {
        separator = config.get(SEPARATOR_KEY).toString().charAt(0);
    }
    parser = new CSVParserBuilder().withSeparator(separator).build();
}

Example 5 with CSVParserBuilder

use of com.opencsv.CSVParserBuilder in project cu-kfs by CU-CommunityApps.

the class CuDelimitedFlatFileParser method buildFlatFileReader.

protected CSVReader buildFlatFileReader(Reader flatFileContent) {
    FlatFileSpecificationForProcessingPreSplitLines preSplitSpec = getFlatFileSpecificationForPreSplitLines();
    String delimiter = preSplitSpec.getDelimiter();
    if (StringUtils.length(delimiter) != 1) {
        throw new IllegalStateException("The flat file specification should have had a single-character delimiter, but instead had '" + delimiter + "'");
    }
    CSVParser parser = new CSVParserBuilder().withSeparator(delimiter.charAt(0)).build();
    return new CSVReaderBuilder(flatFileContent).withCSVParser(parser).build();
}