Examples with CsvReader com.csvreader.CsvReader used on opensource projects

Example 21 with CsvReader

use of com.csvreader.CsvReader in project dhis2-core by dhis2.

the class DefaultCsvImportService method setOptionGroupSetFromCsv.

/**
 * Option group set format:
 * <p>
 * <ul>
 * <li>option group set name</li>
 * <li>option group set uid</li>
 * <li>option group set code</li>
 * <li>option group set description</li>
 * <li>data dimension</li>
 * <li>option set uid</li>
 * <li>option set code</li>
 * </ul>
 */
private List<OptionGroupSet> setOptionGroupSetFromCsv(CsvReader reader) throws IOException {
    List<OptionGroupSet> optionGroupSets = new ArrayList<>();
    Map<String, OptionSet> mapOptionSet = new HashMap<>();
    while (reader.readRecord()) {
        String[] values = reader.getValues();
        if (values != null && values.length > 0) {
            OptionGroupSet optionGroupSet = new OptionGroupSet();
            optionGroupSet.setAutoFields();
            setIdentifiableObject(optionGroupSet, values);
            optionGroupSet.setDescription(getSafe(values, 4));
            optionGroupSet.setDataDimension(// boolean
            Boolean.parseBoolean(getSafe(values, 3, Boolean.FALSE.toString(), 40)));
            OptionSet optionSet = new OptionSet();
            optionSet.setUid(getSafe(values, 5, 11));
            optionSet.setCode(getSafe(values, 6, 50));
            if (optionSet.getUid() == null && optionSet.getCode() == null) {
                continue;
            }
            OptionSet persistedOptionSet = optionSet.getUid() != null ? mapOptionSet.computeIfAbsent(optionSet.getUid(), key -> optionService.getOptionSet(optionSet.getUid())) : mapOptionSet.computeIfAbsent(optionSet.getCode(), key -> optionService.getOptionSetByCode(optionSet.getCode()));
            if (persistedOptionSet == null) {
                continue;
            }
            optionGroupSet.setOptionSet(optionSet);
            optionGroupSets.add(optionGroupSet);
        }
    }
    return optionGroupSets;
}

Example 22 with CsvReader

use of com.csvreader.CsvReader in project dhis2-core by dhis2.

the class CsvUtils method readCsvAsList.

/**
 * Returns the CSV file represented by the given input stream as a list of
 * string arrays.
 *
 * @param in the {@link InputStream} representing the CSV file.
 * @param ignoreFirstRow whether to ignore the first row.
 * @return a list of string arrays.
 * @throws IOException
 */
public static List<String[]> readCsvAsList(InputStream in, boolean ignoreFirstRow) throws IOException {
    CsvReader reader = getReader(in);
    if (ignoreFirstRow) {
        reader.readRecord();
    }
    List<String[]> lines = new ArrayList<>();
    while (reader.readRecord()) {
        lines.add(reader.getValues());
    }
    return lines;
}

Example 23 with CsvReader

use of com.csvreader.CsvReader in project DataX by alibaba.

the class UnstructuredStorageReaderUtil method doReadFromStream.

public static void doReadFromStream(BufferedReader reader, String context, Configuration readerSliceConfig, RecordSender recordSender, TaskPluginCollector taskPluginCollector) {
    String encoding = readerSliceConfig.getString(Key.ENCODING, Constant.DEFAULT_ENCODING);
    Character fieldDelimiter = null;
    String delimiterInStr = readerSliceConfig.getString(Key.FIELD_DELIMITER);
    if (null != delimiterInStr && 1 != delimiterInStr.length()) {
        throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.ILLEGAL_VALUE, String.format("仅仅支持单字符切分, 您配置的切分为 : [%s]", delimiterInStr));
    }
    if (null == delimiterInStr) {
        LOG.warn(String.format("您没有配置列分隔符, 使用默认值[%s]", Constant.DEFAULT_FIELD_DELIMITER));
    }
    // warn: default value ',', fieldDelimiter could be \n(lineDelimiter)
    // for no fieldDelimiter
    fieldDelimiter = readerSliceConfig.getChar(Key.FIELD_DELIMITER, Constant.DEFAULT_FIELD_DELIMITER);
    Boolean skipHeader = readerSliceConfig.getBool(Key.SKIP_HEADER, Constant.DEFAULT_SKIP_HEADER);
    // warn: no default value '\N'
    String nullFormat = readerSliceConfig.getString(Key.NULL_FORMAT);
    // warn: Configuration -> List<ColumnEntry> for performance
    // List<Configuration> column = readerSliceConfig
    // .getListConfiguration(Key.COLUMN);
    List<ColumnEntry> column = UnstructuredStorageReaderUtil.getListColumnEntry(readerSliceConfig, Key.COLUMN);
    CsvReader csvReader = null;
    // every line logic
    try {
        // TODO lineDelimiter
        if (skipHeader) {
            String fetchLine = reader.readLine();
            LOG.info(String.format("Header line %s has been skiped.", fetchLine));
        }
        csvReader = new CsvReader(reader);
        csvReader.setDelimiter(fieldDelimiter);
        setCsvReaderConfig(csvReader);
        String[] parseRows;
        while ((parseRows = UnstructuredStorageReaderUtil.splitBufferedReader(csvReader)) != null) {
            UnstructuredStorageReaderUtil.transportOneRecord(recordSender, column, parseRows, nullFormat, taskPluginCollector);
        }
    } catch (UnsupportedEncodingException uee) {
        throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.OPEN_FILE_WITH_CHARSET_ERROR, String.format("不支持的编码格式 : [%s]", encoding), uee);
    } catch (FileNotFoundException fnfe) {
        throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.FILE_NOT_EXISTS, String.format("无法找到文件 : [%s]", context), fnfe);
    } catch (IOException ioe) {
        throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.READ_FILE_IO_ERROR, String.format("读取文件错误 : [%s]", context), ioe);
    } catch (Exception e) {
        throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.RUNTIME_EXCEPTION, String.format("运行时异常 : %s", e.getMessage()), e);
    } finally {
        csvReader.close();
        IOUtils.closeQuietly(reader);
    }
}

Example 24 with CsvReader

use of com.csvreader.CsvReader in project graphhopper by graphhopper.

the class FareTest method parseFares.

public static Map<String, Fare> parseFares(String feedId, String fareAttributes, String fareRules) {
    GTFSFeed feed = new GTFSFeed();
    feed.feedId = feedId;
    HashMap<String, Fare> fares = new HashMap<>();
    new FareAttribute.Loader(feed, fares) {

        void load(String input) {
            reader = new CsvReader(new StringReader(input));
            reader.setHeaders(new String[] { "fare_id", "price", "currency_type", "payment_method", "transfers", "transfer_duration" });
            try {
                while (reader.readRecord()) {
                    loadOneRow();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }.load(fareAttributes);
    new FareRule.Loader(feed, fares) {

        void load(String input) {
            reader = new CsvReader(new StringReader(input));
            reader.setHeaders(new String[] { "fare_id", "route_id", "origin_id", "destination_id", "contains_id" });
            try {
                while (reader.readRecord()) {
                    loadOneRow();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }.load(fareRules);
    return fares;
}