Examples with CSVRecord org.apache.commons.csv.CSVRecord used on opensource projects

Example 31 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project sw360portal by sw360.

the class ConvertRecord method convertRisks.

public static List<Risk> convertRisks(List<CSVRecord> records, Map<Integer, RiskCategory> categories) {
    List<Risk> list = new ArrayList<>(records.size());
    for (CSVRecord record : records) {
        if (record.size() < 3)
            break;
        int id = Integer.parseInt(record.get(0));
        int catId = Integer.parseInt(record.get(1));
        String text = record.get(2);
        Risk risk = new Risk().setRiskId(id).setText(text);
        risk.setCategory(categories.get(catId));
        list.add(risk);
    }
    return list;
}

Example 32 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project sw360portal by sw360.

the class ComponentAndAttachmentAwareDBTest method getCompCSVRecordsFromTestFile.

protected static FluentIterable<ComponentCSVRecord> getCompCSVRecordsFromTestFile(String fileName) throws IOException {
    InputStream testStream = spy(ComponentImportUtilsTest.class.getResourceAsStream(fileName));
    List<CSVRecord> testRecords = ImportCSV.readAsCSVRecords(testStream);
    verify(testStream).close();
    return convertCSVRecordsToCompCSVRecords(testRecords);
}

Example 33 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project solr-cmd-utils by tblsoft.

the class CSVReader method read.

@Override
public void read() {
    String absoluteFilename;
    boolean addMeta = false;
    try {
        String charset = getProperty("charset", StandardCharsets.UTF_8.name());
        String filename = getProperty("filename", null);
        absoluteFilename = IOUtils.getAbsoluteFile(getBaseDir(), filename);
        addMeta = getPropertyAsBoolean("addMeta", false);
        Long maxRows = getPropertyAsInteger("maxRows", Long.MAX_VALUE);
        String delimiter = getProperty("delimiter", ",");
        String arrayDelimiter = getProperty("arrayDelimiter", null);
        String[] headers = getPropertyAsArray("headers", null);
        InputStream in = IOUtils.getInputStream(absoluteFilename);
        java.io.Reader reader = new InputStreamReader(in, charset);
        CSVFormat format = CSVFormat.RFC4180;
        if (headers == null) {
            format = format.withHeader();
        } else {
            format = format.withHeader(headers);
        }
        format = format.withDelimiter(delimiter.charAt(0));
        CSVParser parser = format.parse(reader);
        Iterator<CSVRecord> csvIterator = parser.iterator();
        long rowNumber = 0;
        while (csvIterator.hasNext()) {
            if (rowNumber >= maxRows) {
                break;
            }
            rowNumber++;
            CSVRecord record = csvIterator.next();
            Map<String, Integer> header = parser.getHeaderMap();
            Document document = new Document();
            for (Map.Entry<String, Integer> entry : header.entrySet()) {
                String key = entry.getKey();
                try {
                    String value = record.get(key);
                    if (StringUtils.isEmpty(arrayDelimiter)) {
                        document.addField(key, value);
                    } else {
                        List<String> valueList = new ArrayList<String>();
                        String[] values = value.split(arrayDelimiter);
                        if (values.length > 0) {
                            for (String val : values) {
                                if (StringUtils.isNotEmpty(val)) {
                                    valueList.add(val);
                                }
                            }
                        }
                        document.setField(key, valueList);
                    }
                } catch (IllegalArgumentException e) {
                }
            }
            if (addMeta) {
                document.addField("rowNumber", String.valueOf(rowNumber));
                document.addField("fileName", absoluteFilename);
            }
            executer.document(document);
        }
        // executer.end();
        in.close();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 34 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project solr-cmd-utils by tblsoft.

the class WhitelistTopicTermsFilter method init.

@Override
public void init() {
    fieldTopic = getProperty("fieldTopic", null);
    fieldValue = getProperty("fieldValue", null);
    override = getPropertyAsBoolean("override", true);
    arrayDelimiter = getProperty("arrayDelimiter", ";");
    topicValues = new HashMap<String, HashMap<String, Document>>();
    topicsOverriden = new HashMap<String, HashMap<String, Boolean>>();
    InputStream in = null;
    try {
        String filename = getProperty("filename", null);
        String absoluteFilename = IOUtils.getAbsoluteFile(getBaseDir(), filename);
        in = IOUtils.getInputStream(absoluteFilename);
        java.io.Reader reader = new InputStreamReader(in, StandardCharsets.UTF_8.name());
        CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(',');
        CSVParser parser = format.parse(reader);
        Iterator<CSVRecord> csvIterator = parser.iterator();
        while (csvIterator.hasNext()) {
            CSVRecord record = csvIterator.next();
            Map<String, Integer> header = parser.getHeaderMap();
            Document document = new Document();
            for (Map.Entry<String, Integer> entry : header.entrySet()) {
                String key = entry.getKey();
                try {
                    String[] values = record.get(key).split(arrayDelimiter);
                    document.addField(key, Arrays.asList(values));
                } catch (IllegalArgumentException ignored) {
                }
            }
            String topic = record.get(header.get(fieldTopic));
            String value = record.get(header.get(fieldValue));
            if (!topicValues.containsKey(topic)) {
                topicValues.put(topic, new HashMap<String, Document>());
                topicsOverriden.put(topic, new HashMap<String, Boolean>());
            }
            topicValues.get(topic).put(value, document);
            topicsOverriden.get(topic).put(value, false);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (IOException ignored) {
            }
        }
    }
    super.init();
}

Example 35 with CSVRecord

use of org.apache.commons.csv.CSVRecord in project solr-cmd-utils by tblsoft.

the class BaseBlacklistFilter method init.

@Override
public void init() {
    topicValues = new HashMap<String, HashSet<String>>();
    InputStream in = null;
    try {
        String filename = getProperty("filename", null);
        String absoluteFilename = IOUtils.getAbsoluteFile(getBaseDir(), filename);
        in = IOUtils.getInputStream(absoluteFilename);
        java.io.Reader reader = new InputStreamReader(in, StandardCharsets.UTF_8.name());
        CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(',');
        CSVParser parser = format.parse(reader);
        Iterator<CSVRecord> csvIterator = parser.iterator();
        while (csvIterator.hasNext()) {
            CSVRecord record = csvIterator.next();
            String topic = record.get(0);
            String value = record.get(1);
            if (!topicValues.containsKey(topic)) {
                topicValues.put(topic, new HashSet<String>());
            }
            topicValues.get(topic).add(value);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (IOException ignored) {
            }
        }
    }
    fieldTopic = getProperty("fieldTopic", null);
    fieldValue = getProperty("fieldValue", null);
    super.init();
}