Examples with CsvReader com.csvreader.CsvReader used on opensource projects

Example 16 with CsvReader

use of com.csvreader.CsvReader in project dhis2-core by dhis2.

the class DefaultCsvImportService method fromCsv.

// -------------------------------------------------------------------------
// CsvImportService implementation
// -------------------------------------------------------------------------
@Override
public Metadata fromCsv(InputStream input, Class<? extends IdentifiableObject> clazz) throws IOException {
    CsvReader reader = new CsvReader(input, Charset.forName("UTF-8"));
    // Ignore first row
    reader.readRecord();
    Metadata metadata = new Metadata();
    if (DataElement.class.equals(clazz)) {
        metadata.setDataElements(dataElementsFromCsv(reader));
    } else if (DataElementGroup.class.equals(clazz)) {
        metadata.setDataElementGroups(dataElementGroupsFromCsv(reader));
    } else if (DataElementCategoryOption.class.equals(clazz)) {
        metadata.setCategoryOptions(categoryOptionsFromCsv(reader));
    } else if (CategoryOptionGroup.class.equals(clazz)) {
        metadata.setCategoryOptionGroups(categoryOptionGroupsFromCsv(reader));
    } else if (OrganisationUnit.class.equals(clazz)) {
        metadata.setOrganisationUnits(organisationUnitsFromCsv(reader));
    } else if (OrganisationUnitGroup.class.equals(clazz)) {
        metadata.setOrganisationUnitGroups(organisationUnitGroupsFromCsv(reader));
    } else if (ValidationRule.class.equals(clazz)) {
        metadata.setValidationRules(validationRulesFromCsv(reader));
    } else if (OptionSet.class.equals(clazz)) {
        setOptionSetsFromCsv(reader, metadata);
    }
    return metadata;
}

Example 17 with CsvReader

use of com.csvreader.CsvReader in project OpenTripPlanner by opentripplanner.

the class PointSet method fromCsv.

/**
 * Rather than trying to load anything any everything, we stick to a strict
 * format and rely on other tools to get the data into the correct format.
 * This includes column headers in the category:subcategory:attribute format
 * and coordinates in WGS84. Comment lines are allowed in these input files, and begin with a #.
 */
public static PointSet fromCsv(File filename) throws IOException {
    /* First, scan through the file to count lines and check for errors. */
    CsvReader reader = new CsvReader(filename.getAbsolutePath(), ',', Charset.forName("UTF8"));
    reader.readHeaders();
    int nCols = reader.getHeaderCount();
    while (reader.readRecord()) {
        if (reader.getColumnCount() != nCols) {
            LOG.error("CSV record {} has the wrong number of fields.", reader.getCurrentRecord());
            return null;
        }
    }
    // getCurrentRecord is zero-based and does not include headers or blank lines
    int nRecs = (int) reader.getCurrentRecord() + 1;
    reader.close();
    /* If we reached here, the file is entirely readable. Start over. */
    reader = new CsvReader(filename.getAbsolutePath(), ',', Charset.forName("UTF8"));
    PointSet ret = new PointSet(nRecs);
    reader.readHeaders();
    if (reader.getHeaderCount() != nCols) {
        LOG.error("Number of headers changed.");
        return null;
    }
    int latCol = -1;
    int lonCol = -1;
    // An array of property magnitudes corresponding to each column in the input.
    // Some of these will remain null (specifically, the lat and lon columns which do not contain magnitudes)
    int[][] properties = new int[nCols][ret.capacity];
    for (int c = 0; c < nCols; c++) {
        String header = reader.getHeader(c);
        if (header.equalsIgnoreCase("lat") || header.equalsIgnoreCase("latitude")) {
            latCol = c;
        } else if (header.equalsIgnoreCase("lon") || header.equalsIgnoreCase("longitude")) {
            lonCol = c;
        } else {
            ret.getOrCreatePropertyForId(header);
            properties[c] = ret.properties.get(header);
        }
    }
    if (latCol < 0 || lonCol < 0) {
        LOG.error("CSV file did not contain a latitude or longitude column.");
        throw new IOException();
    }
    ret.lats = new double[nRecs];
    ret.lons = new double[nRecs];
    while (reader.readRecord()) {
        int rec = (int) reader.getCurrentRecord();
        for (int c = 0; c < nCols; c++) {
            if (c == latCol || c == lonCol) {
                continue;
            }
            int[] prop = properties[c];
            int mag = Integer.parseInt(reader.get(c));
            prop[rec] = mag;
        }
        ret.lats[rec] = Double.parseDouble(reader.get(latCol));
        ret.lons[rec] = Double.parseDouble(reader.get(lonCol));
    }
    ret.capacity = nRecs;
    return ret;
}

Example 18 with CsvReader

use of com.csvreader.CsvReader in project components by Talend.

the class BulkResultSetTest method prepareSafetySwitchTest.

private int prepareSafetySwitchTest(boolean safetySwitchParameter, int columnLength) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter csvWriter = new CsvWriter(new BufferedOutputStream(out), ',', Charset.forName("UTF-8"));
    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < columnLength; i++) {
        sb.append("a");
    }
    String[] data = new String[] { "fieldValueA", "fieldValueB", sb.toString() };
    csvWriter.writeRecord(data);
    csvWriter.close();
    CsvReader csvReader = new CsvReader(new BufferedInputStream(new ByteArrayInputStream(out.toByteArray())), ',', Charset.forName("UTF-8"));
    csvReader.setSafetySwitch(safetySwitchParameter);
    BulkResultSet resultSet = new BulkResultSet(csvReader, Arrays.asList("fieldA", "fieldB", "fieldC"));
    BulkResult result = resultSet.next();
    return ((String) result.getValue("fieldC")).length();
}

Example 19 with CsvReader

use of com.csvreader.CsvReader in project ice by Netflix.

the class BillingFileProcessor method processBillingFile.

private void processBillingFile(String fileName, InputStream tempIn, boolean withTags) {
    CsvReader reader = new CsvReader(new InputStreamReader(tempIn), ',');
    long lineNumber = 0;
    List<String[]> delayedItems = Lists.newArrayList();
    try {
        reader.readRecord();
        String[] headers = reader.getValues();
        config.lineItemProcessor.initIndexes(config, withTags, headers);
        while (reader.readRecord()) {
            String[] items = reader.getValues();
            try {
                processOneLine(delayedItems, items);
            } catch (Exception e) {
                logger.error(StringUtils.join(items, ","), e);
            }
            lineNumber++;
            if (lineNumber % 500000 == 0) {
                logger.info("processed " + lineNumber + " lines...");
            }
        // if (lineNumber == 40000000) {//100000000      //
        // break;
        // }
        }
    } catch (IOException e) {
        logger.error("Error processing " + fileName + " at line " + lineNumber, e);
    } finally {
        try {
            reader.close();
        } catch (Exception e) {
            logger.error("Cannot close BufferedReader...", e);
        }
    }
    for (String[] items : delayedItems) {
        processOneLine(null, items);
    }
}

Example 20 with CsvReader

use of com.csvreader.CsvReader in project dhis2-core by dhis2.

the class DefaultCsvImportService method fromCsv.

// -------------------------------------------------------------------------
// CsvImportService implementation
// -------------------------------------------------------------------------
// TODO Add unit tests
@Override
public Metadata fromCsv(InputStream input, CsvImportOptions options) throws IOException {
    CsvReader reader = CsvUtils.getReader(input);
    // Disabled due to large geometry
    reader.setSafetySwitch(false);
    if (options.isFirstRowIsHeader()) {
        // Ignore first row
        reader.readRecord();
    }
    Metadata metadata = new Metadata();
    switch(options.getImportClass()) {
        case DATA_ELEMENT:
            metadata.setDataElements(dataElementsFromCsv(reader));
            break;
        case DATA_ELEMENT_GROUP:
            metadata.setDataElementGroups(dataElementGroupsFromCsv(reader));
            break;
        case DATA_ELEMENT_GROUP_MEMBERSHIP:
            metadata.setDataElementGroups(dataElementGroupMembersFromCsv(reader));
            break;
        case INDICATOR_GROUP_MEMBERSHIP:
            metadata.setIndicatorGroups(indicatorGroupMembersFromCsv(reader));
            break;
        case CATEGORY_OPTION:
            metadata.setCategoryOptions(categoryOptionsFromCsv(reader));
            break;
        case CATEGORY:
            metadata.setCategories(categoriesFromCsv(reader));
            break;
        case CATEGORY_COMBO:
            metadata.setCategoryCombos(categoryCombosFromCsv(reader));
            break;
        case CATEGORY_OPTION_GROUP:
            metadata.setCategoryOptionGroups(categoryOptionGroupsFromCsv(reader));
            break;
        case ORGANISATION_UNIT:
            metadata.setOrganisationUnits(orgUnitsFromCsv(reader));
            break;
        case ORGANISATION_UNIT_GROUP:
            metadata.setOrganisationUnitGroups(orgUnitGroupsFromCsv(reader));
            break;
        case ORGANISATION_UNIT_GROUP_MEMBERSHIP:
            metadata.setOrganisationUnitGroups(orgUnitGroupMembersFromCsv(reader));
            break;
        case VALIDATION_RULE:
            metadata.setValidationRules(validationRulesFromCsv(reader));
            break;
        case OPTION_SET:
            setOptionSetsFromCsv(reader, metadata);
            break;
        case OPTION_GROUP:
            setOptionGroupsFromCsv(reader, metadata);
            break;
        case OPTION_GROUP_SET:
            metadata.setOptionGroupSets(setOptionGroupSetFromCsv(reader));
            break;
        case OPTION_GROUP_SET_MEMBERSHIP:
            metadata.setOptionGroupSets(optionGroupSetMembersFromCsv(reader));
            break;
        default:
            break;
    }
    return metadata;
}