Examples with CSVReader au.com.bytecode.opencsv.CSVReader used on opensource projects

Example 41 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project hale by halestudio.

the class CSVUtil method readFirst.

/**
 * Reads only the first line of a given CSV file
 *
 * @param provider provider to get the parameters from
 * @return a reader containing the first line of the CSV file
 * @throws IOException if an I/O operation fails
 */
public static CSVReader readFirst(ImportProvider provider) throws IOException {
    Reader streamReader = new BufferedReader(new InputStreamReader(provider.getSource().getInput(), provider.getCharset()));
    CSVReader reader = new CSVReader(streamReader, getSep(provider), getQuote(provider), getEscape(provider));
    return reader;
}

Example 42 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project collect by opendatakit.

the class ExternalSQLiteOpenHelper method onCreateNamed.

private void onCreateNamed(SQLiteDatabase db, String tableName) throws Exception {
    Timber.w("Reading data from '%s", dataSetFile.toString());
    onProgress(Collect.getInstance().getString(R.string.ext_import_progress_message, dataSetFile.getName(), ""));
    CSVReader reader = null;
    try {
        reader = new CSVReader(new InputStreamReader(new FileInputStream(dataSetFile), "UTF-8"), DELIMITING_CHAR, QUOTE_CHAR, ESCAPE_CHAR);
        String[] headerRow = reader.readNext();
        if (!ExternalDataUtil.containsAnyData(headerRow)) {
            throw new ExternalDataException(Collect.getInstance().getString(R.string.ext_file_no_data_error));
        }
        List<String> conflictingColumns = ExternalDataUtil.findMatchingColumnsAfterSafeningNames(headerRow);
        if (conflictingColumns != null && conflictingColumns.size() > 0) {
            // so the create table query will fail with "duplicate column" error.
            throw new ExternalDataException(Collect.getInstance().getString(R.string.ext_conflicting_columns_error, conflictingColumns));
        }
        Map<String, String> columnNamesCache = new HashMap<String, String>();
        StringBuilder sb = new StringBuilder();
        boolean sortColumnAlreadyPresent = false;
        sb.append("CREATE TABLE ");
        sb.append(tableName);
        sb.append(" ( ");
        for (int i = 0; i < headerRow.length; i++) {
            String columnName = headerRow[i].trim();
            if (columnName.length() == 0) {
                continue;
            }
            if (i != 0) {
                sb.append(", ");
            }
            String safeColumnName = ExternalDataUtil.toSafeColumnName(columnName, columnNamesCache);
            if (safeColumnName.equals(ExternalDataUtil.SORT_COLUMN_NAME)) {
                sortColumnAlreadyPresent = true;
                sb.append(safeColumnName).append(" real ");
            } else {
                sb.append(safeColumnName).append(" text collate nocase ");
            }
        }
        if (!sortColumnAlreadyPresent) {
            sb.append(", ");
            sb.append(ExternalDataUtil.SORT_COLUMN_NAME).append(" real ");
        }
        sb.append(" );");
        String sql = sb.toString();
        Timber.w("Creating database for %s with query: %s", dataSetFile, sql);
        db.execSQL(sql);
        // create the indexes.
        // save the sql for later because inserts will be much faster if we don't have
        // indexes already.
        List<String> createIndexesCommands = new ArrayList<String>();
        for (String header : headerRow) {
            if (header.endsWith("_key")) {
                String indexSQL = "CREATE INDEX " + header + "_idx ON " + tableName + " (" + ExternalDataUtil.toSafeColumnName(header, columnNamesCache) + ");";
                createIndexesCommands.add(indexSQL);
                Timber.w("Will create an index on %s later.", header);
            }
        }
        // populate the database
        String[] row = reader.readNext();
        int rowCount = 0;
        while (row != null && !formLoaderTask.isCancelled()) {
            // SCTO-894 - first we should make sure that this is not an empty line
            if (!ExternalDataUtil.containsAnyData(row)) {
                // yes, that is an empty row, ignore it
                row = reader.readNext();
                continue;
            }
            // we will just fill up the rest with empty strings
            if (row.length < headerRow.length) {
                row = ExternalDataUtil.fillUpNullValues(row, headerRow);
            }
            ContentValues values = new ContentValues();
            if (!sortColumnAlreadyPresent) {
                values.put(ExternalDataUtil.SORT_COLUMN_NAME, rowCount + 1);
            }
            for (int i = 0; i < row.length && i < headerRow.length; i++) {
                String columnName = headerRow[i].trim();
                String columnValue = row[i];
                if (columnName.length() == 0) {
                    continue;
                }
                String safeColumnName = ExternalDataUtil.toSafeColumnName(columnName, columnNamesCache);
                if (safeColumnName.equals(ExternalDataUtil.SORT_COLUMN_NAME)) {
                    try {
                        values.put(safeColumnName, Double.parseDouble(columnValue));
                    } catch (NumberFormatException e) {
                        throw new ExternalDataException(Collect.getInstance().getString(R.string.ext_sortBy_numeric_error, columnValue));
                    }
                } else {
                    values.put(safeColumnName, columnValue);
                }
            }
            db.insertOrThrow(tableName, null, values);
            row = reader.readNext();
            rowCount++;
            if (rowCount % 100 == 0) {
                onProgress(Collect.getInstance().getString(R.string.ext_import_progress_message, dataSetFile.getName(), " (" + rowCount + " records so far)"));
            }
        }
        if (formLoaderTask.isCancelled()) {
            Timber.w("User canceled reading data from %s", dataSetFile.toString());
            onProgress(Collect.getInstance().getString(R.string.ext_import_cancelled_message));
        } else {
            onProgress(Collect.getInstance().getString(R.string.ext_import_finalizing_message));
            // now create the indexes
            for (String createIndexCommand : createIndexesCommands) {
                Timber.w(createIndexCommand);
                db.execSQL(createIndexCommand);
            }
            Timber.w("Read all data from %s", dataSetFile.toString());
            onProgress(Collect.getInstance().getString(R.string.ext_import_completed_message));
        }
    } finally {
        if (reader != null) {
            try {
                reader.close();
            } catch (IOException e) {
                Timber.e(e);
            }
        }
    }
}

Example 43 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project spatial-portal by AtlasOfLivingAustralia.

the class SamplingDownloadUtil method downloadSecond.

public static byte[] downloadSecond(MapComposer mc, Query downloadSecondQuery, String[] downloadSecondLayers, String[] downloadSecondLayersDN) {
    LOGGER.debug("attempting to sample biocache records with analysis layers: " + downloadSecondQuery);
    if (downloadSecondQuery != null) {
        try {
            List<QueryField> fields = new ArrayList<QueryField>();
            fields.add(new QueryField(downloadSecondQuery.getRecordIdFieldName()));
            fields.add(new QueryField(downloadSecondQuery.getRecordLongitudeFieldName()));
            fields.add(new QueryField(downloadSecondQuery.getRecordLatitudeFieldName()));
            String results = downloadSecondQuery.sample(fields);
            if (results != null) {
                CSVReader csvreader = new CSVReader(new StringReader(results));
                List<String[]> csv = csvreader.readAll();
                csvreader.close();
                int longitudeColumn = Util.findInArray(downloadSecondQuery.getRecordLongitudeFieldDisplayName(), csv.get(0));
                int latitudeColumn = Util.findInArray(downloadSecondQuery.getRecordLatitudeFieldDisplayName(), csv.get(0));
                int idColumn = Util.findInArray(downloadSecondQuery.getRecordIdFieldDisplayName(), csv.get(0));
                double[] points = new double[(csv.size() - 1) * 2];
                String[] ids = new String[csv.size() - 1];
                int pos = 0;
                for (int i = 1; i < csv.size(); i++) {
                    try {
                        points[pos] = Double.parseDouble(csv.get(i)[longitudeColumn]);
                        points[pos + 1] = Double.parseDouble(csv.get(i)[latitudeColumn]);
                    } catch (Exception e) {
                        points[pos] = Double.NaN;
                        points[pos + 1] = Double.NaN;
                    }
                    ids[pos / 2] = csv.get(i)[idColumn];
                    pos += 2;
                }
                double[][] p = new double[points.length / 2][2];
                for (int i = 0; i < points.length; i += 2) {
                    p[i / 2][0] = points[i];
                    p[i / 2][1] = points[i + 1];
                }
                List<String> layers = new ArrayList<String>();
                StringBuilder sb = new StringBuilder();
                sb.append("id,longitude,latitude");
                for (int i = 0; i < downloadSecondLayers.length; i++) {
                    String layer = downloadSecondLayers[i];
                    sb.append(",");
                    String name = downloadSecondLayersDN[i];
                    sb.append(name);
                    layers.add(CommonData.getLayerFacetName(layer));
                }
                List<String[]> sample = Sampling.sampling(layers, p);
                if (!sample.isEmpty()) {
                    for (int j = 0; j < sample.get(0).length; j++) {
                        sb.append("\n");
                        sb.append(ids[j]).append(",").append(p[j][0]).append(",").append(p[j][1]);
                        for (int i = 0; i < sample.size(); i++) {
                            sb.append(",").append(sample.get(i)[j]);
                        }
                    }
                }
                ByteArrayOutputStream baos = new ByteArrayOutputStream();
                ZipOutputStream zos = new ZipOutputStream(baos);
                ZipEntry anEntry = new ZipEntry("analysis_output_intersect.csv");
                zos.putNextEntry(anEntry);
                zos.write(sb.toString().getBytes());
                zos.close();
                return baos.toByteArray();
            }
        } catch (Exception e) {
            LOGGER.error("error downloading samping records", e);
        }
    }
    return null;
}

Example 44 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project cytoscape-impl by cytoscape.

the class DefaultAttributeTableReader method readTable.

/**
 * Read table from the data source.
 */
@Override
public void readTable(CyTable table) throws IOException {
    try {
        BufferedReader bufRd = null;
        if (is == null)
            is = URLUtil.getInputStream(source);
        try {
            // This data is shared by both the OpenCSV and the old method of reading files.
            int lineCount = 0;
            bufRd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8").newDecoder()));
            /*
				 * Read & extract one line at a time. The line can be Tab delimited,
				 */
            final String delimiter = mapping.getDelimiterRegEx();
            // If the delimiter contains a comma, treat the file as a CSV file.
            if (delimiter.contains(TextDelimiter.COMMA.getDelimiter()) && mapping.getDelimiters().size() == 1) {
                // Use OpenCSV.. New method...
                CSVReader reader = new CSVReader(bufRd);
                // Note that rowData is roughly equivalent to "parts" in the old code.
                String[] rowData;
                while ((rowData = reader.readNext()) != null) {
                    // If key dos not exists, ignore the line.
                    if (lineCount >= startLineNumber && rowData.length >= mapping.getKeyIndex() + 1) {
                        try {
                            parser.parseAll(table, rowData);
                        } catch (Exception ex) {
                            logger.warn("Couldn't parse row from OpenCSV: " + lineCount);
                        }
                        globalCounter++;
                    }
                    lineCount++;
                }
                try {
                    reader.close();
                } catch (Exception e) {
                }
            } else {
                // Use the "old" method for splitting the lines.
                String line;
                String[] parts = null;
                while ((line = bufRd.readLine()) != null) {
                    /*
						 * Ignore Empty & Commnet lines.
						 */
                    if ((commentChar != null) && line.startsWith(commentChar)) {
                    // Do nothing
                    } else if ((lineCount >= startLineNumber) && (line.trim().length() > 0)) {
                        parts = line.split(delimiter);
                        // If key does not exists, ignore the line.
                        if (parts.length >= mapping.getKeyIndex() + 1) {
                            try {
                                parser.parseAll(table, parts);
                            } catch (Exception ex) {
                                logger.warn("Couldn't parse row: " + lineCount);
                            }
                            globalCounter++;
                        }
                    }
                    lineCount++;
                }
            }
        } catch (MalformedInputException mie) {
            throw new IOException("Unable to import table: illegal character encoding in input");
        } finally {
            if (bufRd != null)
                bufRd.close();
        }
    } finally {
        if (is != null)
            is.close();
    }
}

Example 45 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project molgenis by molgenis.

the class HPORepository method getEntitiesByGeneSymbol.

private Map<String, List<Entity>> getEntitiesByGeneSymbol() {
    if (entitiesByGeneSymbol == null) {
        entitiesByGeneSymbol = new LinkedHashMap<>();
        try (CSVReader csvReader = new CSVReader(new InputStreamReader(new FileInputStream(file), UTF_8), '\t', CSVParser.DEFAULT_QUOTE_CHARACTER, 1)) {
            String[] values = csvReader.readNext();
            while (values != null) {
                String geneSymbol = values[1];
                Entity entity = new DynamicEntity(getEntityType());
                entity.set(HPO_DISEASE_ID_COL_NAME, values[0]);
                entity.set(HPO_GENE_SYMBOL_COL_NAME, geneSymbol);
                entity.set(HPO_ID_COL_NAME, values[3]);
                entity.set(HPO_TERM_COL_NAME, values[4]);
                List<Entity> entities = entitiesByGeneSymbol.computeIfAbsent(geneSymbol, k -> new ArrayList<>());
                entities.add(entity);
                values = csvReader.readNext();
            }
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }
    }
    return entitiesByGeneSymbol;
}