Examples with CSVReader au.com.bytecode.opencsv.CSVReader used on opensource projects

Example 71 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project dbeaver by serge-rider.

the class DataImporterCSV method runImport.

@Override
public void runImport(@NotNull DBRProgressMonitor monitor, @NotNull DBPDataSource streamDataSource, @NotNull InputStream inputStream, @NotNull IDataTransferConsumer consumer) throws DBException {
    IStreamDataImporterSite site = getSite();
    StreamEntityMapping entityMapping = site.getSourceObject();
    Map<String, Object> properties = site.getProcessorProperties();
    HeaderPosition headerPosition = getHeaderPosition(properties);
    boolean emptyStringNull = CommonUtils.getBoolean(properties.get(PROP_EMPTY_STRING_NULL), false);
    String nullValueMark = CommonUtils.toString(properties.get(PROP_NULL_STRING));
    DBCExecutionContext context = streamDataSource.getDefaultInstance().getDefaultContext(monitor, false);
    try (DBCSession producerSession = context.openSession(monitor, DBCExecutionPurpose.UTIL, "Transfer stream data")) {
        LocalStatement localStatement = new LocalStatement(producerSession, "SELECT * FROM Stream");
        StreamTransferResultSet resultSet = new StreamTransferResultSet(producerSession, localStatement, entityMapping);
        consumer.fetchStart(producerSession, resultSet, -1, -1);
        applyTransformHints(resultSet, consumer, properties, PROP_TIMESTAMP_FORMAT, PROP_TIMESTAMP_ZONE);
        try (Reader reader = openStreamReader(inputStream, properties)) {
            try (CSVReader csvReader = openCSVReader(reader, properties)) {
                int maxRows = site.getSettings().getMaxRows();
                int targetAttrSize = entityMapping.getStreamColumns().size();
                boolean headerRead = false;
                for (int lineNum = 0; ; ) {
                    if (monitor.isCanceled()) {
                        break;
                    }
                    String[] line = csvReader.readNext();
                    if (line == null) {
                        break;
                    }
                    if (line.length == 0) {
                        continue;
                    }
                    if (headerPosition != HeaderPosition.none && !headerRead) {
                        // First line is a header
                        headerRead = true;
                        continue;
                    }
                    if (maxRows > 0 && lineNum >= maxRows) {
                        break;
                    }
                    if (line.length < targetAttrSize) {
                        // Stream row may be shorter than header
                        String[] newLine = new String[targetAttrSize];
                        System.arraycopy(line, 0, newLine, 0, line.length);
                        for (int i = line.length; i < targetAttrSize; i++) {
                            newLine[i] = null;
                        }
                        line = newLine;
                    }
                    if (emptyStringNull) {
                        for (int i = 0; i < line.length; i++) {
                            if ("".equals(line[i])) {
                                line[i] = null;
                            }
                        }
                    }
                    if (!CommonUtils.isEmpty(nullValueMark)) {
                        for (int i = 0; i < line.length; i++) {
                            if (nullValueMark.equals(line[i])) {
                                line[i] = null;
                            }
                        }
                    }
                    resultSet.setStreamRow(line);
                    consumer.fetchRow(producerSession, resultSet);
                    lineNum++;
                    if (lineNum % 1000 == 0) {
                        monitor.subTask(String.valueOf(lineNum) + " rows processed");
                    }
                }
            }
        } catch (IOException e) {
            throw new DBException("IO error reading CSV", e);
        } finally {
            try {
                consumer.fetchEnd(producerSession, resultSet);
            } finally {
                consumer.close();
            }
        }
    }
}

Example 72 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project dbeaver by serge-rider.

the class DataImporterCSV method readColumnsInfo.

@NotNull
@Override
public List<StreamDataImporterColumnInfo> readColumnsInfo(StreamEntityMapping entityMapping, @NotNull InputStream inputStream) throws DBException {
    List<StreamDataImporterColumnInfo> columnsInfo = new ArrayList<>();
    Map<String, Object> processorProperties = getSite().getProcessorProperties();
    HeaderPosition headerPosition = getHeaderPosition(processorProperties);
    try (Reader reader = openStreamReader(inputStream, processorProperties)) {
        try (CSVReader csvReader = openCSVReader(reader, processorProperties)) {
            String[] header = getNextLine(csvReader);
            if (header == null) {
                return columnsInfo;
            }
            for (int i = 0; i < header.length; i++) {
                String column = null;
                if (headerPosition == HeaderPosition.top) {
                    column = DBUtils.getUnQuotedIdentifier(entityMapping.getDataSource(), header[i]);
                }
                if (CommonUtils.isEmptyTrimmed(column)) {
                    column = "Column" + (i + 1);
                }
                StreamDataImporterColumnInfo columnInfo = new StreamDataImporterColumnInfo(entityMapping, i, column, null, MAX_COLUMN_LENGTH, DBPDataKind.UNKNOWN);
                columnInfo.setMappingMetadataPresent(headerPosition != HeaderPosition.none);
                columnsInfo.add(columnInfo);
            }
            for (int sample = 0; sample < MAX_DATA_TYPE_SAMPLES; sample++) {
                String[] line;
                if (sample == 0 && headerPosition == HeaderPosition.none) {
                    // Include first line (header that does not exist) for sampling
                    line = header;
                } else {
                    line = getNextLine(csvReader);
                    if (line == null) {
                        break;
                    }
                }
                for (int i = 0; i < Math.min(line.length, header.length); i++) {
                    Pair<DBPDataKind, String> dataType = getDataType(line[i]);
                    StreamDataImporterColumnInfo columnInfo = columnsInfo.get(i);
                    switch(dataType.getFirst()) {
                        case STRING:
                            columnInfo.setDataKind(dataType.getFirst());
                            columnInfo.setTypeName(dataType.getSecond());
                            break;
                        case NUMERIC:
                        case BOOLEAN:
                            if (columnInfo.getDataKind() == DBPDataKind.UNKNOWN) {
                                columnInfo.setDataKind(dataType.getFirst());
                                columnInfo.setTypeName(dataType.getSecond());
                            }
                            break;
                    }
                }
            }
            for (StreamDataImporterColumnInfo columnInfo : columnsInfo) {
                if (columnInfo.getDataKind() == DBPDataKind.UNKNOWN) {
                    log.warn("Cannot guess data type for column '" + columnInfo.getName() + "', defaulting to VARCHAR");
                    columnInfo.setDataKind(DBPDataKind.STRING);
                    columnInfo.setTypeName("VARCHAR");
                }
            }
        }
    } catch (IOException e) {
        throw new DBException("IO error reading CSV", e);
    }
    return columnsInfo;
}

Example 73 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project the-app by devops-dojo.

the class AbstractCsvReader method parseCsv.

public List<T> parseCsv() throws IOException {
    ClassPathResource classPathResource = new ClassPathResource(getClassPathFilePath(), this.getClass().getClassLoader());
    InputStreamReader ioReader = new InputStreamReader(classPathResource.getInputStream(), "UTF-8");
    CSVReader reader = new CSVReader(ioReader, ';');
    ColumnPositionMappingStrategy<T> strat = new ColumnPositionMappingStrategy<>();
    strat.setType(getDestinationClass());
    strat.setColumnMapping(getColumnMapping());
    CsvToBean<T> csv = getParser();
    return csv.parse(strat, reader);
}

Example 74 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project ma-core-public by infiniteautomation.

the class CsvRowMessageConverter method readInternal.

/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * org.springframework.http.converter.AbstractHttpMessageConverter#readInternal
	 * (java.lang.Class, org.springframework.http.HttpInputMessage)
	 */
@SuppressWarnings({ "rawtypes" })
@Override
protected AbstractVoModel<?> readInternal(Class<? extends AbstractRestModel<?>> clazz, HttpInputMessage inputMessage) throws IOException, HttpMessageNotReadableException {
    CSVPojoReader in = new CSVPojoReader(new CSVReader(new InputStreamReader(inputMessage.getBody(), Common.UTF8_CS), separator, quote));
    AbstractVoModel<?> record = (AbstractVoModel<?>) in.readNext();
    in.close();
    return record;
}

Example 75 with CSVReader

use of au.com.bytecode.opencsv.CSVReader in project calcite by apache.

the class CsvEnumerator method deduceRowType.

/**
 * Deduces the names and types of a table's columns by reading the first line
 * of a CSV file.
 */
static RelDataType deduceRowType(JavaTypeFactory typeFactory, Source source, List<CsvFieldType> fieldTypes, Boolean stream) {
    final List<RelDataType> types = new ArrayList<>();
    final List<String> names = new ArrayList<>();
    CSVReader reader = null;
    if (stream) {
        names.add(CsvSchemaFactory.ROWTIME_COLUMN_NAME);
        types.add(typeFactory.createSqlType(SqlTypeName.TIMESTAMP));
    }
    try {
        reader = openCsv(source);
        String[] strings = reader.readNext();
        if (strings == null) {
            strings = new String[] { "EmptyFileHasNoColumns:boolean" };
        }
        for (String string : strings) {
            final String name;
            final CsvFieldType fieldType;
            final int colon = string.indexOf(':');
            if (colon >= 0) {
                name = string.substring(0, colon);
                String typeString = string.substring(colon + 1);
                fieldType = CsvFieldType.of(typeString);
                if (fieldType == null) {
                    System.out.println("WARNING: Found unknown type: " + typeString + " in file: " + source.path() + " for column: " + name + ". Will assume the type of column is string");
                }
            } else {
                name = string;
                fieldType = null;
            }
            final RelDataType type;
            if (fieldType == null) {
                type = typeFactory.createSqlType(SqlTypeName.VARCHAR);
            } else {
                type = fieldType.toType(typeFactory);
            }
            names.add(name);
            types.add(type);
            if (fieldTypes != null) {
                fieldTypes.add(fieldType);
            }
        }
    } catch (IOException e) {
    // ignore
    } finally {
        if (reader != null) {
            try {
                reader.close();
            } catch (IOException e) {
            // ignore
            }
        }
    }
    if (names.isEmpty()) {
        names.add("line");
        types.add(typeFactory.createSqlType(SqlTypeName.VARCHAR));
    }
    return typeFactory.createStructType(Pair.zip(names, types));
}