Examples with CsvReader org.openforis.commons.io.csv.CsvReader used on opensource projects

Example 1 with CsvReader

use of org.openforis.commons.io.csv.CsvReader in project collect by openforis.

the class DataRestoreSummaryTask method execute.

@Override
protected void execute() throws Throwable {
    if (fullSummary || dataSummaryFile == null) {
        List<Integer> idsToImport = extractEntryIdsToImport();
        for (Integer entryId : idsToImport) {
            if (!isRunning()) {
                break;
            }
            for (Step step : Step.values()) {
                if (!isRunning()) {
                    break;
                }
                createSummaryForEntry(entryId, step);
            }
        }
    } else {
        CsvReader dataSummaryCSVReader = null;
        try {
            dataSummaryCSVReader = new CsvReader(dataSummaryFile);
            dataSummaryCSVReader.readHeaders();
            CsvLine line = dataSummaryCSVReader.readNextLine();
            while (line != null) {
                Integer entryId = line.getValue("entry_id", Integer.class);
                String stepVal = line.getValue("step", String.class);
                Step step = Step.valueOf(stepVal);
                CollectRecordSummary recordSummary = new CollectRecordSummary();
                recordSummary.setStep(step);
                Integer rootEntityDefId = line.getValue("root_entity_id", Integer.class);
                recordSummary.setRootEntityDefinitionId(rootEntityDefId);
                Date creationDate = Dates.parseDateTime(line.getValue("created_on", String.class));
                Date modifiedDate = Dates.parseDateTime(line.getValue("last_modified", String.class));
                recordSummary.setCreationDate(creationDate);
                recordSummary.setModifiedDate(modifiedDate);
                StepSummary stepSummary = new StepSummary(step);
                List<String> rootEntityKeyValues = Arrays.asList(line.getValue("key1", String.class), line.getValue("key2", String.class), line.getValue("key3", String.class));
                stepSummary.setRootEntityKeyValues(rootEntityKeyValues);
                stepSummary.setCreationDate(creationDate);
                stepSummary.setModifiedDate(modifiedDate);
                recordSummary.addStepSummary(stepSummary);
                recordSummaryByEntryId.put(entryId, recordSummary);
                for (Step s : Step.values()) {
                    if (s.beforeEqual(step)) {
                        addStepPerEntry(entryId, s);
                        incrementTotalPerStep(step);
                        incrementProcessedItems();
                    }
                }
                CollectRecordSummary oldRecord = findAlreadyExistingRecordSummary(entryId, step, rootEntityDefId, rootEntityKeyValues);
                if (oldRecord != null) {
                    conflictingRecordByEntryId.put(entryId, oldRecord);
                }
                line = dataSummaryCSVReader.readNextLine();
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            IOUtils.closeQuietly(dataSummaryCSVReader);
        }
    }
}

Example 2 with CsvReader

use of org.openforis.commons.io.csv.CsvReader in project collect by openforis.

the class DataRestoreSummaryTask method extractEntryIdsToImport.

private List<Integer> extractEntryIdsToImport() {
    if (dataSummaryFile == null) {
        return recordProvider.findEntryIds();
    } else {
        CsvReader dataSummaryCSVReader = null;
        try {
            dataSummaryCSVReader = new CsvReader(dataSummaryFile);
            dataSummaryCSVReader.readHeaders();
            List<Integer> entryIds = new ArrayList<Integer>(dataSummaryCSVReader.size());
            CsvLine line = dataSummaryCSVReader.readNextLine();
            while (line != null) {
                Integer entryId = line.getValue("entry_id", Integer.class);
                entryIds.add(entryId);
                line = dataSummaryCSVReader.readNextLine();
            }
            return entryIds;
        } catch (Exception e) {
            throw new RuntimeException(e);
        } finally {
            IOUtils.closeQuietly(dataSummaryCSVReader);
        }
    }
}

Example 3 with CsvReader

use of org.openforis.commons.io.csv.CsvReader in project collect by openforis.

the class CollectEarthGridTemplateGenerator method validate.

public CSVFileValidationResult validate(File file, CollectSurvey survey, ValidationParameters validationParameters) {
    CsvReader csvReader = null;
    CSVFileValidationResult validationResults = null;
    List<CSVRowValidationResult> rowValidations = new ArrayList<CSVRowValidationResult>();
    try {
        csvReader = new CsvReader(file);
        boolean headersFound = false;
        if (csvReader.size() == 1) {
            headersFound = true;
        } else {
            try {
                csvReader.readHeaders();
                List<String> firstLineValues = csvReader.getColumnNames();
                headersFound = lineContainsHeaders(survey, firstLineValues);
                if (headersFound) {
                    validationResults = validateCSVHeaders(firstLineValues, survey);
                } else {
                    // Check that the number of columns coincide with the number of attributes expected
                    // Get the list of attribute types expected per row!
                    List<AttributeDefinition> attributesPerRow = getAttributesPerRow(survey);
                    // Validate that the number of columns in the CSV and the expected number of columns match!!!!
                    if (firstLineValues.size() != attributesPerRow.size()) {
                        // The expected number of columns and the actual columns do not fit!!
                        // Break the operation and return a validation error!
                        validationResults = new CSVFileValidationResult(ErrorType.INVALID_HEADERS, getExpectedHeaders(survey), firstLineValues);
                    }
                }
            } catch (Exception e) {
                // this may happen when there are duplicate values in the first row
                headersFound = false;
                csvReader.setHeadersRead(true);
            }
        }
        if (validationResults == null) {
            rowValidations.addAll(validateCsvRows(csvReader, survey, headersFound, validationParameters.isValidateOnlyFirstLines()));
            long linesRead = csvReader.getLinesRead();
            if (linesRead > CSV_LENGTH_ERROR) {
                validationResults = new CSVFileValidationResult(ErrorType.INVALID_NUMBER_OF_PLOTS_TOO_LARGE);
                validationResults.setNumberOfRows((int) linesRead);
            } else if (csvReader.getLinesRead() > CSV_LENGTH_WARNING) {
                validationResults = new CSVFileValidationResult(ErrorType.INVALID_NUMBER_OF_PLOTS_WARNING);
                validationResults.setNumberOfRows((int) linesRead);
            }
        }
    } catch (Exception e) {
        logger.error("Error reading CSV file ", e);
        validationResults = new CSVFileValidationResult(ErrorType.INVALID_FILE_TYPE);
    } finally {
        IOUtils.closeQuietly(csvReader);
    }
    if (validationResults == null) {
        validationResults = new CSVFileValidationResult();
    }
    validationResults.setRowValidations(rowValidations);
    return validationResults;
}