Examples with ParsingError org.openforis.collect.io.metadata.parsing.ParsingError used on opensource projects

Example 21 with ParsingError

use of org.openforis.collect.io.metadata.parsing.ParsingError in project collect by openforis.

the class CSVDataImportProcess method validateRecordKey.

private boolean validateRecordKey(DataLine line) {
    long currentRowNumber = line.getLineNumber();
    EntityDefinition parentEntityDefn = getParentEntityDefinition();
    EntityDefinition rootEntityDefn = parentEntityDefn.getRootEntity();
    Value[] recordKeyValues = line.getRecordKeyValues(rootEntityDefn);
    String[] recordKeyStringValues = Values.toStringValues(recordKeyValues);
    RecordFilter filter = new RecordFilter(survey);
    filter.setRootEntityId(rootEntityDefn.getId());
    filter.setKeyValues(recordKeyStringValues);
    List<CollectRecordSummary> recordSummaries = recordManager.loadSummaries(filter);
    String[] recordKeyColumnNames = DataCSVReader.getKeyAttributeColumnNames(parentEntityDefn, rootEntityDefn.getKeyAttributeDefinitions());
    String errorMessageKey = null;
    if (settings.isInsertNewRecords()) {
        if (!recordSummaries.isEmpty()) {
            errorMessageKey = ONLY_NEW_RECORDS_ALLOWED_MESSAGE_KEY;
        }
    } else if (recordSummaries.isEmpty() && settings.isReportNoRecordFoundErrors()) {
        errorMessageKey = NO_RECORD_FOUND_ERROR_MESSAGE_KEY;
    } else if (recordSummaries.size() > 1) {
        errorMessageKey = MULTIPLE_RECORDS_FOUND_ERROR_MESSAGE_KEY;
    }
    if (errorMessageKey == null) {
        return true;
    } else {
        ParsingError parsingError = new ParsingError(ErrorType.INVALID_VALUE, currentRowNumber, recordKeyColumnNames, errorMessageKey);
        parsingError.setMessageArgs(new String[] { StringUtils.join(recordKeyStringValues) });
        status.addParsingError(currentRowNumber, parsingError);
        return false;
    }
}

Example 22 with ParsingError

use of org.openforis.collect.io.metadata.parsing.ParsingError in project collect by openforis.

the class CSVDataImportProcess method processLine.

private void processLine(DataLine line) throws RecordPersistenceException {
    if (!validateRecordKey(line)) {
        return;
    }
    if (settings.isInsertNewRecords()) {
        // create new record
        EntityDefinition rootEntityDefn = survey.getSchema().getRootEntityDefinition(parentEntityDefinitionId);
        CollectRecord record = recordManager.instantiateRecord(survey, rootEntityDefn.getName(), adminUser, settings.getNewRecordVersionName(), Step.ENTRY);
        NodeChangeSet changes = recordManager.initializeRecord(record);
        if (nodeChangeBatchProcessor != null) {
            nodeChangeBatchProcessor.add(changes, adminUser.getUsername());
        }
        setRecordKeys(line, record);
        setValuesInRecord(line, record, Step.ENTRY);
        insertRecord(record);
    } else {
        CollectRecordSummary recordSummary = loadRecordSummaryIfAny(line);
        if (recordSummary != null) {
            Step originalRecordStep = recordSummary.getStep();
            if (step == null) {
                // set values in each step data
                for (Step currentStep : Step.values()) {
                    if (currentStep.beforeEqual(originalRecordStep)) {
                        CollectRecord record = loadRecord(recordSummary.getId(), currentStep);
                        setValuesInRecord(line, record, currentStep);
                        // always save record when updating multiple record steps in the same process
                        updateRecord(record, originalRecordStep, currentStep);
                    }
                }
            } else {
                if (step.beforeEqual(originalRecordStep)) {
                    CollectRecord record;
                    boolean recordChanged = lastModifiedRecordSummary == null || !recordSummary.getId().equals(lastModifiedRecordSummary.getId());
                    if (recordChanged) {
                        // record changed
                        if (lastModifiedRecordSummary != null) {
                            saveLastModifiedRecord();
                        }
                        record = loadRecord(recordSummary.getId(), this.step);
                    } else {
                        record = lastModifiedRecord;
                    }
                    setValuesInRecord(line, record, step);
                    lastModifiedRecordSummary = recordSummary;
                    lastModifiedRecord = record;
                } else {
                    status.addParsingError(new ParsingError(ErrorType.INVALID_VALUE, line.getLineNumber(), (String) null, RECORD_NOT_IN_SELECTED_STEP_MESSAGE_KEY));
                }
            }
        }
    }
    status.addProcessedRow(line.getLineNumber());
}

Example 23 with ParsingError

use of org.openforis.collect.io.metadata.parsing.ParsingError in project collect by openforis.

the class CSVDataImportProcess method processFile.

protected void processFile() {
    long currentRowNumber = 0;
    DataCSVReader reader = null;
    try {
        EntityDefinition parentEntityDefn = getParentEntityDefinition();
        reader = new DataCSVReader(file, parentEntityDefn);
        reader.init();
        status.addProcessedRow(1);
        status.setTotal(reader.size());
        currentRowNumber = 1;
        while (status.isRunning()) {
            currentRowNumber++;
            try {
                DataLine line = reader.readNextLine();
                if (line != null) {
                    processLine(line);
                }
                if (!reader.isReady()) {
                    // end of file reached
                    if (step != null) {
                        saveLastModifiedRecord();
                    }
                    break;
                }
            } catch (ParsingException e) {
                status.addParsingError(currentRowNumber, e.getError());
            }
        }
        if (status.hasErrors()) {
            status.error();
        } else if (status.isRunning()) {
            status.complete();
        }
    } catch (ParsingException e) {
        status.error();
        status.addParsingError(1, e.getError());
    } catch (Exception e) {
        status.error();
        status.addParsingError(currentRowNumber, new ParsingError(ErrorType.IOERROR, e.toString()));
        LOG.error("Error importing CSV file", e);
    } finally {
        close(reader);
    }
}

Example 24 with ParsingError

use of org.openforis.collect.io.metadata.parsing.ParsingError in project collect by openforis.

the class CSVDataImportProcess method setUnitField.

private void setUnitField(Attribute<?, ?> attr, String value, long row, String colName) {
    if (StringUtils.isBlank(value)) {
        ((NumberAttribute<?, ?>) attr).setUnit(null);
    } else {
        Survey survey = attr.getSurvey();
        Unit unit = survey.getUnit(value);
        NumericAttributeDefinition defn = (NumericAttributeDefinition) attr.getDefinition();
        if (unit == null || !defn.getUnits().contains(unit)) {
            ParsingError parsingError = new ParsingError(ErrorType.INVALID_VALUE, row, colName, UNIT_NOT_FOUND_MESSAGE_KEY);
            parsingError.setMessageArgs(new String[] { value });
            status.addParsingError(parsingError);
        } else {
            Field<Integer> field = ((NumberAttribute<?, ?>) attr).getUnitField();
            NodeChangeSet changes = recordUpdater.updateField(field, unit.getId());
            if (nodeChangeBatchProcessor != null) {
                nodeChangeBatchProcessor.add(changes, adminUser.getUsername());
            }
        }
    }
}

Example 25 with ParsingError

use of org.openforis.collect.io.metadata.parsing.ParsingError in project collect by openforis.

the class CSVDataImportJobIntegrationTest method missingRequiredColumnsTest.

@Test
public void missingRequiredColumnsTest() throws Exception {
    EntityDefinition clusterDefn = survey.getSchema().getRootEntityDefinition("cluster");
    EntityDefinition plotDefn = (EntityDefinition) clusterDefn.getChildDefinition("plot");
    CSVDataImportJob process = importCSVFile(MISSING_REQUIRED_COLUMNS_TEST_CSV, plotDefn.getId());
    assertFalse(process.isCompleted());
    assertTrue(process.isFailed());
    List<DataParsingError> errors = process.getParsingErrors();
    assertEquals(1, errors.size());
    ParsingError headerError = errors.get(0);
    assertEquals(ErrorType.MISSING_REQUIRED_COLUMNS, headerError.getErrorType());
}