Examples with CsvPreference org.supercsv.prefs.CsvPreference used on opensource projects

Example 6 with CsvPreference

use of org.supercsv.prefs.CsvPreference in project openscoring by openscoring.

the class ModelResource method doEvaluateCsv.

private Response doEvaluateCsv(String id, String delimiterChar, String quoteChar, final Charset charset, InputStream is) {
    final CsvPreference format;
    final CsvUtil.Table<EvaluationRequest> requestTable;
    try {
        BufferedReader reader = new BufferedReader(new InputStreamReader(is, charset)) {

            @Override
            public void close() {
            // The closing of the underlying java.io.InputStream is handled elsewhere
            }
        };
        try {
            if (delimiterChar != null) {
                format = CsvUtil.getFormat(delimiterChar, quoteChar);
            } else {
                format = CsvUtil.getFormat(reader);
            }
            requestTable = CsvUtil.readTable(reader, format);
        } finally {
            reader.close();
        }
    } catch (Exception e) {
        logger.error("Failed to load CSV document", e);
        throw new BadRequestException(e);
    }
    List<EvaluationRequest> requests = requestTable.getRows();
    List<EvaluationResponse> responses = doEvaluate(id, requests, true, "evaluate.csv");
    final CsvUtil.Table<EvaluationResponse> responseTable = new CsvUtil.Table<>();
    responseTable.setId(requestTable.getId());
    responseTable.setRows(responses);
    StreamingOutput entity = new StreamingOutput() {

        @Override
        public void write(OutputStream os) throws IOException {
            BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset)) {

                @Override
                public void close() throws IOException {
                    flush();
                // The closing of the underlying java.io.OutputStream is handled elsewhere
                }
            };
            try {
                CsvUtil.writeTable(writer, format, responseTable);
            } finally {
                writer.close();
            }
        }
    };
    return (Response.ok().entity(entity)).type(MediaType.TEXT_PLAIN_TYPE.withCharset(charset.name())).header(HttpHeaders.CONTENT_DISPOSITION, // XXX
    "attachment; filename=" + id + ".csv").build();
}

Example 7 with CsvPreference

use of org.supercsv.prefs.CsvPreference in project openscoring by openscoring.

the class CsvUtilTest method getFormat.

@Test
public void getFormat() throws IOException {
    CsvPreference first;
    CsvPreference second;
    String csv = "1\tone\n" + "2\ttwo\n" + "3\tthree";
    try (BufferedReader reader = new BufferedReader(new StringReader(csv))) {
        first = CsvUtil.getFormat(reader);
        second = CsvUtil.getFormat(reader);
    }
    assertNotSame(first.getEncoder(), second.getEncoder());
}

Example 8 with CsvPreference

use of org.supercsv.prefs.CsvPreference in project nifi by apache.

the class ValidateCsv method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final CsvPreference csvPref = getPreference(context, flowFile);
    final boolean header = context.getProperty(HEADER).asBoolean();
    final ComponentLog logger = getLogger();
    final String schema = context.getProperty(SCHEMA).evaluateAttributeExpressions(flowFile).getValue();
    final CellProcessor[] cellProcs = this.parseSchema(schema);
    final boolean isWholeFFValidation = context.getProperty(VALIDATION_STRATEGY).getValue().equals(VALIDATE_WHOLE_FLOWFILE.getValue());
    final AtomicReference<Boolean> valid = new AtomicReference<Boolean>(true);
    final AtomicReference<Boolean> isFirstLineValid = new AtomicReference<Boolean>(true);
    final AtomicReference<Boolean> isFirstLineInvalid = new AtomicReference<Boolean>(true);
    final AtomicReference<Integer> okCount = new AtomicReference<Integer>(0);
    final AtomicReference<Integer> totalCount = new AtomicReference<Integer>(0);
    final AtomicReference<FlowFile> invalidFF = new AtomicReference<FlowFile>(null);
    final AtomicReference<FlowFile> validFF = new AtomicReference<FlowFile>(null);
    if (!isWholeFFValidation) {
        invalidFF.set(session.create(flowFile));
        validFF.set(session.create(flowFile));
    }
    session.read(flowFile, new InputStreamCallback() {

        @Override
        public void process(final InputStream in) throws IOException {
            NifiCsvListReader listReader = null;
            try {
                listReader = new NifiCsvListReader(new InputStreamReader(in), csvPref);
                // handling of header
                if (header) {
                    List<String> headerList = listReader.read();
                    if (!isWholeFFValidation) {
                        invalidFF.set(session.append(invalidFF.get(), new OutputStreamCallback() {

                            @Override
                            public void process(OutputStream out) throws IOException {
                                out.write(print(headerList, csvPref, isFirstLineInvalid.get()));
                            }
                        }));
                        validFF.set(session.append(validFF.get(), new OutputStreamCallback() {

                            @Override
                            public void process(OutputStream out) throws IOException {
                                out.write(print(headerList, csvPref, isFirstLineValid.get()));
                            }
                        }));
                        isFirstLineValid.set(false);
                        isFirstLineInvalid.set(false);
                    }
                }
                boolean stop = false;
                while (!stop) {
                    try {
                        final List<Object> list = listReader.read(cellProcs);
                        stop = list == null;
                        if (!isWholeFFValidation && !stop) {
                            validFF.set(session.append(validFF.get(), new OutputStreamCallback() {

                                @Override
                                public void process(OutputStream out) throws IOException {
                                    out.write(print(list, csvPref, isFirstLineValid.get()));
                                }
                            }));
                            okCount.set(okCount.get() + 1);
                            if (isFirstLineValid.get()) {
                                isFirstLineValid.set(false);
                            }
                        }
                    } catch (final SuperCsvException e) {
                        valid.set(false);
                        if (isWholeFFValidation) {
                            logger.debug("Failed to validate {} against schema due to {}; routing to 'invalid'", new Object[] { flowFile }, e);
                            break;
                        } else {
                            // we append the invalid line to the flow file that will be routed to invalid relationship
                            invalidFF.set(session.append(invalidFF.get(), new OutputStreamCallback() {

                                @Override
                                public void process(OutputStream out) throws IOException {
                                    out.write(print(e.getCsvContext().getRowSource(), csvPref, isFirstLineInvalid.get()));
                                }
                            }));
                            if (isFirstLineInvalid.get()) {
                                isFirstLineInvalid.set(false);
                            }
                        }
                    } finally {
                        if (!isWholeFFValidation) {
                            totalCount.set(totalCount.get() + 1);
                        }
                    }
                }
            } catch (final IOException e) {
                valid.set(false);
                logger.error("Failed to validate {} against schema due to {}", new Object[] { flowFile }, e);
            } finally {
                if (listReader != null) {
                    listReader.close();
                }
            }
        }
    });
    if (isWholeFFValidation) {
        if (valid.get()) {
            logger.debug("Successfully validated {} against schema; routing to 'valid'", new Object[] { flowFile });
            session.getProvenanceReporter().route(flowFile, REL_VALID);
            session.transfer(flowFile, REL_VALID);
        } else {
            session.getProvenanceReporter().route(flowFile, REL_INVALID);
            session.transfer(flowFile, REL_INVALID);
        }
    } else {
        if (valid.get()) {
            logger.debug("Successfully validated {} against schema; routing to 'valid'", new Object[] { validFF.get() });
            session.getProvenanceReporter().route(validFF.get(), REL_VALID, "All " + totalCount.get() + " line(s) are valid");
            session.putAttribute(validFF.get(), "count.valid.lines", Integer.toString(totalCount.get()));
            session.putAttribute(validFF.get(), "count.total.lines", Integer.toString(totalCount.get()));
            session.transfer(validFF.get(), REL_VALID);
            session.remove(invalidFF.get());
            session.remove(flowFile);
        } else if (okCount.get() != 0) {
            // because of the finally within the 'while' loop
            totalCount.set(totalCount.get() - 1);
            logger.debug("Successfully validated {}/{} line(s) in {} against schema; routing valid lines to 'valid' and invalid lines to 'invalid'", new Object[] { okCount.get(), totalCount.get(), flowFile });
            session.getProvenanceReporter().route(validFF.get(), REL_VALID, okCount.get() + " valid line(s)");
            session.putAttribute(validFF.get(), "count.total.lines", Integer.toString(totalCount.get()));
            session.putAttribute(validFF.get(), "count.valid.lines", Integer.toString(okCount.get()));
            session.transfer(validFF.get(), REL_VALID);
            session.getProvenanceReporter().route(invalidFF.get(), REL_INVALID, (totalCount.get() - okCount.get()) + " invalid line(s)");
            session.putAttribute(invalidFF.get(), "count.invalid.lines", Integer.toString((totalCount.get() - okCount.get())));
            session.putAttribute(invalidFF.get(), "count.total.lines", Integer.toString(totalCount.get()));
            session.transfer(invalidFF.get(), REL_INVALID);
            session.remove(flowFile);
        } else {
            logger.debug("All lines in {} are invalid; routing to 'invalid'", new Object[] { invalidFF.get() });
            session.getProvenanceReporter().route(invalidFF.get(), REL_INVALID, "All " + totalCount.get() + " line(s) are invalid");
            session.putAttribute(invalidFF.get(), "count.invalid.lines", Integer.toString(totalCount.get()));
            session.putAttribute(invalidFF.get(), "count.total.lines", Integer.toString(totalCount.get()));
            session.transfer(invalidFF.get(), REL_INVALID);
            session.remove(validFF.get());
            session.remove(flowFile);
        }
    }
}

Example 9 with CsvPreference

use of org.supercsv.prefs.CsvPreference in project waltz by khartec.

the class TabularFileAnalyserTest method foo.

@Test
public void foo() throws IOException {
    char[] delimeters = new char[] { ',', '|', '\t', ';', '!' };
    char[] quoteChars = new char[] { '"', '\'' };
    List<ParseAnalysis> analysisResults = ListUtilities.newArrayList();
    for (char quoteChar : quoteChars) {
        for (char delimeter : delimeters) {
            InputStreamReader simpleReader = getReader();
            CsvPreference prefs = new CsvPreference.Builder(quoteChar, delimeter, "\n").ignoreEmptyLines(false).build();
            CsvListReader csvReader = new CsvListReader(simpleReader, prefs);
            List<String> cells = csvReader.read();
            ImmutableParseAnalysis.Builder parseAnalysisBuilder = ImmutableParseAnalysis.builder().quoteChar(quoteChar).delimiterChar(delimeter);
            while (cells != null) {
                parseAnalysisBuilder.addFieldCounts(cells.size());
                cells = csvReader.read();
            }
            ParseAnalysis parseAnalysis = parseAnalysisBuilder.build();
            analysisResults.add(parseAnalysis);
        }
    }
    analysisResults.forEach(r -> {
        System.out.println(r.quoteChar() + " " + r.delimiterChar() + " => [ " + r.fieldCounts().size() + " ] " + r.fieldCounts());
    });
}

Example 10 with CsvPreference

use of org.supercsv.prefs.CsvPreference in project apex-malhar by apache.

the class AbstractCsvParser method setup.

@Override
public void setup(OperatorContext context) {
    if (fieldmappingFile != null) {
        Configuration conf = new Configuration();
        try {
            FileSystem fs = FileSystem.get(conf);
            Path filepath = new Path(fieldmappingFile);
            if (fs.exists(filepath)) {
                BufferedReader bfr = new BufferedReader(new InputStreamReader(fs.open(filepath)));
                String str;
                while ((str = bfr.readLine()) != null) {
                    logger.debug("string is {}", str);
                    String[] temp = str.split(fieldmappingFileDelimiter);
                    Field field = new Field();
                    field.setName(temp[0]);
                    field.setType(temp[1]);
                    getFields().add(field);
                }
            } else {
                logger.debug("File containing fields and their data types does not exist.Please specify the fields and data type through properties of this operator.");
            }
        } catch (IOException ex) {
            DTThrowable.rethrow(ex);
        }
    }
    int countKeyValue = getFields().size();
    properties = new String[countKeyValue];
    processors = new CellProcessor[countKeyValue];
    initialise(properties, processors);
    CsvPreference preference = new CsvPreference.Builder('"', fieldDelimiter, lineDelimiter).build();
    csvReader = getReader(csvStringReader, preference);
}