Examples with RFC4180Parser com.opencsv.RFC4180Parser used on opensource projects

Example 1 with RFC4180Parser

use of com.opencsv.RFC4180Parser in project leetcode-practice by snehasishroy.

the class SenseMachineCoding method readOrderProducts.

private void readOrderProducts() throws IOException, CsvValidationException {
    RFC4180Parser rfc4180Parser = new RFC4180ParserBuilder().build();
    // this ensures correct parsing of files with special character like a,"this is a big string",10,20
    try (CSVReader reader = new CSVReaderBuilder(new FileReader(directory + "order_products__prior.csv")).withCSVParser(rfc4180Parser).withSkipLines(1).build()) {
        String[] lineInArray;
        while ((lineInArray = reader.readNext()) != null) {
            int productId = Integer.parseInt(lineInArray[1]);
            int departmentId = productsToDepartmentIds.get(productId);
            int orderId = Integer.parseInt(lineInArray[0]);
            Pair<Integer, Integer> timeInfo = orders.get(orderId);
            int dayOfWeek = timeInfo.getKey();
            int hourOfDay = timeInfo.getValue();
            int curUserId = orderToUserMapping.get(orderId);
            // dayOfWeek -> {hour -> {department_id -> frequency}}}
            Map<Integer, Integer> departmentFrequencies = frequencyMapping.get(dayOfWeek).get(hourOfDay);
            Set<Integer> userIds = userMapping.get(dayOfWeek).get(hourOfDay).computeIfAbsent(departmentId, __ -> new HashSet<>());
            if (!userIds.contains(curUserId)) {
                // if user has not shopped in the current department during the current hour, then process it
                departmentFrequencies.put(departmentId, departmentFrequencies.getOrDefault(departmentId, 0) + 1);
                userIds.add(curUserId);
            }
        }
    }
}

Example 2 with RFC4180Parser

use of com.opencsv.RFC4180Parser in project structr by structr.

the class CsvHelper method cleanAndParseCSV.

public static Iterable<JsonInput> cleanAndParseCSV(final SecurityContext securityContext, final Reader input, final Class type, final Character fieldSeparator, final Character quoteCharacter, final String range, final Map<String, String> propertyMapping, final boolean rfc4180Mode, final boolean strictQuotes) throws FrameworkException, IOException {
    final CSVReader reader;
    if (rfc4180Mode) {
        reader = new CSVReader(input, 0, new RFC4180Parser());
    } else if (quoteCharacter == null) {
        reader = new CSVReader(input, fieldSeparator);
    } else {
        reader = new CSVReader(input, fieldSeparator, quoteCharacter, strictQuotes);
    }
    final String[] propertyNames = reader.readNext();
    CsvHelper.checkPropertyNames(securityContext, propertyNames);
    return new Iterable<JsonInput>() {

        @Override
        public Iterator<JsonInput> iterator() {
            final Iterator<JsonInput> iterator = new CsvIterator(reader, propertyNames, propertyMapping, type, securityContext.getUser(false).getName());
            if (StringUtils.isNotBlank(range)) {
                return new RangesIterator<>(iterator, range);
            } else {
                return iterator;
            }
        }
    };
}

Example 3 with RFC4180Parser

use of com.opencsv.RFC4180Parser in project unomi by apache.

the class LineSplitProcessor method process.

@Override
public void process(Exchange exchange) throws Exception {
    // In case of one shot import we check the header and overwrite import config
    ImportConfiguration importConfigOneShot = (ImportConfiguration) exchange.getIn().getHeader(RouterConstants.HEADER_IMPORT_CONFIG_ONESHOT);
    String configType = (String) exchange.getIn().getHeader(RouterConstants.HEADER_CONFIG_TYPE);
    if (importConfigOneShot != null) {
        fieldsMapping = (Map<String, Integer>) importConfigOneShot.getProperties().get("mapping");
        propertiesToOverwrite = importConfigOneShot.getPropertiesToOverwrite();
        mergingProperty = importConfigOneShot.getMergingProperty();
        overwriteExistingProfiles = importConfigOneShot.isOverwriteExistingProfiles();
        columnSeparator = importConfigOneShot.getColumnSeparator();
        hasHeader = importConfigOneShot.isHasHeader();
        hasDeleteColumn = importConfigOneShot.isHasDeleteColumn();
        multiValueSeparator = importConfigOneShot.getMultiValueSeparator();
        multiValueDelimiter = importConfigOneShot.getMultiValueDelimiter();
    }
    if ((Integer) exchange.getProperty("CamelSplitIndex") == 0 && hasHeader) {
        exchange.setProperty(Exchange.ROUTE_STOP, Boolean.TRUE);
        return;
    }
    RFC4180Parser rfc4180Parser = new RFC4180ParserBuilder().withSeparator(columnSeparator.charAt(0)).build();
    logger.debug("$$$$ : LineSplitProcessor : BODY : " + (String) exchange.getIn().getBody());
    String[] profileData = rfc4180Parser.parseLine(((String) exchange.getIn().getBody()));
    ProfileToImport profileToImport = new ProfileToImport();
    profileToImport.setItemId(UUID.randomUUID().toString());
    profileToImport.setItemType("profile");
    profileToImport.setScope(RouterConstants.SYSTEM_SCOPE);
    if (profileData.length > 0 && StringUtils.isNotBlank(profileData[0])) {
        if ((hasDeleteColumn && (fieldsMapping.size() > (profileData.length - 1))) || (!hasDeleteColumn && (fieldsMapping.size() > (profileData.length)))) {
            throw new BadProfileDataFormatException("The mapping does not match the number of column : line [" + ((Integer) exchange.getProperty("CamelSplitIndex") + 1) + "]", new Throwable("MAPPING_COLUMN_MATCH"));
        }
        logger.debug("$$$$ : LineSplitProcessor : MAPPING : " + fieldsMapping.keySet());
        Map<String, Object> properties = new HashMap<>();
        for (String fieldMappingKey : fieldsMapping.keySet()) {
            PropertyType propertyType = RouterUtils.getPropertyTypeById(profilePropertyTypes, fieldMappingKey);
            if (fieldMappingKey != null && fieldsMapping.get(fieldMappingKey) != null && profileData != null && profileData[fieldsMapping.get(fieldMappingKey)] != null) {
                logger.debug("$$$$ : LineSplitProcessor : PropType value : {}", profileData[fieldsMapping.get(fieldMappingKey)].trim());
            } else {
                logger.debug("$$$$ : LineSplitProcessor : no profileData found for fieldMappingKey=" + fieldMappingKey);
            }
            if (profileData.length > fieldsMapping.get(fieldMappingKey)) {
                try {
                    if (propertyType == null) {
                        logger.error("No valid property type found for propertyTypeId=" + fieldMappingKey);
                    } else {
                        if (propertyType.getValueTypeId() == null) {
                            logger.error("No value type id found for property type " + propertyType.getItemId());
                        }
                    }
                    if (propertyType.getValueTypeId().equals("string") || propertyType.getValueTypeId().equals("email") || propertyType.getValueTypeId().equals("date")) {
                        if (BooleanUtils.isTrue(propertyType.isMultivalued())) {
                            String multivalueArray = profileData[fieldsMapping.get(fieldMappingKey)].trim();
                            if (StringUtils.isNotBlank(multiValueDelimiter) && multiValueDelimiter.length() == 2) {
                                multivalueArray = multivalueArray.replaceAll("\\" + multiValueDelimiter.charAt(0), "").replaceAll("\\" + multiValueDelimiter.charAt(1), "");
                            }
                            if (multivalueArray.contains(multiValueSeparator)) {
                                String[] valuesArray = multivalueArray.split("\\" + multiValueSeparator);
                                properties.put(fieldMappingKey, valuesArray);
                            } else {
                                if (StringUtils.isNotBlank(multivalueArray)) {
                                    properties.put(fieldMappingKey, new String[] { multivalueArray });
                                } else {
                                    properties.put(fieldMappingKey, new String[] {});
                                }
                            }
                        } else {
                            String singleValue = profileData[fieldsMapping.get(fieldMappingKey)].trim();
                            properties.put(fieldMappingKey, singleValue);
                        }
                    } else if (propertyType.getValueTypeId().equals("boolean")) {
                        properties.put(fieldMappingKey, new Boolean(profileData[fieldsMapping.get(fieldMappingKey)].trim()));
                    } else if (propertyType.getValueTypeId().equals("integer")) {
                        properties.put(fieldMappingKey, new Integer(profileData[fieldsMapping.get(fieldMappingKey)].trim()));
                    } else if (propertyType.getValueTypeId().equals("long")) {
                        properties.put(fieldMappingKey, new Long(profileData[fieldsMapping.get(fieldMappingKey)].trim()));
                    }
                } catch (Throwable t) {
                    logger.error("Error converting profileData", t);
                    if (fieldMappingKey != null && fieldsMapping.get(fieldMappingKey) != null && profileData != null && profileData[fieldsMapping.get(fieldMappingKey)] != null) {
                        throw new BadProfileDataFormatException("Unable to convert '" + profileData[fieldsMapping.get(fieldMappingKey)].trim() + "' to " + propertyType != null ? propertyType.getValueTypeId() : "Null propertyType ", new Throwable("DATA_TYPE"));
                    } else {
                        throw new BadProfileDataFormatException("Unable to find profile data for key " + fieldMappingKey, new Throwable("DATA_TYPE"));
                    }
                }
            }
        }
        profileToImport.setProperties(properties);
        profileToImport.setMergingProperty(mergingProperty);
        profileToImport.setPropertiesToOverwrite(propertiesToOverwrite);
        profileToImport.setOverwriteExistingProfiles(overwriteExistingProfiles);
        if (hasDeleteColumn && StringUtils.isNotBlank(profileData[profileData.length - 1]) && Boolean.parseBoolean(profileData[profileData.length - 1].trim())) {
            profileToImport.setProfileToDelete(true);
        }
    } else {
        throw new BadProfileDataFormatException("Empty line : line [" + ((Integer) exchange.getProperty("CamelSplitIndex") + 1) + "]", new Throwable("EMPTY_LINE"));
    }
    exchange.getIn().setBody(profileToImport, ProfileToImport.class);
    if (RouterConstants.CONFIG_TYPE_KAFKA.equals(configType)) {
        exchange.getIn().setHeader(KafkaConstants.PARTITION_KEY, 0);
        exchange.getIn().setHeader(KafkaConstants.KEY, "1");
    }
}

Example 4 with RFC4180Parser

use of com.opencsv.RFC4180Parser in project tribuo by oracle.

the class SQLToCSV method main.

/**
 * Reads an SQL query from the standard input and writes the results of the
 * query to the standard output.
 *
 * @param args Single arg is the JDBC connection string.
 */
public static void main(String[] args) {
    LabsLogFormatter.setAllLogFormatters();
    SQLToCSVOptions opts = new SQLToCSVOptions();
    ConfigurationManager cm;
    try {
        cm = new ConfigurationManager(args, opts);
    } catch (UsageException e) {
        logger.info(e.getUsage());
        System.exit(1);
    }
    if (opts.dbConfig == null) {
        if (opts.connString == null) {
            logger.log(Level.SEVERE, "Must specify connection string with -n");
            System.exit(1);
        }
        if (opts.username != null || opts.password != null) {
            if (opts.username == null || opts.password == null) {
                logger.log(Level.SEVERE, "Must specify both of user and password with -u, -p if one is specified!");
                System.exit(1);
            }
        }
    } else if (opts.username != null || opts.password != null || opts.connString != null) {
        logger.warning("dbConfig provided but username/password/connstring also provided. Options from -u, -p, -n being ignored");
    }
    String query;
    try (BufferedReader br = opts.inputPath != null ? Files.newBufferedReader(opts.inputPath) : new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8))) {
        StringBuilder qsb = new StringBuilder();
        String l;
        while ((l = br.readLine()) != null) {
            qsb.append(l);
            qsb.append("\n");
        }
        query = qsb.toString().trim();
    } catch (IOException ex) {
        logger.log(Level.SEVERE, "Error reading query: " + ex);
        System.exit(1);
        return;
    }
    if (query.isEmpty()) {
        logger.log(Level.SEVERE, "Query is empty string");
        System.exit(1);
    }
    Connection conn = null;
    try {
        if (opts.dbConfig != null) {
            conn = opts.dbConfig.getConnection();
        } else if (opts.username != null) {
            conn = DriverManager.getConnection(opts.connString, opts.username, opts.password);
        } else {
            conn = DriverManager.getConnection(opts.connString);
        }
    } catch (SQLException ex) {
        logger.log(Level.SEVERE, "Can't connect to database: " + opts.connString, ex);
        System.exit(1);
    }
    try (Statement stmt = conn.createStatement()) {
        stmt.setFetchSize(1000);
        stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
        ResultSet results;
        try {
            results = stmt.executeQuery(query);
        } catch (SQLException ex) {
            logger.log(Level.SEVERE, "Error running query", ex);
            try {
                conn.close();
            } catch (SQLException ex1) {
                logger.log(Level.SEVERE, "Failed to close connection", ex1);
            }
            return;
        }
        try (ICSVWriter writer = new CSVParserWriter(opts.outputPath != null ? Files.newBufferedWriter(opts.outputPath) : new BufferedWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8), 1024 * 1024), new RFC4180Parser(), "\n")) {
            writer.writeAll(results, true);
        } catch (IOException ex) {
            logger.log(Level.SEVERE, "Error writing CSV", ex);
            System.exit(1);
        } catch (SQLException ex) {
            logger.log(Level.SEVERE, "Error retrieving results", ex);
            System.exit(1);
        }
    } catch (SQLException ex) {
        logger.log(Level.SEVERE, "Couldn't create statement", ex);
        try {
            conn.close();
        } catch (SQLException ex1) {
            logger.log(Level.SEVERE, "Failed to close connection", ex1);
        }
        System.exit(1);
        return;
    }
    try {
        conn.close();
    } catch (SQLException ex1) {
        logger.log(Level.SEVERE, "Failed to close connection", ex1);
    }
}