Examples with CSVFormat org.apache.commons.csv.CSVFormat used on opensource projects

Example 16 with CSVFormat

use of org.apache.commons.csv.CSVFormat in project gephi by gephi.

the class ExporterSpreadsheet method exportData.

private void exportData(Graph graph) throws Exception {
    final CSVFormat format = CSVFormat.DEFAULT.withDelimiter(fieldDelimiter);
    try (CSVPrinter csvWriter = new CSVPrinter(writer, format)) {
        boolean isEdgeTable = tableToExport != ExportTable.NODES;
        Table table = isEdgeTable ? graph.getModel().getEdgeTable() : graph.getModel().getNodeTable();
        ElementIterable<? extends Element> rows;
        Object[] edgeLabels = graph.getModel().getEdgeTypeLabels();
        boolean includeEdgeKindColumn = false;
        for (Object edgeLabel : edgeLabels) {
            if (edgeLabel != null && !edgeLabel.toString().isEmpty()) {
                includeEdgeKindColumn = true;
            }
        }
        TimeFormat timeFormat = graph.getModel().getTimeFormat();
        DateTimeZone timeZone = graph.getModel().getTimeZone();
        List<Column> columns = new ArrayList<>();
        if (columnIdsToExport != null) {
            for (String columnId : columnIdsToExport) {
                Column column = table.getColumn(columnId);
                if (column != null) {
                    columns.add(column);
                }
            }
        } else {
            for (Column column : table) {
                columns.add(column);
            }
        }
        // Write column headers:
        if (isEdgeTable) {
            csvWriter.print("Source");
            csvWriter.print("Target");
            csvWriter.print("Type");
            if (includeEdgeKindColumn) {
                csvWriter.print("Kind");
            }
        }
        for (Column column : columns) {
            // Use the title only if it's the same as the id (case insensitive):
            String columnId = column.getId();
            String columnTitle = column.getTitle();
            String columnHeader = columnId.equalsIgnoreCase(columnTitle) ? columnTitle : columnId;
            csvWriter.print(columnHeader);
        }
        csvWriter.println();
        // Write rows:
        if (isEdgeTable) {
            rows = graph.getEdges();
        } else {
            rows = graph.getNodes();
        }
        for (Element row : rows) {
            if (isEdgeTable) {
                Edge edge = (Edge) row;
                csvWriter.print(edge.getSource().getId());
                csvWriter.print(edge.getTarget().getId());
                csvWriter.print(edge.isDirected() ? "Directed" : "Undirected");
                if (includeEdgeKindColumn) {
                    csvWriter.print(edge.getTypeLabel().toString());
                }
            }
            for (Column column : columns) {
                Object value = row.getAttribute(column);
                String text;
                if (value != null) {
                    if (value instanceof Number) {
                        text = NUMBER_FORMAT.format(value);
                    } else {
                        text = AttributeUtils.print(value, timeFormat, timeZone);
                    }
                } else {
                    text = "";
                }
                csvWriter.print(text);
            }
            csvWriter.println();
        }
    }
}

Example 17 with CSVFormat

use of org.apache.commons.csv.CSVFormat in project ofbiz-framework by apache.

the class PartyServices method importParty.

public static Map<String, Object> importParty(DispatchContext dctx, Map<String, Object> context) {
    Delegator delegator = dctx.getDelegator();
    LocalDispatcher dispatcher = dctx.getDispatcher();
    Locale locale = (Locale) context.get("locale");
    GenericValue userLogin = (GenericValue) context.get("userLogin");
    ByteBuffer fileBytes = (ByteBuffer) context.get("uploadedFile");
    String encoding = System.getProperty("file.encoding");
    String csvString = Charset.forName(encoding).decode(fileBytes).toString();
    final BufferedReader csvReader = new BufferedReader(new StringReader(csvString));
    CSVFormat fmt = CSVFormat.DEFAULT.withHeader();
    List<String> errMsgs = new LinkedList<>();
    List<String> newErrMsgs = new LinkedList<>();
    // last partyId read from the csv file
    String lastPartyId = null;
    // current partyId from the csv file
    String currentPartyId = null;
    // new to create/update partyId in the system
    String newPartyId = null;
    String newCompanyPartyId = null;
    int partiesCreated = 0;
    Map<String, Object> result = null;
    String newContactMechId = null;
    String currentContactMechTypeId = null;
    String lastAddress1 = null;
    String lastAddress2 = null;
    String lastCity = null;
    String lastCountryGeoId = null;
    String lastEmailAddress = null;
    String lastCountryCode = null;
    String lastAreaCode = null;
    String lastContactNumber = null;
    String lastContactMechPurposeTypeId = null;
    String currentContactMechPurposeTypeId = null;
    // when modify party, contact mech not added again
    Boolean addParty = false;
    try {
        for (final CSVRecord rec : fmt.parse(csvReader)) {
            if (UtilValidate.isNotEmpty(rec.get("partyId"))) {
                currentPartyId = rec.get("partyId");
            }
            if (lastPartyId == null || !currentPartyId.equals(lastPartyId)) {
                newPartyId = null;
                currentContactMechPurposeTypeId = null;
                lastAddress1 = null;
                lastAddress2 = null;
                lastCity = null;
                lastCountryGeoId = null;
                lastEmailAddress = null;
                lastCountryCode = null;
                lastAreaCode = null;
                lastContactNumber = null;
                // party validation
                List<GenericValue> currencyCheck = EntityQuery.use(delegator).from("Uom").where("abbreviation", rec.get("preferredCurrencyUomId"), "uomTypeId", "CURRENCY_MEASURE").queryList();
                if (UtilValidate.isNotEmpty(rec.get("preferredCurrencyUomId")) && currencyCheck.size() == 0) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Currency code not found for: " + rec.get("preferredCurrencyUomId"));
                }
                if (UtilValidate.isEmpty(rec.get("roleTypeId"))) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": Mandatory roletype is missing, possible values: CUSTOMER, SUPPLIER, EMPLOYEE and more....");
                } else if (EntityQuery.use(delegator).from("RoleType").where("roleTypeId", rec.get("roleTypeId")).queryOne() == null) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": RoletypeId is not valid: " + rec.get("roleTypeId"));
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && EntityQuery.use(delegator).from("ContactMechType").where("contactMechTypeId", rec.get("contactMechTypeId")).cache().queryOne() == null) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + " contactMechTypeId code not found for: " + rec.get("contactMechTypeId"));
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechPurposeTypeId")) && EntityQuery.use(delegator).from("ContactMechPurposeType").where("contactMechPurposeTypeId", rec.get("contactMechPurposeTypeId")).cache().queryOne() == null) {
                    newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "contactMechPurposeTypeId code not found for: " + rec.get("contactMechPurposeTypeId"));
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "POSTAL_ADDRESS".equals(rec.get("contactMechTypeId"))) {
                    if (UtilValidate.isEmpty(rec.get("countryGeoId"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + ": partyId: " + currentPartyId + "Country code missing");
                    } else {
                        List<GenericValue> countryCheck = EntityQuery.use(delegator).from("Geo").where("geoTypeId", "COUNTRY", "abbreviation", rec.get("countryGeoId")).queryList();
                        if (countryCheck.size() == 0) {
                            newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid Country code: " + rec.get("countryGeoId"));
                        }
                    }
                    if (UtilValidate.isEmpty(rec.get("city"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + "City name is missing");
                    }
                    if (UtilValidate.isNotEmpty(rec.get("stateProvinceGeoId"))) {
                        List<GenericValue> stateCheck = EntityQuery.use(delegator).from("Geo").where("geoTypeId", "STATE", "abbreviation", rec.get("stateProvinceGeoId")).queryList();
                        if (stateCheck.size() == 0) {
                            newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " Invalid stateProvinceGeoId code: " + rec.get("countryGeoId"));
                        }
                    }
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "TELECOM_NUMBER".equals(rec.get("contactMechTypeId"))) {
                    if (UtilValidate.isEmpty(rec.get("telAreaCode")) && UtilValidate.isEmpty(rec.get("telAreaCode"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " telephone number missing");
                    }
                }
                if (UtilValidate.isNotEmpty(rec.get("contactMechTypeId")) && "EMAIL_ADDRESS".equals(rec.get("contactMechTypeId"))) {
                    if (UtilValidate.isEmpty(rec.get("emailAddress"))) {
                        newErrMsgs.add("Line number " + rec.getRecordNumber() + " partyId: " + currentPartyId + " email address missing");
                    }
                }
                if (errMsgs.size() == 0) {
                    List<GenericValue> partyCheck = EntityQuery.use(delegator).from("PartyIdentification").where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")).queryList();
                    addParty = partyCheck.size() == 0;
                    if (!addParty) {
                        // update party
                        newPartyId = EntityUtil.getFirst(partyCheck).getString("partyId");
                        if (UtilValidate.isNotEmpty(rec.get("groupName"))) {
                            Map<String, Object> partyGroup = UtilMisc.toMap("partyId", newPartyId, "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin);
                            result = dispatcher.runSync("updatePartyGroup", partyGroup);
                            if (ServiceUtil.isError(result)) {
                                return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                            }
                        } else {
                            // person
                            Map<String, Object> person = UtilMisc.toMap("partyId", newPartyId, "firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "userLogin", userLogin);
                            result = dispatcher.runSync("updatePerson", person);
                            if (ServiceUtil.isError(result)) {
                                return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                            }
                        }
                    } else {
                        // create new party
                        if (UtilValidate.isNotEmpty(rec.get("groupName"))) {
                            Map<String, Object> partyGroup = UtilMisc.toMap("preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "groupName", rec.get("groupName"), "userLogin", userLogin, "statusId", "PARTY_ENABLED");
                            result = dispatcher.runSync("createPartyGroup", partyGroup);
                            if (ServiceUtil.isError(result)) {
                                return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                            }
                        } else {
                            // person
                            Map<String, Object> person = UtilMisc.toMap("firstName", rec.get("firstName"), "middleName", rec.get("middleName"), "lastName", rec.get("lastName"), "preferredCurrencyUomId", rec.get("preferredCurrencyUomId"), "statusId", "PARTY_ENABLED", "userLogin", userLogin);
                            result = dispatcher.runSync("createPerson", person);
                            if (ServiceUtil.isError(result)) {
                                return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                            }
                        }
                        newPartyId = (String) result.get("partyId");
                        Map<String, Object> partyIdentification = UtilMisc.toMap("partyId", newPartyId, "partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId"), "userLogin", userLogin);
                        result = dispatcher.runSync("createPartyIdentification", partyIdentification);
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                        Map<String, Object> partyRole = UtilMisc.toMap("partyId", newPartyId, "roleTypeId", rec.get("roleTypeId"), "userLogin", userLogin);
                        dispatcher.runSync("createPartyRole", partyRole);
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                        if (UtilValidate.isNotEmpty(rec.get("companyPartyId"))) {
                            List<GenericValue> companyCheck = EntityQuery.use(delegator).from("PartyIdentification").where("partyIdentificationTypeId", "PARTY_IMPORT", "idValue", rec.get("partyId")).queryList();
                            if (companyCheck.size() == 0) {
                                // update party group
                                // company does not exist so create
                                Map<String, Object> companyPartyGroup = UtilMisc.toMap("partyId", newCompanyPartyId, "statusId", "PARTY_ENABLED", "userLogin", userLogin);
                                result = dispatcher.runSync("createPartyGroup", companyPartyGroup);
                                if (ServiceUtil.isError(result)) {
                                    return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                                }
                                newCompanyPartyId = (String) result.get("partyId");
                            } else {
                                newCompanyPartyId = EntityUtil.getFirst(companyCheck).getString("partyId");
                            }
                            Map<String, Object> companyRole = UtilMisc.toMap("partyId", newCompanyPartyId, "roleTypeId", "ACCOUNT", "userLogin", userLogin);
                            Map<String, Object> serviceResult = dispatcher.runSync("createPartyRole", companyRole);
                            if (ServiceUtil.isError(serviceResult)) {
                                return ServiceUtil.returnError(ServiceUtil.getErrorMessage(serviceResult));
                            }
                            // company exist, so create link
                            Map<String, Object> partyRelationship = UtilMisc.toMap("partyIdTo", newPartyId, "partyIdFrom", newCompanyPartyId, "roleTypeIdFrom", "ACCOUNT", "partyRelationshipTypeId", "EMPLOYMENT", "userLogin", userLogin);
                            result = dispatcher.runSync("createPartyRelationship", partyRelationship);
                            if (ServiceUtil.isError(result)) {
                                return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                            }
                        }
                    }
                    Debug.logInfo(" =========================================================party created id: " + newPartyId, module);
                    partiesCreated++;
                } else {
                    errMsgs.addAll(newErrMsgs);
                    newErrMsgs = new LinkedList<>();
                }
            }
            currentContactMechTypeId = rec.get("contactMechTypeId");
            currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId");
            // party correctly created (not updated) and contactMechtype provided?
            if (newPartyId != null && addParty && UtilValidate.isNotEmpty(currentContactMechTypeId)) {
                // fill maps and check changes
                Map<String, Object> emailAddress = UtilMisc.toMap("contactMechTypeId", "EMAIL_ADDRESS", "userLogin", userLogin);
                Boolean emailAddressChanged = false;
                if ("EMAIL_ADDRESS".equals(currentContactMechTypeId)) {
                    emailAddress.put("infoString", rec.get("emailAddress"));
                    emailAddressChanged = lastEmailAddress == null || !lastEmailAddress.equals(rec.get("emailAddress"));
                    lastEmailAddress = rec.get("emailAddress");
                }
                // casting is here necessary for some compiler versions
                Map<String, Object> postalAddress = UtilMisc.toMap("userLogin", (Object) userLogin);
                Boolean postalAddressChanged = false;
                if ("POSTAL_ADDRESS".equals(currentContactMechTypeId)) {
                    postalAddress.put("address1", rec.get("address1"));
                    postalAddress.put("address2", rec.get("address2"));
                    postalAddress.put("city", rec.get("city"));
                    postalAddress.put("stateProvinceGeoId", rec.get("stateProvinceGeoId"));
                    postalAddress.put("countryGeoId", rec.get("countryGeoId"));
                    postalAddress.put("postalCode", rec.get("postalCode"));
                    postalAddressChanged = lastAddress1 == null || !lastAddress1.equals(postalAddress.get("address1")) || lastAddress2 == null || !lastAddress2.equals(postalAddress.get("address2")) || lastCity == null || !lastCity.equals(postalAddress.get("city")) || lastCountryGeoId == null || !lastCountryGeoId.equals(postalAddress.get("countryGeoId"));
                    lastAddress1 = (String) postalAddress.get("address1");
                    lastAddress2 = (String) postalAddress.get("address2");
                    lastCity = (String) postalAddress.get("city");
                    lastCountryGeoId = (String) postalAddress.get("countryGeoId");
                }
                // casting is here necessary for some compiler versions
                Map<String, Object> telecomNumber = UtilMisc.toMap("userLogin", (Object) userLogin);
                Boolean telecomNumberChanged = false;
                if ("TELECOM_NUMBER".equals(currentContactMechTypeId)) {
                    telecomNumber.put("countryCode", rec.get("telCountryCode"));
                    telecomNumber.put("areaCode", rec.get("telAreaCode"));
                    telecomNumber.put("contactNumber", rec.get("telContactNumber"));
                    telecomNumberChanged = lastCountryCode == null || !lastCountryCode.equals(telecomNumber.get("countryCode")) || lastAreaCode == null || !lastAreaCode.equals(telecomNumber.get("areaCode")) || lastContactNumber == null || !lastContactNumber.equals(telecomNumber.get("contactNumber"));
                    lastCountryCode = (String) telecomNumber.get("countryCode");
                    lastAreaCode = (String) telecomNumber.get("areaCode");
                    lastContactNumber = (String) telecomNumber.get("contactNumber");
                }
                Map<String, Object> partyContactMechPurpose = UtilMisc.toMap("partyId", newPartyId, "userLogin", userLogin);
                Boolean partyContactMechPurposeChanged = false;
                currentContactMechPurposeTypeId = rec.get("contactMechPurposeTypeId");
                if (currentContactMechPurposeTypeId != null && ("TELECOM_NUMBER".equals(currentContactMechTypeId) || "POSTAL_ADDRESS".equals(currentContactMechTypeId) || "EMAIL_ADDRESS".equals(currentContactMechTypeId))) {
                    partyContactMechPurpose.put("contactMechPurposeTypeId", currentContactMechPurposeTypeId);
                    partyContactMechPurposeChanged = (lastContactMechPurposeTypeId == null || !lastContactMechPurposeTypeId.equals(currentContactMechPurposeTypeId)) && !telecomNumberChanged && !postalAddressChanged && !emailAddressChanged;
                    Debug.logInfo("===================================last:" + lastContactMechPurposeTypeId + " current: " + currentContactMechPurposeTypeId + " t :" + telecomNumberChanged + " p: " + postalAddressChanged + " e: " + emailAddressChanged + " result: " + partyContactMechPurposeChanged, module);
                }
                lastContactMechPurposeTypeId = currentContactMechPurposeTypeId;
                // update
                if (errMsgs.size() == 0) {
                    if (postalAddressChanged) {
                        result = dispatcher.runSync("createPostalAddress", postalAddress);
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                        newContactMechId = (String) result.get("contactMechId");
                        if (currentContactMechPurposeTypeId == null) {
                            currentContactMechPurposeTypeId = "GENERAL_LOCATION";
                        }
                        Map<String, Object> serviceResult = dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
                        if (ServiceUtil.isError(serviceResult)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(serviceResult));
                        }
                    }
                    if (telecomNumberChanged) {
                        result = dispatcher.runSync("createTelecomNumber", telecomNumber);
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                        newContactMechId = (String) result.get("contactMechId");
                        if (currentContactMechPurposeTypeId == null) {
                            currentContactMechPurposeTypeId = "PHONE_WORK";
                        }
                        Map<String, Object> resultMap = dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                    }
                    if (emailAddressChanged) {
                        result = dispatcher.runSync("createContactMech", emailAddress);
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                        newContactMechId = (String) result.get("contactMechId");
                        if (currentContactMechPurposeTypeId == null) {
                            currentContactMechPurposeTypeId = "PRIMARY_EMAIL";
                        }
                        Map<String, Object> resultMap = dispatcher.runSync("createPartyContactMech", UtilMisc.toMap("partyId", newPartyId, "contactMechId", newContactMechId, "contactMechPurposeTypeId", currentContactMechPurposeTypeId, "userLogin", userLogin));
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                    }
                    if (partyContactMechPurposeChanged) {
                        partyContactMechPurpose.put("contactMechId", newContactMechId);
                        result = dispatcher.runSync("createPartyContactMechPurpose", partyContactMechPurpose);
                        if (ServiceUtil.isError(result)) {
                            return ServiceUtil.returnError(ServiceUtil.getErrorMessage(result));
                        }
                    }
                    lastPartyId = currentPartyId;
                    errMsgs.addAll(newErrMsgs);
                    newErrMsgs = new LinkedList<>();
                }
            }
        }
    } catch (GenericServiceException | GenericEntityException | IOException e) {
        Debug.logError(e, module);
        return ServiceUtil.returnError(e.getMessage());
    }
    if (errMsgs.size() > 0) {
        return ServiceUtil.returnError(errMsgs);
    }
    result = ServiceUtil.returnSuccess(UtilProperties.getMessage(resource, "PartyNewPartiesCreated", UtilMisc.toMap("partiesCreated", partiesCreated), locale));
    return result;
}

Example 18 with CSVFormat

use of org.apache.commons.csv.CSVFormat in project sw360portal by sw360.

the class UserPortlet method getUsersFromRequest.

private List<UserCSV> getUsersFromRequest(PortletRequest request, String fileUploadFormId) throws IOException, TException {
    final UploadPortletRequest uploadPortletRequest = PortalUtil.getUploadPortletRequest(request);
    final InputStream stream = uploadPortletRequest.getFileAsStream(fileUploadFormId);
    Reader reader = new InputStreamReader(stream);
    CSVFormat format = CommonUtils.sw360CsvFormat;
    CSVParser parser = new CSVParser(reader, format);
    List<CSVRecord> records;
    records = parser.getRecords();
    if (records.size() > 0) {
        // Remove header
        records.remove(0);
    }
    return getUsersFromCSV(records);
}

Example 19 with CSVFormat

use of org.apache.commons.csv.CSVFormat in project solr-cmd-utils by tblsoft.

the class CSVReader method read.

@Override
public void read() {
    String absoluteFilename;
    boolean addMeta = false;
    try {
        String charset = getProperty("charset", StandardCharsets.UTF_8.name());
        String filename = getProperty("filename", null);
        absoluteFilename = IOUtils.getAbsoluteFile(getBaseDir(), filename);
        addMeta = getPropertyAsBoolean("addMeta", false);
        Long maxRows = getPropertyAsInteger("maxRows", Long.MAX_VALUE);
        String delimiter = getProperty("delimiter", ",");
        String arrayDelimiter = getProperty("arrayDelimiter", null);
        String[] headers = getPropertyAsArray("headers", null);
        InputStream in = IOUtils.getInputStream(absoluteFilename);
        java.io.Reader reader = new InputStreamReader(in, charset);
        CSVFormat format = CSVFormat.RFC4180;
        if (headers == null) {
            format = format.withHeader();
        } else {
            format = format.withHeader(headers);
        }
        format = format.withDelimiter(delimiter.charAt(0));
        CSVParser parser = format.parse(reader);
        Iterator<CSVRecord> csvIterator = parser.iterator();
        long rowNumber = 0;
        while (csvIterator.hasNext()) {
            if (rowNumber >= maxRows) {
                break;
            }
            rowNumber++;
            CSVRecord record = csvIterator.next();
            Map<String, Integer> header = parser.getHeaderMap();
            Document document = new Document();
            for (Map.Entry<String, Integer> entry : header.entrySet()) {
                String key = entry.getKey();
                try {
                    String value = record.get(key);
                    if (StringUtils.isEmpty(arrayDelimiter)) {
                        document.addField(key, value);
                    } else {
                        List<String> valueList = new ArrayList<String>();
                        String[] values = value.split(arrayDelimiter);
                        if (values.length > 0) {
                            for (String val : values) {
                                if (StringUtils.isNotEmpty(val)) {
                                    valueList.add(val);
                                }
                            }
                        }
                        document.setField(key, valueList);
                    }
                } catch (IllegalArgumentException e) {
                }
            }
            if (addMeta) {
                document.addField("rowNumber", String.valueOf(rowNumber));
                document.addField("fileName", absoluteFilename);
            }
            executer.document(document);
        }
        // executer.end();
        in.close();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 20 with CSVFormat

use of org.apache.commons.csv.CSVFormat in project solr-cmd-utils by tblsoft.

the class WhitelistTopicTermsFilter method init.

@Override
public void init() {
    fieldTopic = getProperty("fieldTopic", null);
    fieldValue = getProperty("fieldValue", null);
    override = getPropertyAsBoolean("override", true);
    arrayDelimiter = getProperty("arrayDelimiter", ";");
    topicValues = new HashMap<String, HashMap<String, Document>>();
    topicsOverriden = new HashMap<String, HashMap<String, Boolean>>();
    InputStream in = null;
    try {
        String filename = getProperty("filename", null);
        String absoluteFilename = IOUtils.getAbsoluteFile(getBaseDir(), filename);
        in = IOUtils.getInputStream(absoluteFilename);
        java.io.Reader reader = new InputStreamReader(in, StandardCharsets.UTF_8.name());
        CSVFormat format = CSVFormat.RFC4180.withHeader().withDelimiter(',');
        CSVParser parser = format.parse(reader);
        Iterator<CSVRecord> csvIterator = parser.iterator();
        while (csvIterator.hasNext()) {
            CSVRecord record = csvIterator.next();
            Map<String, Integer> header = parser.getHeaderMap();
            Document document = new Document();
            for (Map.Entry<String, Integer> entry : header.entrySet()) {
                String key = entry.getKey();
                try {
                    String[] values = record.get(key).split(arrayDelimiter);
                    document.addField(key, Arrays.asList(values));
                } catch (IllegalArgumentException ignored) {
                }
            }
            String topic = record.get(header.get(fieldTopic));
            String value = record.get(header.get(fieldValue));
            if (!topicValues.containsKey(topic)) {
                topicValues.put(topic, new HashMap<String, Document>());
                topicsOverriden.put(topic, new HashMap<String, Boolean>());
            }
            topicValues.get(topic).put(value, document);
            topicsOverriden.get(topic).put(value, false);
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        if (in != null) {
            try {
                in.close();
            } catch (IOException ignored) {
            }
        }
    }
    super.init();
}