Examples with CSVStrategy org.apache.solr.internal.csv.CSVStrategy used on opensource projects

Example 1 with CSVStrategy

use of org.apache.solr.internal.csv.CSVStrategy in project lucene-solr by apache.

the class CSVWriter method writeResponse.

public void writeResponse() throws IOException {
    SolrParams params = req.getParams();
    strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true, "\n");
    CSVStrategy strat = strategy;
    String sep = params.get(CSV_SEPARATOR);
    if (sep != null) {
        if (sep.length() != 1)
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid separator:'" + sep + "'");
        strat.setDelimiter(sep.charAt(0));
    }
    String nl = params.get(CSV_NEWLINE);
    if (nl != null) {
        if (nl.length() == 0)
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid newline:'" + nl + "'");
        strat.setPrinterNewline(nl);
    }
    String encapsulator = params.get(CSV_ENCAPSULATOR);
    String escape = params.get(CSV_ESCAPE);
    if (encapsulator != null) {
        if (encapsulator.length() != 1)
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid encapsulator:'" + encapsulator + "'");
        strat.setEncapsulator(encapsulator.charAt(0));
    }
    if (escape != null) {
        if (escape.length() != 1)
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid escape:'" + escape + "'");
        strat.setEscape(escape.charAt(0));
        if (encapsulator == null) {
            strat.setEncapsulator(CSVStrategy.ENCAPSULATOR_DISABLED);
        }
    }
    if (strat.getEscape() == '\\') {
        // If the escape is the standard backslash, then also enable
        // unicode escapes (it's harmless since 'u' would not otherwise
        // be escaped.
        strat.setUnicodeEscapeInterpretation(true);
    }
    printer = new CSVPrinter(writer, strategy);
    CSVStrategy mvStrategy = new CSVStrategy(strategy.getDelimiter(), CSVStrategy.ENCAPSULATOR_DISABLED, CSVStrategy.COMMENTS_DISABLED, '\\', false, false, false, false, "\n");
    strat = mvStrategy;
    sep = params.get(MV_SEPARATOR);
    if (sep != null) {
        if (sep.length() != 1)
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid mv separator:'" + sep + "'");
        strat.setDelimiter(sep.charAt(0));
    }
    encapsulator = params.get(MV_ENCAPSULATOR);
    escape = params.get(MV_ESCAPE);
    if (encapsulator != null) {
        if (encapsulator.length() != 1)
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid mv encapsulator:'" + encapsulator + "'");
        strat.setEncapsulator(encapsulator.charAt(0));
        if (escape == null) {
            strat.setEscape(CSVStrategy.ESCAPE_DISABLED);
        }
    }
    escape = params.get(MV_ESCAPE);
    if (escape != null) {
        if (escape.length() != 1)
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid mv escape:'" + escape + "'");
        strat.setEscape(escape.charAt(0));
    // encapsulator will already be disabled if it wasn't specified
    }
    Collection<String> fields = returnFields.getRequestedFieldNames();
    Object responseObj = rsp.getResponse();
    boolean returnOnlyStored = false;
    if (fields == null || returnFields.hasPatternMatching()) {
        if (responseObj instanceof SolrDocumentList) {
            // get the list of fields from the SolrDocumentList
            if (fields == null) {
                fields = new LinkedHashSet<>();
            }
            for (SolrDocument sdoc : (SolrDocumentList) responseObj) {
                fields.addAll(sdoc.getFieldNames());
            }
        } else {
            // get the list of fields from the index
            Iterable<String> all = req.getSearcher().getFieldNames();
            if (fields == null) {
                fields = Sets.newHashSet(all);
            } else {
                Iterables.addAll(fields, all);
            }
        }
        if (returnFields.wantsScore()) {
            fields.add("score");
        } else {
            fields.remove("score");
        }
        returnOnlyStored = true;
    }
    CSVSharedBufPrinter csvPrinterMV = new CSVSharedBufPrinter(mvWriter, mvStrategy);
    for (String field : fields) {
        if (!returnFields.wantsField(field)) {
            continue;
        }
        if (field.equals("score")) {
            CSVField csvField = new CSVField();
            csvField.name = "score";
            csvFields.put("score", csvField);
            continue;
        }
        SchemaField sf = schema.getFieldOrNull(field);
        if (sf == null) {
            FieldType ft = new StrField();
            sf = new SchemaField(field, ft);
        }
        // Return only stored fields, unless an explicit field list is specified
        if (returnOnlyStored && sf != null && !sf.stored()) {
            continue;
        }
        // check for per-field overrides
        sep = params.get("f." + field + '.' + CSV_SEPARATOR);
        encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
        escape = params.get("f." + field + '.' + CSV_ESCAPE);
        // if polyfield and no escape is provided, add "\\" escape by default
        if (sf.isPolyField()) {
            escape = (escape == null) ? "\\" : escape;
        }
        CSVSharedBufPrinter csvPrinter = csvPrinterMV;
        if (sep != null || encapsulator != null || escape != null) {
            // create a new strategy + printer if there were any per-field overrides
            strat = (CSVStrategy) mvStrategy.clone();
            if (sep != null) {
                if (sep.length() != 1)
                    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid mv separator:'" + sep + "'");
                strat.setDelimiter(sep.charAt(0));
            }
            if (encapsulator != null) {
                if (encapsulator.length() != 1)
                    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid mv encapsulator:'" + encapsulator + "'");
                strat.setEncapsulator(encapsulator.charAt(0));
                if (escape == null) {
                    strat.setEscape(CSVStrategy.ESCAPE_DISABLED);
                }
            }
            if (escape != null) {
                if (escape.length() != 1)
                    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Invalid mv escape:'" + escape + "'");
                strat.setEscape(escape.charAt(0));
                if (encapsulator == null) {
                    strat.setEncapsulator(CSVStrategy.ENCAPSULATOR_DISABLED);
                }
            }
            csvPrinter = new CSVSharedBufPrinter(mvWriter, strat);
        }
        CSVField csvField = new CSVField();
        csvField.name = field;
        csvField.sf = sf;
        csvField.mvPrinter = csvPrinter;
        csvFields.put(field, csvField);
    }
    NullValue = params.get(CSV_NULL, "");
    if (params.getBool(CSV_HEADER, true)) {
        for (CSVField csvField : csvFields.values()) {
            printer.print(csvField.name);
        }
        printer.println();
    }
    if (responseObj instanceof ResultContext) {
        writeDocuments(null, (ResultContext) responseObj);
    } else if (responseObj instanceof DocList) {
        ResultContext ctx = new BasicResultContext((DocList) responseObj, returnFields, null, null, req);
        writeDocuments(null, ctx);
    } else if (responseObj instanceof SolrDocumentList) {
        writeSolrDocumentList(null, (SolrDocumentList) responseObj, returnFields);
    }
}

Example 2 with CSVStrategy

use of org.apache.solr.internal.csv.CSVStrategy in project lucene-solr by apache.

the class CSVLoaderBase method prepareFields.

/** create the FieldAdders that control how each field  is indexed */
void prepareFields() {
    // Possible future optimization: for really rapid incremental indexing
    // from a POST, one could cache all of this setup info based on the params.
    // The link from FieldAdder to this would need to be severed for that to happen.
    adders = new CSVLoaderBase.FieldAdder[fieldnames.length];
    String skipStr = params.get(SKIP);
    List<String> skipFields = skipStr == null ? null : StrUtils.splitSmart(skipStr, ',');
    CSVLoaderBase.FieldAdder adder = new CSVLoaderBase.FieldAdder();
    CSVLoaderBase.FieldAdder adderKeepEmpty = new CSVLoaderBase.FieldAdderEmpty();
    for (int i = 0; i < fieldnames.length; i++) {
        String fname = fieldnames[i];
        // to skip a field, leave the entries in fields and addrs null
        if (fname.length() == 0 || (skipFields != null && skipFields.contains(fname)))
            continue;
        boolean keepEmpty = params.getFieldBool(fname, EMPTY, false);
        adders[i] = keepEmpty ? adderKeepEmpty : adder;
        // Order that operations are applied: split -> trim -> map -> add
        // so create in reverse order.
        // Creation of FieldAdders could be optimized and shared among fields
        String[] fmap = params.getFieldParams(fname, MAP);
        if (fmap != null) {
            for (String mapRule : fmap) {
                String[] mapArgs = colonSplit.split(mapRule, -1);
                if (mapArgs.length != 2)
                    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Map rules must be of the form 'from:to' ,got '" + mapRule + "'");
                adders[i] = new CSVLoaderBase.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]);
            }
        }
        if (params.getFieldBool(fname, TRIM, false)) {
            adders[i] = new CSVLoaderBase.FieldTrimmer(adders[i]);
        }
        if (params.getFieldBool(fname, SPLIT, false)) {
            String sepStr = params.getFieldParam(fname, SEPARATOR);
            char fsep = sepStr == null || sepStr.length() == 0 ? ',' : sepStr.charAt(0);
            String encStr = params.getFieldParam(fname, ENCAPSULATOR);
            char fenc = encStr == null || encStr.length() == 0 ? (char) -2 : encStr.charAt(0);
            String escStr = params.getFieldParam(fname, ESCAPE);
            char fesc = escStr == null || escStr.length() == 0 ? CSVStrategy.ESCAPE_DISABLED : escStr.charAt(0);
            CSVStrategy fstrat = new CSVStrategy(fsep, fenc, CSVStrategy.COMMENTS_DISABLED, fesc, false, false, false, false, "\n");
            adders[i] = new CSVLoaderBase.FieldSplitter(fstrat, adders[i]);
        }
    }
    // look for any literal fields - literal.foo=xyzzy
    Iterator<String> paramNames = params.getParameterNamesIterator();
    while (paramNames.hasNext()) {
        String pname = paramNames.next();
        if (!pname.startsWith(LITERALS_PREFIX))
            continue;
        String name = pname.substring(LITERALS_PREFIX.length());
        literals.put(name, params.get(pname));
    }
}