Examples with ContentStreamBase org.apache.solr.common.util.ContentStreamBase used on opensource projects

Example 1 with ContentStreamBase

use of org.apache.solr.common.util.ContentStreamBase in project lucene-solr by apache.

the class DataImportHandler method handleRequestBody.

@Override
@SuppressWarnings("unchecked")
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
    rsp.setHttpCaching(false);
    //TODO: figure out why just the first one is OK...
    ContentStream contentStream = null;
    Iterable<ContentStream> streams = req.getContentStreams();
    if (streams != null) {
        for (ContentStream stream : streams) {
            contentStream = stream;
            break;
        }
    }
    SolrParams params = req.getParams();
    NamedList defaultParams = (NamedList) initArgs.get("defaults");
    RequestInfo requestParams = new RequestInfo(req, getParamsMap(params), contentStream);
    String command = requestParams.getCommand();
    if (DataImporter.SHOW_CONF_CMD.equals(command)) {
        String dataConfigFile = params.get("config");
        String dataConfig = params.get("dataConfig");
        if (dataConfigFile != null) {
            dataConfig = SolrWriter.getResourceAsString(req.getCore().getResourceLoader().openResource(dataConfigFile));
        }
        if (dataConfig == null) {
            rsp.add("status", DataImporter.MSG.NO_CONFIG_FOUND);
        } else {
            // Modify incoming request params to add wt=raw
            ModifiableSolrParams rawParams = new ModifiableSolrParams(req.getParams());
            rawParams.set(CommonParams.WT, "raw");
            req.setParams(rawParams);
            ContentStreamBase content = new ContentStreamBase.StringStream(dataConfig);
            rsp.add(RawResponseWriter.CONTENT, content);
        }
        return;
    }
    rsp.add("initArgs", initArgs);
    String message = "";
    if (command != null) {
        rsp.add("command", command);
    }
    // If importer is still null
    if (importer == null) {
        rsp.add("status", DataImporter.MSG.NO_INIT);
        return;
    }
    if (command != null && DataImporter.ABORT_CMD.equals(command)) {
        importer.runCmd(requestParams, null);
    } else if (importer.isBusy()) {
        message = DataImporter.MSG.CMD_RUNNING;
    } else if (command != null) {
        if (DataImporter.FULL_IMPORT_CMD.equals(command) || DataImporter.DELTA_IMPORT_CMD.equals(command) || IMPORT_CMD.equals(command)) {
            importer.maybeReloadConfiguration(requestParams, defaultParams);
            UpdateRequestProcessorChain processorChain = req.getCore().getUpdateProcessorChain(params);
            UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
            SolrResourceLoader loader = req.getCore().getResourceLoader();
            DIHWriter sw = getSolrWriter(processor, loader, requestParams, req);
            if (requestParams.isDebug()) {
                if (debugEnabled) {
                    // Synchronous request for the debug mode
                    importer.runCmd(requestParams, sw);
                    rsp.add("mode", "debug");
                    rsp.add("documents", requestParams.getDebugInfo().debugDocuments);
                    if (requestParams.getDebugInfo().debugVerboseOutput != null) {
                        rsp.add("verbose-output", requestParams.getDebugInfo().debugVerboseOutput);
                    }
                } else {
                    message = DataImporter.MSG.DEBUG_NOT_ENABLED;
                }
            } else {
                // Asynchronous request for normal mode
                if (requestParams.getContentStream() == null && !requestParams.isSyncMode()) {
                    importer.runAsync(requestParams, sw);
                } else {
                    importer.runCmd(requestParams, sw);
                }
            }
        } else if (DataImporter.RELOAD_CONF_CMD.equals(command)) {
            if (importer.maybeReloadConfiguration(requestParams, defaultParams)) {
                message = DataImporter.MSG.CONFIG_RELOADED;
            } else {
                message = DataImporter.MSG.CONFIG_NOT_RELOADED;
            }
        }
    }
    rsp.add("status", importer.isBusy() ? "busy" : "idle");
    rsp.add("importResponse", message);
    rsp.add("statusMessages", importer.getStatusMessages());
}

Example 2 with ContentStreamBase

use of org.apache.solr.common.util.ContentStreamBase in project lucene-solr by apache.

the class SolrRequestParsers method buildRequestFrom.

private SolrQueryRequest buildRequestFrom(SolrCore core, SolrParams params, Collection<ContentStream> streams, RTimerTree requestTimer, final HttpServletRequest req) throws Exception {
    // The content type will be applied to all streaming content
    String contentType = params.get(CommonParams.STREAM_CONTENTTYPE);
    // Handle anything with a remoteURL
    String[] strs = params.getParams(CommonParams.STREAM_URL);
    if (strs != null) {
        if (!enableRemoteStreams) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Remote Streaming is disabled.");
        }
        for (final String url : strs) {
            ContentStreamBase stream = new ContentStreamBase.URLStream(new URL(url));
            if (contentType != null) {
                stream.setContentType(contentType);
            }
            streams.add(stream);
        }
    }
    // Handle streaming files
    strs = params.getParams(CommonParams.STREAM_FILE);
    if (strs != null) {
        if (!enableRemoteStreams) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Remote Streaming is disabled.");
        }
        for (final String file : strs) {
            ContentStreamBase stream = new ContentStreamBase.FileStream(new File(file));
            if (contentType != null) {
                stream.setContentType(contentType);
            }
            streams.add(stream);
        }
    }
    // Check for streams in the request parameters
    strs = params.getParams(CommonParams.STREAM_BODY);
    if (strs != null) {
        for (final String body : strs) {
            ContentStreamBase stream = new ContentStreamBase.StringStream(body);
            if (contentType != null) {
                stream.setContentType(contentType);
            }
            streams.add(stream);
        }
    }
    final HttpSolrCall httpSolrCall = req == null ? null : (HttpSolrCall) req.getAttribute(HttpSolrCall.class.getName());
    SolrQueryRequestBase q = new SolrQueryRequestBase(core, params, requestTimer) {

        @Override
        public Principal getUserPrincipal() {
            return req == null ? null : req.getUserPrincipal();
        }

        @Override
        public List<CommandOperation> getCommands(boolean validateInput) {
            if (httpSolrCall != null) {
                return httpSolrCall.getCommands(validateInput);
            }
            return super.getCommands(validateInput);
        }

        @Override
        public Map<String, String> getPathTemplateValues() {
            if (httpSolrCall != null && httpSolrCall instanceof V2HttpCall) {
                return ((V2HttpCall) httpSolrCall).getUrlParts();
            }
            return super.getPathTemplateValues();
        }

        @Override
        public HttpSolrCall getHttpSolrCall() {
            return httpSolrCall;
        }
    };
    if (streams != null && streams.size() > 0) {
        q.setContentStreams(streams);
    }
    return q;
}

Example 3 with ContentStreamBase

use of org.apache.solr.common.util.ContentStreamBase in project lucene-solr by apache.

the class AutoCommitTest method toContentStreams.

/**
   * Take a string and make it an iterable ContentStream
   * 
   * This should be moved to a helper class. (it is useful for the client too!)
   */
public static Collection<ContentStream> toContentStreams(final String str, final String contentType) {
    ArrayList<ContentStream> streams = new ArrayList<>();
    ContentStreamBase stream = new ContentStreamBase.StringStream(str);
    if (contentType != null)
        stream.setContentType(contentType);
    streams.add(stream);
    return streams;
}

Example 4 with ContentStreamBase

use of org.apache.solr.common.util.ContentStreamBase in project lucene-solr by apache.

the class ClientUtils method toContentStreams.

/**
   * Take a string and make it an iterable ContentStream
   */
public static Collection<ContentStream> toContentStreams(final String str, final String contentType) {
    if (str == null)
        return null;
    ArrayList<ContentStream> streams = new ArrayList<>(1);
    ContentStreamBase ccc = new ContentStreamBase.StringStream(str);
    ccc.setContentType(contentType);
    streams.add(ccc);
    return streams;
}

Example 5 with ContentStreamBase

use of org.apache.solr.common.util.ContentStreamBase in project nifi by apache.

the class PutSolrContentStream method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final AtomicReference<Exception> error = new AtomicReference<>(null);
    final AtomicReference<Exception> connectionError = new AtomicReference<>(null);
    final boolean isSolrCloud = SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue());
    final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions(flowFile).getValue();
    final Long commitWithin = context.getProperty(COMMIT_WITHIN).evaluateAttributeExpressions(flowFile).asLong();
    final String contentStreamPath = context.getProperty(CONTENT_STREAM_PATH).evaluateAttributeExpressions(flowFile).getValue();
    final MultiMapSolrParams requestParams = new MultiMapSolrParams(getRequestParams(context, flowFile));
    StopWatch timer = new StopWatch(true);
    session.read(flowFile, new InputStreamCallback() {

        @Override
        public void process(final InputStream in) throws IOException {
            ContentStreamUpdateRequest request = new ContentStreamUpdateRequest(contentStreamPath);
            request.setParams(new ModifiableSolrParams());
            // add the extra params, don't use 'set' in case of repeating params
            Iterator<String> paramNames = requestParams.getParameterNamesIterator();
            while (paramNames.hasNext()) {
                String paramName = paramNames.next();
                for (String paramValue : requestParams.getParams(paramName)) {
                    request.getParams().add(paramName, paramValue);
                }
            }
            // specify the collection for SolrCloud
            if (isSolrCloud) {
                request.setParam(COLLECTION_PARAM_NAME, collection);
            }
            if (commitWithin != null && commitWithin > 0) {
                request.setParam(COMMIT_WITHIN_PARAM_NAME, commitWithin.toString());
            }
            // if a username and password were provided then pass them for basic auth
            if (isBasicAuthEnabled()) {
                request.setBasicAuthCredentials(getUsername(), getPassword());
            }
            try (final BufferedInputStream bufferedIn = new BufferedInputStream(in)) {
                // add the FlowFile's content on the UpdateRequest
                request.addContentStream(new ContentStreamBase() {

                    @Override
                    public InputStream getStream() throws IOException {
                        return bufferedIn;
                    }

                    @Override
                    public String getContentType() {
                        return context.getProperty(CONTENT_TYPE).evaluateAttributeExpressions().getValue();
                    }
                });
                UpdateResponse response = request.process(getSolrClient());
                getLogger().debug("Got {} response from Solr", new Object[] { response.getStatus() });
            } catch (SolrException e) {
                error.set(e);
            } catch (SolrServerException e) {
                if (causedByIOException(e)) {
                    connectionError.set(e);
                } else {
                    error.set(e);
                }
            } catch (IOException e) {
                connectionError.set(e);
            }
        }
    });
    timer.stop();
    if (error.get() != null) {
        getLogger().error("Failed to send {} to Solr due to {}; routing to failure", new Object[] { flowFile, error.get() });
        session.transfer(flowFile, REL_FAILURE);
    } else if (connectionError.get() != null) {
        getLogger().error("Failed to send {} to Solr due to {}; routing to connection_failure", new Object[] { flowFile, connectionError.get() });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_CONNECTION_FAILURE);
    } else {
        StringBuilder transitUri = new StringBuilder("solr://");
        transitUri.append(getSolrLocation());
        if (isSolrCloud) {
            transitUri.append(":").append(collection);
        }
        final long duration = timer.getDuration(TimeUnit.MILLISECONDS);
        session.getProvenanceReporter().send(flowFile, transitUri.toString(), duration, true);
        getLogger().info("Successfully sent {} to Solr in {} millis", new Object[] { flowFile, duration });
        session.transfer(flowFile, REL_SUCCESS);
    }
}