Examples with ProcessException org.apache.nifi.processor.exception.ProcessException used on opensource projects

Example 96 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class GetSolr method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    final AtomicBoolean continuePaging = new AtomicBoolean(true);
    final SolrQuery solrQuery = new SolrQuery();
    try {
        if (id_field == null) {
            id_field = getFieldNameOfUniqueKey();
        }
        final String dateField = context.getProperty(DATE_FIELD).getValue();
        final Map<String, String> stateMap = new HashMap<String, String>();
        stateMap.putAll(context.getStateManager().getState(Scope.CLUSTER).toMap());
        solrQuery.setQuery("*:*");
        final String query = context.getProperty(SOLR_QUERY).getValue();
        if (!StringUtils.isBlank(query) && !query.equals("*:*")) {
            solrQuery.addFilterQuery(query);
        }
        final StringBuilder automatedFilterQuery = (new StringBuilder()).append(dateField).append(":[").append(stateMap.get(STATE_MANAGER_FILTER)).append(" TO *]");
        solrQuery.addFilterQuery(automatedFilterQuery.toString());
        final List<String> fieldList = new ArrayList<String>();
        final String returnFields = context.getProperty(RETURN_FIELDS).getValue();
        if (!StringUtils.isBlank(returnFields)) {
            fieldList.addAll(Arrays.asList(returnFields.trim().split("[,]")));
            if (!fieldList.contains(dateField)) {
                fieldList.add(dateField);
                dateFieldNotInSpecifiedFieldsList.set(true);
            }
            for (String returnField : fieldList) {
                solrQuery.addField(returnField.trim());
            }
        }
        solrQuery.setParam(CursorMarkParams.CURSOR_MARK_PARAM, stateMap.get(STATE_MANAGER_CURSOR_MARK));
        solrQuery.setRows(context.getProperty(BATCH_SIZE).asInteger());
        final StringBuilder sortClause = (new StringBuilder()).append(dateField).append(" asc,").append(id_field).append(" asc");
        solrQuery.setParam("sort", sortClause.toString());
        while (continuePaging.get()) {
            final QueryRequest req = new QueryRequest(solrQuery);
            if (isBasicAuthEnabled()) {
                req.setBasicAuthCredentials(getUsername(), getPassword());
            }
            logger.debug(solrQuery.toQueryString());
            final QueryResponse response = req.process(getSolrClient());
            final SolrDocumentList documentList = response.getResults();
            if (response.getResults().size() > 0) {
                final SolrDocument lastSolrDocument = documentList.get(response.getResults().size() - 1);
                final String latestDateValue = df.format(lastSolrDocument.get(dateField));
                final String newCursorMark = response.getNextCursorMark();
                solrQuery.setParam(CursorMarkParams.CURSOR_MARK_PARAM, newCursorMark);
                stateMap.put(STATE_MANAGER_CURSOR_MARK, newCursorMark);
                stateMap.put(STATE_MANAGER_FILTER, latestDateValue);
                FlowFile flowFile = session.create();
                flowFile = session.putAttribute(flowFile, "solrQuery", solrQuery.toString());
                if (context.getProperty(RETURN_TYPE).getValue().equals(MODE_XML.getValue())) {
                    if (dateFieldNotInSpecifiedFieldsList.get()) {
                        for (SolrDocument doc : response.getResults()) {
                            doc.removeFields(dateField);
                        }
                    }
                    flowFile = session.write(flowFile, SolrUtils.getOutputStreamCallbackToTransformSolrResponseToXml(response));
                    flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/xml");
                } else {
                    final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
                    final RecordSchema schema = writerFactory.getSchema(null, null);
                    final RecordSet recordSet = SolrUtils.solrDocumentsToRecordSet(response.getResults(), schema);
                    final StringBuffer mimeType = new StringBuffer();
                    flowFile = session.write(flowFile, new OutputStreamCallback() {

                        @Override
                        public void process(final OutputStream out) throws IOException {
                            try {
                                final RecordSetWriter writer = writerFactory.createWriter(getLogger(), schema, out);
                                writer.write(recordSet);
                                writer.flush();
                                mimeType.append(writer.getMimeType());
                            } catch (SchemaNotFoundException e) {
                                throw new ProcessException("Could not parse Solr response", e);
                            }
                        }
                    });
                    flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), mimeType.toString());
                }
                session.transfer(flowFile, REL_SUCCESS);
            }
            continuePaging.set(response.getResults().size() == Integer.parseInt(context.getProperty(BATCH_SIZE).getValue()));
        }
        context.getStateManager().setState(stateMap, Scope.CLUSTER);
    } catch (SolrServerException | SchemaNotFoundException | IOException e) {
        context.yield();
        session.rollback();
        logger.error("Failed to execute query {} due to {}", new Object[] { solrQuery.toString(), e }, e);
        throw new ProcessException(e);
    } catch (final Throwable t) {
        context.yield();
        session.rollback();
        logger.error("Failed to execute query {} due to {}", new Object[] { solrQuery.toString(), t }, t);
        throw t;
    }
}

Example 97 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class PutSolrContentStream method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final AtomicReference<Exception> error = new AtomicReference<>(null);
    final AtomicReference<Exception> connectionError = new AtomicReference<>(null);
    final boolean isSolrCloud = SOLR_TYPE_CLOUD.equals(context.getProperty(SOLR_TYPE).getValue());
    final String collection = context.getProperty(COLLECTION).evaluateAttributeExpressions(flowFile).getValue();
    final Long commitWithin = context.getProperty(COMMIT_WITHIN).evaluateAttributeExpressions(flowFile).asLong();
    final String contentStreamPath = context.getProperty(CONTENT_STREAM_PATH).evaluateAttributeExpressions(flowFile).getValue();
    final MultiMapSolrParams requestParams = new MultiMapSolrParams(getRequestParams(context, flowFile));
    StopWatch timer = new StopWatch(true);
    session.read(flowFile, new InputStreamCallback() {

        @Override
        public void process(final InputStream in) throws IOException {
            ContentStreamUpdateRequest request = new ContentStreamUpdateRequest(contentStreamPath);
            request.setParams(new ModifiableSolrParams());
            // add the extra params, don't use 'set' in case of repeating params
            Iterator<String> paramNames = requestParams.getParameterNamesIterator();
            while (paramNames.hasNext()) {
                String paramName = paramNames.next();
                for (String paramValue : requestParams.getParams(paramName)) {
                    request.getParams().add(paramName, paramValue);
                }
            }
            // specify the collection for SolrCloud
            if (isSolrCloud) {
                request.setParam(COLLECTION_PARAM_NAME, collection);
            }
            if (commitWithin != null && commitWithin > 0) {
                request.setParam(COMMIT_WITHIN_PARAM_NAME, commitWithin.toString());
            }
            // if a username and password were provided then pass them for basic auth
            if (isBasicAuthEnabled()) {
                request.setBasicAuthCredentials(getUsername(), getPassword());
            }
            try (final BufferedInputStream bufferedIn = new BufferedInputStream(in)) {
                // add the FlowFile's content on the UpdateRequest
                request.addContentStream(new ContentStreamBase() {

                    @Override
                    public InputStream getStream() throws IOException {
                        return bufferedIn;
                    }

                    @Override
                    public String getContentType() {
                        return context.getProperty(CONTENT_TYPE).evaluateAttributeExpressions().getValue();
                    }
                });
                UpdateResponse response = request.process(getSolrClient());
                getLogger().debug("Got {} response from Solr", new Object[] { response.getStatus() });
            } catch (SolrException e) {
                error.set(e);
            } catch (SolrServerException e) {
                if (causedByIOException(e)) {
                    connectionError.set(e);
                } else {
                    error.set(e);
                }
            } catch (IOException e) {
                connectionError.set(e);
            }
        }
    });
    timer.stop();
    if (error.get() != null) {
        getLogger().error("Failed to send {} to Solr due to {}; routing to failure", new Object[] { flowFile, error.get() });
        session.transfer(flowFile, REL_FAILURE);
    } else if (connectionError.get() != null) {
        getLogger().error("Failed to send {} to Solr due to {}; routing to connection_failure", new Object[] { flowFile, connectionError.get() });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_CONNECTION_FAILURE);
    } else {
        StringBuilder transitUri = new StringBuilder("solr://");
        transitUri.append(getSolrLocation());
        if (isSolrCloud) {
            transitUri.append(":").append(collection);
        }
        final long duration = timer.getDuration(TimeUnit.MILLISECONDS);
        session.getProvenanceReporter().send(flowFile, transitUri.toString(), duration, true);
        getLogger().info("Successfully sent {} to Solr in {} millis", new Object[] { flowFile, duration });
        session.transfer(flowFile, REL_SUCCESS);
    }
}

Example 98 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class ExecuteSparkInteractive method onTrigger.

@Override
public void onTrigger(ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog log = getLogger();
    final LivySessionService livySessionService = context.getProperty(LIVY_CONTROLLER_SERVICE).asControllerService(LivySessionService.class);
    final Map<String, String> livyController = livySessionService.getSession();
    if (livyController == null || livyController.isEmpty()) {
        log.debug("No Spark session available (yet), routing flowfile to wait");
        session.transfer(flowFile, REL_WAIT);
        return;
    }
    final long statusCheckInterval = context.getProperty(STATUS_CHECK_INTERVAL).evaluateAttributeExpressions(flowFile).asTimePeriod(TimeUnit.MILLISECONDS);
    Charset charset;
    try {
        charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    } catch (Exception e) {
        log.warn("Illegal character set name specified, defaulting to UTF-8");
        charset = StandardCharsets.UTF_8;
    }
    String sessionId = livyController.get("sessionId");
    String livyUrl = livyController.get("livyUrl");
    String code = context.getProperty(CODE).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(code)) {
        try (InputStream inputStream = session.read(flowFile)) {
            // If no code was provided, assume it is in the content of the incoming flow file
            code = IOUtils.toString(inputStream, charset);
        } catch (IOException ioe) {
            log.error("Error reading input flowfile, penalizing and routing to failure", new Object[] { flowFile, ioe.getMessage() }, ioe);
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
    }
    code = StringEscapeUtils.escapeJavaScript(code);
    String payload = "{\"code\":\"" + code + "\"}";
    try {
        final JSONObject result = submitAndHandleJob(livyUrl, livySessionService, sessionId, payload, statusCheckInterval);
        log.debug("ExecuteSparkInteractive Result of Job Submit: " + result);
        if (result == null) {
            session.transfer(flowFile, REL_FAILURE);
        } else {
            try {
                final JSONObject output = result.getJSONObject("data");
                flowFile = session.write(flowFile, out -> out.write(output.toString().getBytes()));
                flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), LivySessionService.APPLICATION_JSON);
                session.transfer(flowFile, REL_SUCCESS);
            } catch (JSONException je) {
                // The result doesn't contain the data, just send the output object as the flow file content to failure (after penalizing)
                log.error("Spark Session returned an error, sending the output JSON object as the flow file content to failure (after penalizing)");
                flowFile = session.write(flowFile, out -> out.write(result.toString().getBytes()));
                flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), LivySessionService.APPLICATION_JSON);
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
        }
    } catch (IOException ioe) {
        log.error("Failure processing flowfile {} due to {}, penalizing and routing to failure", new Object[] { flowFile, ioe.getMessage() }, ioe);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 99 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class GetSplunk method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    final long currentTime = System.currentTimeMillis();
    synchronized (isInitialized) {
        if (!isInitialized.get()) {
            splunkService = createSplunkService(context);
            isInitialized.set(true);
        }
    }
    final String query = context.getProperty(QUERY).getValue();
    final String outputMode = context.getProperty(OUTPUT_MODE).getValue();
    final String timeRangeStrategy = context.getProperty(TIME_RANGE_STRATEGY).getValue();
    final String timeZone = context.getProperty(TIME_ZONE).getValue();
    final String timeFieldStrategy = context.getProperty(TIME_FIELD_STRATEGY).getValue();
    final JobExportArgs exportArgs = new JobExportArgs();
    exportArgs.setSearchMode(JobExportArgs.SearchMode.NORMAL);
    exportArgs.setOutputMode(JobExportArgs.OutputMode.valueOf(outputMode));
    String earliestTime = null;
    String latestTime = null;
    if (PROVIDED_VALUE.getValue().equals(timeRangeStrategy)) {
        // for provided we just use the values of the properties
        earliestTime = context.getProperty(EARLIEST_TIME).getValue();
        latestTime = context.getProperty(LATEST_TIME).getValue();
    } else {
        try {
            // not provided so we need to check the previous state
            final TimeRange previousRange = loadState(context.getStateManager());
            final SimpleDateFormat dateFormat = new SimpleDateFormat(DATE_TIME_FORMAT);
            dateFormat.setTimeZone(TimeZone.getTimeZone(timeZone));
            if (previousRange == null) {
                // no previous state so set the earliest time based on the strategy
                if (MANAGED_CURRENT_VALUE.getValue().equals(timeRangeStrategy)) {
                    earliestTime = dateFormat.format(new Date(currentTime));
                }
                // no previous state so set the latest time to the current time
                latestTime = dateFormat.format(new Date(currentTime));
                // initial time saved and next execution will be the first real execution
                if (latestTime.equals(earliestTime)) {
                    saveState(context.getStateManager(), new TimeRange(earliestTime, latestTime));
                    return;
                }
            } else {
                // we have previous state so set earliestTime to (latestTime + 1) of last range
                try {
                    final String previousLastTime = previousRange.getLatestTime();
                    final Date previousLastDate = dateFormat.parse(previousLastTime);
                    earliestTime = dateFormat.format(new Date(previousLastDate.getTime() + 1));
                    latestTime = dateFormat.format(new Date(currentTime));
                } catch (ParseException e) {
                    throw new ProcessException(e);
                }
            }
        } catch (IOException e) {
            getLogger().error("Unable to load data from State Manager due to {}", new Object[] { e.getMessage() }, e);
            context.yield();
            return;
        }
    }
    if (!StringUtils.isBlank(earliestTime)) {
        if (EVENT_TIME_VALUE.getValue().equalsIgnoreCase(timeFieldStrategy)) {
            exportArgs.setEarliestTime(earliestTime);
        } else {
            exportArgs.setIndexEarliest(earliestTime);
        }
    }
    if (!StringUtils.isBlank(latestTime)) {
        if (EVENT_TIME_VALUE.getValue().equalsIgnoreCase(timeFieldStrategy)) {
            exportArgs.setLatestTime(latestTime);
        } else {
            exportArgs.setIndexLatest(latestTime);
        }
    }
    if (EVENT_TIME_VALUE.getValue().equalsIgnoreCase(timeFieldStrategy)) {
        getLogger().debug("Using earliest_time of {} and latest_time of {}", new Object[] { earliestTime, latestTime });
    } else {
        getLogger().debug("Using index_earliest of {} and index_latest of {}", new Object[] { earliestTime, latestTime });
    }
    InputStream export;
    try {
        export = splunkService.export(query, exportArgs);
    // Catch Stale connection exception, reinitialize, and retry
    } catch (com.splunk.HttpException e) {
        getLogger().error("Splunk request status code:" + e.getStatus() + " Retrying the request.");
        splunkService.logout();
        splunkService = createSplunkService(context);
        export = splunkService.export(query, exportArgs);
    }
    final InputStream exportSearch = export;
    FlowFile flowFile = session.create();
    flowFile = session.write(flowFile, new OutputStreamCallback() {

        @Override
        public void process(OutputStream rawOut) throws IOException {
            try (BufferedOutputStream out = new BufferedOutputStream(rawOut)) {
                IOUtils.copyLarge(exportSearch, out);
            }
        }
    });
    final Map<String, String> attributes = new HashMap<>(3);
    attributes.put(EARLIEST_TIME_ATTR, earliestTime);
    attributes.put(LATEST_TIME_ATTR, latestTime);
    attributes.put(QUERY_ATTR, query);
    flowFile = session.putAllAttributes(flowFile, attributes);
    session.getProvenanceReporter().receive(flowFile, transitUri);
    session.transfer(flowFile, REL_SUCCESS);
    getLogger().debug("Received {} from Splunk", new Object[] { flowFile });
    // only need to do this for the managed time strategies
    if (!PROVIDED_VALUE.getValue().equals(timeRangeStrategy)) {
        try {
            saveState(context.getStateManager(), new TimeRange(earliestTime, latestTime));
        } catch (IOException e) {
            getLogger().error("Unable to load data from State Manager due to {}", new Object[] { e.getMessage() }, e);
            session.rollback();
            context.yield();
        }
    }
}

Example 100 with ProcessException

use of org.apache.nifi.processor.exception.ProcessException in project nifi by apache.

the class DeleteRethinkDB method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    Charset charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    String id = context.getProperty(RETHINKDB_DOCUMENT_ID).evaluateAttributeExpressions(flowFile).getValue();
    String durablity = context.getProperty(DURABILITY).evaluateAttributeExpressions(flowFile).getValue();
    Boolean returnChanges = context.getProperty(RETURN_CHANGES).evaluateAttributeExpressions(flowFile).asBoolean();
    if (StringUtils.isEmpty(id)) {
        getLogger().error(DOCUMENT_ID_EMPTY_MESSAGE);
        flowFile = session.putAttribute(flowFile, RETHINKDB_ERROR_MESSAGE, DOCUMENT_ID_EMPTY_MESSAGE);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    try {
        long startTimeMillis = System.currentTimeMillis();
        Map<String, Object> result = deleteDocument(id, durablity, returnChanges);
        final long endTimeMillis = System.currentTimeMillis();
        getLogger().debug("Json document {} deleted Result: {}", new Object[] { id, result });
        flowFile = populateAttributes(session, flowFile, result);
        Long deletedCount = ((Long) result.get(RESULT_DELETED_KEY)).longValue();
        if (deletedCount == 0L) {
            getLogger().debug("Deleted count should be 1 but was " + deletedCount + " for document with id '" + id + "'");
            flowFile = populateAttributes(session, flowFile, result);
            flowFile = session.putAttribute(flowFile, RETHINKDB_ERROR_MESSAGE, "Deleted count should be 1 but was " + deletedCount + " for document with id '" + id + "'");
            session.transfer(flowFile, REL_NOT_FOUND);
            return;
        }
        if (returnChanges) {
            String json = gson.toJson(((List) result.get(RESULT_CHANGES_KEY)).get(0));
            byte[] documentBytes = json.getBytes(charset);
            ByteArrayInputStream bais = new ByteArrayInputStream(documentBytes);
            session.importFrom(bais, flowFile);
            session.getProvenanceReporter().modifyContent(flowFile, new StringBuilder("rethinkdb://").append(databaseName).append("/").append(tableName).append("/").append(id).toString(), (endTimeMillis - startTimeMillis));
        }
        session.transfer(flowFile, REL_SUCCESS);
    } catch (Exception exception) {
        getLogger().error("Failed to delete document from RethinkDB due to error {}", new Object[] { exception.getLocalizedMessage() }, exception);
        flowFile = session.putAttribute(flowFile, RETHINKDB_ERROR_MESSAGE, exception.getMessage());
        session.transfer(flowFile, REL_FAILURE);
        context.yield();
    }
}