Examples with ComponentLog org.apache.nifi.logging.ComponentLog used on opensource projects

Example 71 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class GetFileTransfer method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final long pollingIntervalMillis = context.getProperty(FileTransfer.POLLING_INTERVAL).asTimePeriod(TimeUnit.MILLISECONDS);
    final long nextPollTime = lastPollTime.get() + pollingIntervalMillis;
    BlockingQueue<FileInfo> fileQueue = fileQueueRef.get();
    final ComponentLog logger = getLogger();
    // do not do the listing if there are already 100 or more items in our queue
    // 100 is really just a magic number that seems to work out well in practice
    FileTransfer transfer = null;
    if (System.currentTimeMillis() >= nextPollTime && (fileQueue == null || fileQueue.size() < 100) && listingLock.tryLock()) {
        try {
            transfer = getFileTransfer(context);
            try {
                fetchListing(context, session, transfer);
                lastPollTime.set(System.currentTimeMillis());
            } catch (final IOException e) {
                context.yield();
                try {
                    transfer.close();
                } catch (final IOException e1) {
                    logger.warn("Unable to close connection due to {}", new Object[] { e1 });
                }
                logger.error("Unable to fetch listing from remote server due to {}", new Object[] { e });
                return;
            }
        } finally {
            listingLock.unlock();
        }
    }
    fileQueue = fileQueueRef.get();
    if (fileQueue == null || fileQueue.isEmpty()) {
        // nothing to do!
        context.yield();
        if (transfer != null) {
            try {
                transfer.close();
            } catch (final IOException e1) {
                logger.warn("Unable to close connection due to {}", new Object[] { e1 });
            }
        }
        return;
    }
    final String hostname = context.getProperty(FileTransfer.HOSTNAME).evaluateAttributeExpressions().getValue();
    final boolean deleteOriginal = context.getProperty(FileTransfer.DELETE_ORIGINAL).asBoolean();
    final int maxSelects = context.getProperty(FileTransfer.MAX_SELECTS).asInteger();
    if (transfer == null) {
        transfer = getFileTransfer(context);
    }
    try {
        for (int i = 0; i < maxSelects && isScheduled(); i++) {
            final FileInfo file;
            sharableTransferLock.lock();
            try {
                file = fileQueue.poll();
                if (file == null) {
                    return;
                }
                processing.add(file);
            } finally {
                sharableTransferLock.unlock();
            }
            File relativeFile = new File(file.getFullPathFileName());
            final String parentRelativePath = (null == relativeFile.getParent()) ? "" : relativeFile.getParent();
            final String parentRelativePathString = parentRelativePath + "/";
            final Path absPath = relativeFile.toPath().toAbsolutePath();
            final String absPathString = absPath.getParent().toString() + "/";
            try {
                FlowFile flowFile = session.create();
                final StopWatch stopWatch = new StopWatch(false);
                try (final InputStream in = transfer.getInputStream(file.getFullPathFileName())) {
                    stopWatch.start();
                    flowFile = session.importFrom(in, flowFile);
                    stopWatch.stop();
                }
                transfer.flush();
                final long millis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
                final String dataRate = stopWatch.calculateDataRate(flowFile.getSize());
                flowFile = session.putAttribute(flowFile, this.getClass().getSimpleName().toLowerCase() + ".remote.source", hostname);
                flowFile = session.putAttribute(flowFile, CoreAttributes.PATH.key(), parentRelativePathString);
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), relativeFile.getName());
                flowFile = session.putAttribute(flowFile, CoreAttributes.ABSOLUTE_PATH.key(), absPathString);
                Map<String, String> attributes = getAttributesFromFile(file);
                if (attributes.size() > 0) {
                    flowFile = session.putAllAttributes(flowFile, attributes);
                }
                if (deleteOriginal) {
                    try {
                        transfer.deleteFile(flowFile, null, file.getFullPathFileName());
                    } catch (final IOException e) {
                        logger.error("Failed to remove remote file {} due to {}; deleting local copy", new Object[] { file.getFullPathFileName(), e });
                        session.remove(flowFile);
                        return;
                    }
                }
                session.getProvenanceReporter().receive(flowFile, transfer.getProtocolName() + "://" + hostname + "/" + file.getFullPathFileName(), millis);
                session.transfer(flowFile, REL_SUCCESS);
                logger.info("Successfully retrieved {} from {} in {} milliseconds at a rate of {} and transferred to success", new Object[] { flowFile, hostname, millis, dataRate });
                session.commit();
            } catch (final IOException e) {
                context.yield();
                logger.error("Unable to retrieve file {} due to {}", new Object[] { file.getFullPathFileName(), e });
                try {
                    transfer.close();
                } catch (IOException e1) {
                    logger.warn("Unable to close connection to remote host due to {}", new Object[] { e1 });
                }
                session.rollback();
                return;
            } catch (final FlowFileAccessException e) {
                context.yield();
                logger.error("Unable to retrieve file {} due to {}", new Object[] { file.getFullPathFileName(), e.getCause() }, e);
                try {
                    transfer.close();
                } catch (IOException e1) {
                    logger.warn("Unable to close connection to remote host due to {}", e1);
                }
                session.rollback();
                return;
            } finally {
                processing.remove(file);
            }
        }
    } finally {
        try {
            transfer.close();
        } catch (final IOException e) {
            logger.warn("Failed to close connection to {} due to {}", new Object[] { hostname, e });
        }
    }
}

Example 72 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class GetSolr method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final ComponentLog logger = getLogger();
    final AtomicBoolean continuePaging = new AtomicBoolean(true);
    final SolrQuery solrQuery = new SolrQuery();
    try {
        if (id_field == null) {
            id_field = getFieldNameOfUniqueKey();
        }
        final String dateField = context.getProperty(DATE_FIELD).getValue();
        final Map<String, String> stateMap = new HashMap<String, String>();
        stateMap.putAll(context.getStateManager().getState(Scope.CLUSTER).toMap());
        solrQuery.setQuery("*:*");
        final String query = context.getProperty(SOLR_QUERY).getValue();
        if (!StringUtils.isBlank(query) && !query.equals("*:*")) {
            solrQuery.addFilterQuery(query);
        }
        final StringBuilder automatedFilterQuery = (new StringBuilder()).append(dateField).append(":[").append(stateMap.get(STATE_MANAGER_FILTER)).append(" TO *]");
        solrQuery.addFilterQuery(automatedFilterQuery.toString());
        final List<String> fieldList = new ArrayList<String>();
        final String returnFields = context.getProperty(RETURN_FIELDS).getValue();
        if (!StringUtils.isBlank(returnFields)) {
            fieldList.addAll(Arrays.asList(returnFields.trim().split("[,]")));
            if (!fieldList.contains(dateField)) {
                fieldList.add(dateField);
                dateFieldNotInSpecifiedFieldsList.set(true);
            }
            for (String returnField : fieldList) {
                solrQuery.addField(returnField.trim());
            }
        }
        solrQuery.setParam(CursorMarkParams.CURSOR_MARK_PARAM, stateMap.get(STATE_MANAGER_CURSOR_MARK));
        solrQuery.setRows(context.getProperty(BATCH_SIZE).asInteger());
        final StringBuilder sortClause = (new StringBuilder()).append(dateField).append(" asc,").append(id_field).append(" asc");
        solrQuery.setParam("sort", sortClause.toString());
        while (continuePaging.get()) {
            final QueryRequest req = new QueryRequest(solrQuery);
            if (isBasicAuthEnabled()) {
                req.setBasicAuthCredentials(getUsername(), getPassword());
            }
            logger.debug(solrQuery.toQueryString());
            final QueryResponse response = req.process(getSolrClient());
            final SolrDocumentList documentList = response.getResults();
            if (response.getResults().size() > 0) {
                final SolrDocument lastSolrDocument = documentList.get(response.getResults().size() - 1);
                final String latestDateValue = df.format(lastSolrDocument.get(dateField));
                final String newCursorMark = response.getNextCursorMark();
                solrQuery.setParam(CursorMarkParams.CURSOR_MARK_PARAM, newCursorMark);
                stateMap.put(STATE_MANAGER_CURSOR_MARK, newCursorMark);
                stateMap.put(STATE_MANAGER_FILTER, latestDateValue);
                FlowFile flowFile = session.create();
                flowFile = session.putAttribute(flowFile, "solrQuery", solrQuery.toString());
                if (context.getProperty(RETURN_TYPE).getValue().equals(MODE_XML.getValue())) {
                    if (dateFieldNotInSpecifiedFieldsList.get()) {
                        for (SolrDocument doc : response.getResults()) {
                            doc.removeFields(dateField);
                        }
                    }
                    flowFile = session.write(flowFile, SolrUtils.getOutputStreamCallbackToTransformSolrResponseToXml(response));
                    flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), "application/xml");
                } else {
                    final RecordSetWriterFactory writerFactory = context.getProperty(RECORD_WRITER).asControllerService(RecordSetWriterFactory.class);
                    final RecordSchema schema = writerFactory.getSchema(null, null);
                    final RecordSet recordSet = SolrUtils.solrDocumentsToRecordSet(response.getResults(), schema);
                    final StringBuffer mimeType = new StringBuffer();
                    flowFile = session.write(flowFile, new OutputStreamCallback() {

                        @Override
                        public void process(final OutputStream out) throws IOException {
                            try {
                                final RecordSetWriter writer = writerFactory.createWriter(getLogger(), schema, out);
                                writer.write(recordSet);
                                writer.flush();
                                mimeType.append(writer.getMimeType());
                            } catch (SchemaNotFoundException e) {
                                throw new ProcessException("Could not parse Solr response", e);
                            }
                        }
                    });
                    flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), mimeType.toString());
                }
                session.transfer(flowFile, REL_SUCCESS);
            }
            continuePaging.set(response.getResults().size() == Integer.parseInt(context.getProperty(BATCH_SIZE).getValue()));
        }
        context.getStateManager().setState(stateMap, Scope.CLUSTER);
    } catch (SolrServerException | SchemaNotFoundException | IOException e) {
        context.yield();
        session.rollback();
        logger.error("Failed to execute query {} due to {}", new Object[] { solrQuery.toString(), e }, e);
        throw new ProcessException(e);
    } catch (final Throwable t) {
        context.yield();
        session.rollback();
        logger.error("Failed to execute query {} due to {}", new Object[] { solrQuery.toString(), t }, t);
        throw t;
    }
}

Example 73 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class LivySessionController method manageSessions.

private void manageSessions() throws InterruptedException, IOException {
    int idleSessions = 0;
    JSONObject newSessionInfo;
    Map<Integer, JSONObject> sessionsInfo;
    ComponentLog log = getLogger();
    try {
        sessionsInfo = listSessions();
        if (sessions.isEmpty()) {
            log.debug("manageSessions() the active session list is empty, populating from acquired list...");
            sessions.putAll(sessionsInfo);
        }
        for (Integer sessionId : new ArrayList<>(sessions.keySet())) {
            JSONObject currentSession = sessions.get(sessionId);
            log.debug("manageSessions() Updating current session: " + currentSession);
            if (sessionsInfo.containsKey(sessionId)) {
                String state = currentSession.getString("state");
                String sessionKind = currentSession.getString("kind");
                log.debug("manageSessions() controller kind: {}, session kind: {}, session state: {}", new Object[] { controllerKind, sessionKind, state });
                if (state.equalsIgnoreCase("idle") && sessionKind.equalsIgnoreCase(controllerKind)) {
                    // Keep track of how many sessions are in an idle state and thus available
                    idleSessions++;
                    sessions.put(sessionId, sessionsInfo.get(sessionId));
                    // Remove session from session list source of truth snapshot since it has been dealt with
                    sessionsInfo.remove(sessionId);
                } else if ((state.equalsIgnoreCase("busy") || state.equalsIgnoreCase("starting")) && sessionKind.equalsIgnoreCase(controllerKind)) {
                    // Update status of existing sessions
                    sessions.put(sessionId, sessionsInfo.get(sessionId));
                    // Remove session from session list source of truth snapshot since it has been dealt with
                    sessionsInfo.remove(sessionId);
                } else {
                    // Prune sessions of kind != controllerKind and whose state is:
                    // not_started, shutting_down, error, dead, success (successfully stopped)
                    sessions.remove(sessionId);
                    // Remove session from session list source of truth snapshot since it has been dealt with
                    sessionsInfo.remove(sessionId);
                }
            } else {
                // Prune sessions that no longer exist
                log.debug("manageSessions() session exists in session pool but not in source snapshot, removing from pool...");
                sessions.remove(sessionId);
                // Remove session from session list source of truth snapshot since it has been dealt with
                sessionsInfo.remove(sessionId);
            }
        }
        int numSessions = sessions.size();
        log.debug("manageSessions() There are " + numSessions + " sessions in the pool");
        // Open new sessions equal to the number requested by sessionPoolSize
        if (numSessions == 0) {
            for (int i = 0; i < sessionPoolSize; i++) {
                newSessionInfo = openSession();
                sessions.put(newSessionInfo.getInt("id"), newSessionInfo);
                log.debug("manageSessions() Registered new session: " + newSessionInfo);
            }
        } else {
            // Open one new session if there are no idle sessions
            if (idleSessions == 0) {
                log.debug("manageSessions() There are " + numSessions + " sessions in the pool but none of them are idle sessions, creating...");
                newSessionInfo = openSession();
                sessions.put(newSessionInfo.getInt("id"), newSessionInfo);
                log.debug("manageSessions() Registered new session: " + newSessionInfo);
            }
            // Open more sessions if number of sessions is less than target pool size
            if (numSessions < sessionPoolSize) {
                log.debug("manageSessions() There are " + numSessions + ", need more sessions to equal requested pool size of " + sessionPoolSize + ", creating...");
                for (int i = 0; i < sessionPoolSize - numSessions; i++) {
                    newSessionInfo = openSession();
                    sessions.put(newSessionInfo.getInt("id"), newSessionInfo);
                    log.debug("manageSessions() Registered new session: " + newSessionInfo);
                }
            }
        }
    } catch (ConnectException | SocketTimeoutException ce) {
        log.error("Timeout connecting to Livy service to retrieve sessions", ce);
    } catch (JSONException e) {
        throw new IOException(e);
    }
}

Example 74 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class ExecuteSparkInteractive method onTrigger.

@Override
public void onTrigger(ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog log = getLogger();
    final LivySessionService livySessionService = context.getProperty(LIVY_CONTROLLER_SERVICE).asControllerService(LivySessionService.class);
    final Map<String, String> livyController = livySessionService.getSession();
    if (livyController == null || livyController.isEmpty()) {
        log.debug("No Spark session available (yet), routing flowfile to wait");
        session.transfer(flowFile, REL_WAIT);
        return;
    }
    final long statusCheckInterval = context.getProperty(STATUS_CHECK_INTERVAL).evaluateAttributeExpressions(flowFile).asTimePeriod(TimeUnit.MILLISECONDS);
    Charset charset;
    try {
        charset = Charset.forName(context.getProperty(CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    } catch (Exception e) {
        log.warn("Illegal character set name specified, defaulting to UTF-8");
        charset = StandardCharsets.UTF_8;
    }
    String sessionId = livyController.get("sessionId");
    String livyUrl = livyController.get("livyUrl");
    String code = context.getProperty(CODE).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(code)) {
        try (InputStream inputStream = session.read(flowFile)) {
            // If no code was provided, assume it is in the content of the incoming flow file
            code = IOUtils.toString(inputStream, charset);
        } catch (IOException ioe) {
            log.error("Error reading input flowfile, penalizing and routing to failure", new Object[] { flowFile, ioe.getMessage() }, ioe);
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
    }
    code = StringEscapeUtils.escapeJavaScript(code);
    String payload = "{\"code\":\"" + code + "\"}";
    try {
        final JSONObject result = submitAndHandleJob(livyUrl, livySessionService, sessionId, payload, statusCheckInterval);
        log.debug("ExecuteSparkInteractive Result of Job Submit: " + result);
        if (result == null) {
            session.transfer(flowFile, REL_FAILURE);
        } else {
            try {
                final JSONObject output = result.getJSONObject("data");
                flowFile = session.write(flowFile, out -> out.write(output.toString().getBytes()));
                flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), LivySessionService.APPLICATION_JSON);
                session.transfer(flowFile, REL_SUCCESS);
            } catch (JSONException je) {
                // The result doesn't contain the data, just send the output object as the flow file content to failure (after penalizing)
                log.error("Spark Session returned an error, sending the output JSON object as the flow file content to failure (after penalizing)");
                flowFile = session.write(flowFile, out -> out.write(result.toString().getBytes()));
                flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), LivySessionService.APPLICATION_JSON);
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
        }
    } catch (IOException ioe) {
        log.error("Failure processing flowfile {} due to {}, penalizing and routing to failure", new Object[] { flowFile, ioe.getMessage() }, ioe);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 75 with ComponentLog

use of org.apache.nifi.logging.ComponentLog in project nifi by apache.

the class TestPutSplunk method init.

@Before
public void init() {
    ComponentLog logger = Mockito.mock(ComponentLog.class);
    sender = new CapturingChannelSender("localhost", 12345, 0, logger);
    proc = new TestablePutSplunk(sender);
    runner = TestRunners.newTestRunner(proc);
    runner.setProperty(PutSplunk.PORT, "12345");
}