Examples with BulkableAction io.searchbox.action.BulkableAction used on opensource projects

Example 1 with BulkableAction

use of io.searchbox.action.BulkableAction in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method createIndexDocuments.

@Override
public void createIndexDocuments(String indexName, String documentType, Map<String, String> documentMap) {
    LOGGER.info("Creating Elasticsearch index documents, indexName={}, documentType={}", indexName, documentType);
    List<String> allIndices = getAliases(indexName);
    allIndices.forEach((index) -> {
        // Prepare a bulk request builder
        // final BulkRequestBuilder bulkRequestBuilder = new BulkRequestBuilder(new ElasticsearchClientImpl(), BulkAction.INSTANCE);
        Bulk.Builder bulkBuilder = new Bulk.Builder();
        // For each document prepare an insert request and add it to the bulk request builder
        documentMap.forEach((id, jsonString) -> {
            BulkableAction createIndex = new Index.Builder(jsonString).index(index).type(documentType).id(id).build();
            bulkBuilder.addAction(createIndex);
        });
        JestResult jestResult = jestClientHelper.executeAction(bulkBuilder.build());
        // If there are failures log them
        if (!jestResult.isSucceeded()) {
            LOGGER.error("Bulk response error = {}", jestResult.getErrorMessage());
        }
    });
}

Example 2 with BulkableAction

use of io.searchbox.action.BulkableAction in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method deleteIndexDocuments.

/**
 * The delete index documents function will delete a list of document in the index by a list of document ids.
 */
@Override
public final void deleteIndexDocuments(String indexName, String documentType, List<Integer> ids) {
    LOGGER.info("Deleting Elasticsearch documents from index, indexName={}, documentType={}, ids={}.", indexName, documentType, ids.stream().map(Object::toString).collect(Collectors.joining(",")));
    List<String> allIndices = getAliases(indexName);
    allIndices.forEach((index) -> {
        // Prepare a bulk request builder
        Bulk.Builder bulkBuilder = new Bulk.Builder();
        // For each document prepare a delete request and add it to the bulk request builder
        ids.forEach(id -> {
            BulkableAction action = new Delete.Builder(id.toString()).index(index).type(documentType).build();
            bulkBuilder.addAction(action);
        });
        JestResult jestResult = jestClientHelper.executeAction(bulkBuilder.build());
        // If there are failures log them
        if (!jestResult.isSucceeded()) {
            LOGGER.error("Bulk response error = {}", jestResult.getErrorMessage());
        }
    });
}

Example 3 with BulkableAction

use of io.searchbox.action.BulkableAction in project opennms by OpenNMS.

the class EventToIndex method sendEvents.

private void sendEvents(final List<BulkableAction<DocumentResult>> actions) {
    if (actions != null && actions.size() > 0) {
        // Split the actions up by index
        for (Map.Entry<String, List<BulkableAction<DocumentResult>>> entry : actions.stream().collect(Collectors.groupingBy(BulkableAction::getIndex)).entrySet()) {
            try {
                final List<BulkableAction<DocumentResult>> actionList = entry.getValue();
                if (entry.getValue().size() == 1) {
                    // Not bulk
                    final BulkableAction<DocumentResult> action = actionList.get(0);
                    executeSingleAction(getJestClient(), action);
                } else {
                    // The type will be identical for all events in the same index
                    final String type = actionList.get(0).getType();
                    final Bulk.Builder builder = new Bulk.Builder().defaultIndex(entry.getKey()).defaultType(type);
                    // Add all actions to the bulk operation
                    builder.addAction(actionList);
                    final Bulk bulk = builder.build();
                    BulkResult result = getJestClient().execute(bulk);
                    // If the bulk command fails completely...
                    if (result == null || !result.isSucceeded()) {
                        if (result == null) {
                            logEsError("Bulk API action", entry.getKey(), type, null, -1, null);
                        } else {
                            logEsError("Bulk API action", entry.getKey(), type, result.getJsonString(), result.getResponseCode(), result.getErrorMessage());
                        }
                        // Try and issue the bulk actions individually as a fallback
                        for (BulkableAction<DocumentResult> action : entry.getValue()) {
                            executeSingleAction(getJestClient(), action);
                        }
                        continue;
                    }
                    // Check the return codes of the results
                    List<BulkResultItem> items = result.getItems();
                    boolean all404s = true;
                    for (BulkResultItem item : items) {
                        if (item.status != 404) {
                            all404s = false;
                            break;
                        }
                    }
                    if (all404s) {
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("index name " + entry.getKey() + " doesn't exist, creating new index");
                        }
                        createIndex(getJestClient(), entry.getKey(), type);
                        result = getJestClient().execute(bulk);
                    }
                    // If the bulk command fails completely...
                    if (result == null || !result.isSucceeded()) {
                        if (result == null) {
                            logEsError("Bulk API action", entry.getKey(), type, null, -1, null);
                        } else {
                            logEsError("Bulk API action", entry.getKey(), type, result.getJsonString(), result.getResponseCode(), result.getErrorMessage());
                        }
                        // Try and issue the bulk actions individually as a fallback
                        for (BulkableAction<DocumentResult> action : entry.getValue()) {
                            executeSingleAction(getJestClient(), action);
                        }
                        continue;
                    }
                    // Log any unsuccessful completions as errors
                    for (BulkResultItem item : result.getItems()) {
                        if (item.status >= 200 && item.status < 300) {
                            if (LOG.isDebugEnabled()) {
                                // If debug is enabled, log all completions
                                logEsDebug(item.operation, entry.getKey(), item.type, "none", item.status, item.error);
                            }
                        } else {
                            logEsError(item.operation, entry.getKey(), item.type, "none", item.status, item.error);
                        }
                    }
                }
            } catch (Throwable ex) {
                LOG.error("Unexpected problem sending event to Elasticsearch", ex);
                // Shutdown the ES client, it will be recreated as needed
                closeJestClient();
            }
        }
    }
}

Example 4 with BulkableAction

use of io.searchbox.action.BulkableAction in project opennms by OpenNMS.

the class EventToIndex method convertEventsToEsActions.

/**
 * <p>This method converts events into a sequence of Elasticsearch index/update commands.
 * Three types of actions are possible:</p>
 * <ul>
 * <li>Updating an alarm document based on an {@link #ALARM_NOTIFICATION_UEI_STEM} event</li>
 * <li>Indexing the {@link #ALARM_NOTIFICATION_UEI_STEM} events</li>
 * <li>Indexing all other events</li>
 * </ul>
 *
 * @param event
 */
private List<BulkableAction<DocumentResult>> convertEventsToEsActions(List<Event> events) {
    final List<BulkableAction<DocumentResult>> retval = new ArrayList<>();
    for (Event event : events) {
        maybeRefreshCache(event);
        final String uei = event.getUei();
        // change alarm index and add event to alarm change event index
        if (uei.startsWith(ALARM_NOTIFICATION_UEI_STEM)) {
            if (STICKY_MEMO_EVENT.equals(uei) || JOURNAL_MEMO_EVENT.equals(uei)) {
                // currently we just save the event as an event to ES with no other processing
                if (LOG.isDebugEnabled())
                    LOG.debug("Sending Alarm MEMO Event to ES:" + event.toString());
            } else {
                if (LOG.isDebugEnabled()) {
                    if (ALARM_CREATED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Created Event to ES:" + event.toString());
                    } else if (ALARM_DELETED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Deleted Event to ES:" + event.toString());
                    } else if (ALARM_SEVERITY_CHANGED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Changed Severity Event to ES:" + event.toString());
                    } else if (ALARM_CLEARED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Cleared Event to ES:" + event.toString());
                    } else if (ALARM_ACKNOWLEDGED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Acknowledged Event to ES:" + event.toString());
                    } else if (ALARM_UNACKNOWLEDGED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Unacknowledged Event to ES:" + event.toString());
                    } else if (ALARM_SUPPRESSED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Suppressed Event to ES:" + event.toString());
                    } else if (ALARM_UNSUPPRESSED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Unsuppressed Event to ES:" + event.toString());
                    } else if (ALARM_TROUBLETICKET_STATE_CHANGE_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm TroubleTicked state changed Event to ES:" + event.toString());
                    } else if (ALARM_CHANGED_EVENT.equals(uei)) {
                        LOG.debug("Sending Alarm Changed Event to ES:" + event.toString());
                    }
                }
                if (archiveAlarms) {
                    // if an alarm change event, use the alarm change fields to update the alarm index
                    Update alarmUpdate = populateAlarmIndexBodyFromAlarmChangeEvent(event, INDEX_NAMES.get(Indices.ALARMS), INDEX_TYPES.get(Indices.ALARMS));
                    retval.add(alarmUpdate);
                }
            }
            // save all Alarm Change Events including memo change events
            if (archiveAlarmChangeEvents) {
                Index eventIndex = populateEventIndexBodyFromEvent(event, INDEX_NAMES.get(Indices.ALARM_EVENTS), INDEX_TYPES.get(Indices.ALARM_EVENTS));
                retval.add(eventIndex);
            }
        } else {
            // Handle all other event types
            if (archiveRawEvents) {
                // only send events to ES if they are persisted to database or logAllEvents is set to true
                if (logAllEvents || (event.getDbid() != null && event.getDbid() != 0)) {
                    Index eventIndex = populateEventIndexBodyFromEvent(event, INDEX_NAMES.get(Indices.EVENTS), INDEX_TYPES.get(Indices.EVENTS));
                    retval.add(eventIndex);
                } else {
                    if (LOG.isDebugEnabled())
                        LOG.debug("Not Sending Event to ES: null event.getDbid()=" + event.getDbid() + " Event=" + event.toString());
                }
            }
        }
    }
    return retval;
}

Example 5 with BulkableAction

use of io.searchbox.action.BulkableAction in project herd by FINRAOS.

the class IndexFunctionsDaoImpl method updateIndexDocuments.

/**
 * The update index documents function will take as arguments the index name, document type, and a map of documents to update. The document map key is the
 * document id, and the value is the document as a JSON string.
 */
@Override
public final void updateIndexDocuments(String indexName, String documentType, Map<String, String> documentMap) {
    LOGGER.info("Updating Elasticsearch index documents, indexName={}, documentType={}.", indexName, documentType);
    List<String> allIndices = getAliases(indexName);
    allIndices.forEach((index) -> {
        // Prepare a bulk request builder
        Bulk.Builder bulkBuilder = new Bulk.Builder();
        // For each document prepare an update request and add it to the bulk request builder
        documentMap.forEach((id, jsonString) -> {
            BulkableAction updateIndex = new Index.Builder(jsonString).index(index).type(documentType).id(id).build();
            bulkBuilder.addAction(updateIndex);
        });
        // Execute the bulk update request
        JestResult jestResult = jestClientHelper.executeAction(bulkBuilder.build());
        // If there are failures log them
        if (!jestResult.isSucceeded()) {
            LOGGER.error("Bulk response error = {}", jestResult.getErrorMessage());
        }
    });
}