Examples with RestHighLevelClient org.elasticsearch.client.RestHighLevelClient used on opensource projects

Example 36 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project ranger by apache.

the class ElasticSearchAccessAuditsService method searchXAccessAudits.

public VXAccessAuditList searchXAccessAudits(SearchCriteria searchCriteria) {
    RestHighLevelClient client = elasticSearchMgr.getClient();
    final boolean hiveQueryVisibility = PropertiesUtil.getBooleanProperty("ranger.audit.hive.query.visibility", true);
    if (client == null) {
        LOGGER.warn("ElasticSearch client is null, so not running the query.");
        throw restErrorUtil.createRESTException("Error connecting to search engine", MessageEnums.ERROR_SYSTEM);
    }
    List<VXAccessAudit> xAccessAuditList = new ArrayList<VXAccessAudit>();
    Map<String, Object> paramList = searchCriteria.getParamList();
    updateUserExclusion(paramList);
    SearchResponse response;
    try {
        response = elasticSearchUtil.searchResources(searchCriteria, searchFields, sortFields, client, elasticSearchMgr.index);
    } catch (IOException e) {
        LOGGER.warn(String.format("ElasticSearch query failed: %s", e.getMessage()));
        throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM);
    }
    MultiGetItemResponse[] docs;
    try {
        docs = elasticSearchUtil.fetch(client, elasticSearchMgr.index, response.getHits().getHits());
    } catch (IOException e) {
        LOGGER.warn(String.format("ElasticSearch fetch failed: %s", e.getMessage()));
        throw restErrorUtil.createRESTException("Error querying search engine", MessageEnums.ERROR_SYSTEM);
    }
    for (int i = 0; i < docs.length; i++) {
        // NOPMD - This for loop can be replaced by a foreach loop
        MultiGetItemResponse doc = docs[i];
        VXAccessAudit vXAccessAudit = populateViewBean(doc.getResponse());
        if (vXAccessAudit != null) {
            String serviceType = vXAccessAudit.getServiceType();
            boolean isHive = "hive".equalsIgnoreCase(serviceType);
            if (!hiveQueryVisibility && isHive) {
                vXAccessAudit.setRequestData(null);
            } else if (isHive) {
                String accessType = vXAccessAudit.getAccessType();
                if ("grant".equalsIgnoreCase(accessType) || "revoke".equalsIgnoreCase(accessType)) {
                    String requestData = vXAccessAudit.getRequestData();
                    if (requestData != null) {
                        try {
                            vXAccessAudit.setRequestData(java.net.URLDecoder.decode(requestData, "UTF-8"));
                        } catch (UnsupportedEncodingException e) {
                            LOGGER.warn("Error while encoding request data: " + requestData, e);
                        }
                    } else {
                        LOGGER.warn("Error in request data of audit from elasticSearch. AuditData: " + vXAccessAudit.toString());
                    }
                }
            }
        }
        xAccessAuditList.add(vXAccessAudit);
    }
    VXAccessAuditList returnList = new VXAccessAuditList();
    returnList.setPageSize(searchCriteria.getMaxRows());
    returnList.setResultSize(response.getHits().getHits().length);
    returnList.setTotalCount(response.getHits().getTotalHits().value);
    returnList.setStartIndex(searchCriteria.getStartIndex());
    returnList.setVXAccessAudits(xAccessAuditList);
    return returnList;
}

Example 37 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project ranger by apache.

the class ElasticSearchUtil method searchResources.

public SearchResponse searchResources(SearchCriteria searchCriteria, List<SearchField> searchFields, List<SortField> sortFields, RestHighLevelClient client, String index) throws IOException {
    // See Also: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-query-builders.html
    QueryAccumulator queryAccumulator = new QueryAccumulator(searchCriteria);
    if (searchCriteria.getParamList() != null) {
        searchFields.stream().forEach(queryAccumulator::addQuery);
        // hashmap for each field name
        if (queryAccumulator.fromDate != null || queryAccumulator.toDate != null) {
            queryAccumulator.queries.add(setDateRange(queryAccumulator.dateFieldName, queryAccumulator.fromDate, queryAccumulator.toDate));
        }
    }
    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
    queryAccumulator.queries.stream().filter(x -> x != null).forEach(boolQueryBuilder::must);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    setSortClause(searchCriteria, sortFields, searchSourceBuilder);
    searchSourceBuilder.from(searchCriteria.getStartIndex());
    searchSourceBuilder.size(searchCriteria.getMaxRows());
    searchSourceBuilder.fetchSource(true);
    SearchRequest query = new SearchRequest();
    query.indices(index);
    query.source(searchSourceBuilder.query(boolQueryBuilder));
    return client.search(query, RequestOptions.DEFAULT);
}

Example 38 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project ranger by apache.

the class ElasticSearchMgr method connect.

synchronized void connect() {
    if (client == null) {
        synchronized (ElasticSearchAuditDestination.class) {
            if (client == null) {
                String urls = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_URLS);
                String protocol = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PROTOCOL, "http");
                user = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_USER, "");
                password = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PWRD, "");
                int port = Integer.parseInt(PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_PORT));
                this.index = PropertiesUtil.getProperty(CONFIG_PREFIX + "." + CONFIG_INDEX, "ranger_audits");
                String parameterString = String.format(Locale.ROOT, "User:%s, %s://%s:%s/%s", user, protocol, urls, port, index);
                logger.info("Initializing ElasticSearch " + parameterString);
                if (urls != null) {
                    urls = urls.trim();
                }
                if (StringUtils.isBlank(urls) || "NONE".equalsIgnoreCase(urls.trim())) {
                    logger.info(String.format("Clearing URI config value: %s", urls));
                    urls = null;
                }
                try {
                    if (StringUtils.isNotBlank(user) && StringUtils.isNotBlank(password) && password.contains("keytab") && new File(password).exists()) {
                        subject = CredentialsProviderUtil.login(user, password);
                    }
                    RestClientBuilder restClientBuilder = getRestClientBuilder(urls, protocol, user, password, port);
                    client = new RestHighLevelClient(restClientBuilder);
                } catch (Throwable t) {
                    logger.error("Can't connect to ElasticSearch: " + parameterString, t);
                }
            }
        }
    }
}

Example 39 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project ranger by apache.

the class ElasticSearchAuditDestination method log.

@Override
public boolean log(Collection<AuditEventBase> events) {
    boolean ret = false;
    try {
        logStatusIfRequired();
        addTotalCount(events.size());
        RestHighLevelClient client = getClient();
        if (null == client) {
            // ElasticSearch is still not initialized. So need return error
            addDeferredCount(events.size());
            return ret;
        }
        ArrayList<AuditEventBase> eventList = new ArrayList<>(events);
        BulkRequest bulkRequest = new BulkRequest();
        try {
            for (AuditEventBase event : eventList) {
                AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
                String id = authzEvent.getEventId();
                Map<String, Object> doc = toDoc(authzEvent);
                bulkRequest.add(new IndexRequest(index).id(id).source(doc));
            }
        } catch (Exception ex) {
            addFailedCount(eventList.size());
            logFailedEvent(eventList, ex);
        }
        BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT);
        if (response.status().getStatus() >= 400) {
            addFailedCount(eventList.size());
            logFailedEvent(eventList, "HTTP " + response.status().getStatus());
        } else {
            BulkItemResponse[] items = response.getItems();
            for (int i = 0; i < items.length; i++) {
                AuditEventBase itemRequest = eventList.get(i);
                BulkItemResponse itemResponse = items[i];
                if (itemResponse.isFailed()) {
                    addFailedCount(1);
                    logFailedEvent(Arrays.asList(itemRequest), itemResponse.getFailureMessage());
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug(String.format("Indexed %s", itemRequest.getEventKey()));
                    }
                    addSuccessCount(1);
                    ret = true;
                }
            }
        }
    } catch (Throwable t) {
        addDeferredCount(events.size());
        logError("Error sending message to ElasticSearch", t);
    }
    return ret;
}

Example 40 with RestHighLevelClient

use of org.elasticsearch.client.RestHighLevelClient in project ranger by apache.

the class ElasticSearchIndexBootStrapper method createClient.

private void createClient() {
    try {
        RestClientBuilder restClientBuilder = getRestClientBuilder(hosts, protocol, user, password, port);
        client = new RestHighLevelClient(restClientBuilder);
    } catch (Throwable t) {
        lastLoggedAt.updateAndGet(lastLoggedAt -> {
            long now = System.currentTimeMillis();
            long elapsed = now - lastLoggedAt;
            if (elapsed > TimeUnit.MINUTES.toMillis(1)) {
                LOG.severe("Can't connect to ElasticSearch server: " + connectionString() + t);
                return now;
            } else {
                return lastLoggedAt;
            }
        });
    }
}