Examples with ElasticException io.hops.hopsworks.exceptions.ElasticException used on opensource projects

Example 16 with ElasticException

use of io.hops.hopsworks.exceptions.ElasticException in project hopsworks by logicalclocks.

the class PyPiLibraryElasticIndexer method execute.

@Timeout
@TransactionAttribute(TransactionAttributeType.NOT_SUPPORTED)
public void execute(Timer timer) throws ElasticException {
    long errorRescheduleTimeout = 600000;
    LOGGER.log(Level.INFO, "Running PyPi Indexer");
    try {
        ClusterHealthResponse clusterHealthResponse = elasticClientCtrl.clusterHealthGet();
        GetIndexTemplatesResponse templateResponse = elasticClientCtrl.templateGet(Settings.ELASTIC_PYPI_LIBRARIES_ALIAS);
        // If elasticsearch is down or template not in elastic reschedule the timer
        if (clusterHealthResponse.getStatus().equals(ClusterHealthStatus.RED)) {
            scheduleTimer(errorRescheduleTimeout);
            LOGGER.log(Level.INFO, "Elastic currently down, rescheduling indexing for pypi libraries");
            return;
        } else if (templateResponse.getIndexTemplates().isEmpty()) {
            scheduleTimer(errorRescheduleTimeout);
            LOGGER.log(Level.INFO, "Elastic template " + Settings.ELASTIC_PYPI_LIBRARIES_ALIAS + " currently missing, " + "rescheduling indexing for pypi libraries");
            return;
        }
    } catch (Exception e) {
        scheduleTimer(errorRescheduleTimeout);
        LOGGER.log(Level.SEVERE, "Exception occurred trying to index pypi libraries, rescheduling timer", e);
        return;
    }
    String newIndex = Settings.ELASTIC_PYPI_LIBRARIES_INDEX_PATTERN_PREFIX + System.currentTimeMillis();
    try {
        GetAliasesResponse pypiAlias = elasticClientCtrl.getAliases(Settings.ELASTIC_PYPI_LIBRARIES_ALIAS);
        if (!pypiAlias.getAliases().isEmpty()) {
            this.isIndexed = true;
        }
        String[] indicesToDelete = elasticClientCtrl.mngIndicesGetBySimplifiedRegex(Settings.ELASTIC_PYPI_LIBRARIES_INDEX_REGEX);
        Element body = Jsoup.connect(settings.getPyPiSimpleEndpoint()).maxBodySize(0).get().body();
        Elements elements = body.getElementsByTag("a");
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(newIndex);
        elasticClientCtrl.mngIndexCreate(createIndexRequest);
        final int bulkSize = 100;
        int currentBulkSize = 0;
        int currentId = 0;
        BulkRequest bulkRequest = new BulkRequest();
        LOGGER.log(Level.INFO, "Starting to index libraries from pypi simple index");
        for (Element library : elements) {
            IndexRequest indexRequest = new IndexRequest().index(newIndex).id(String.valueOf(currentId)).source(jsonBuilder().startObject().field("library", library.text()).endObject());
            bulkRequest.add(indexRequest);
            currentBulkSize += 1;
            currentId += 1;
            if (currentBulkSize == bulkSize) {
                elasticClientCtrl.bulkUpdateDoc(bulkRequest);
                bulkRequest = new BulkRequest();
                currentBulkSize = 0;
            }
        }
        // Also send last batch
        if (bulkRequest.numberOfActions() > 0) {
            elasticClientCtrl.bulkUpdateDoc(bulkRequest);
        }
        if (pypiAlias.getAliases().isEmpty()) {
            elasticClientCtrl.createAlias(Settings.ELASTIC_PYPI_LIBRARIES_ALIAS, newIndex);
        } else {
            String currentSearchIndex = pypiAlias.getAliases().keySet().iterator().next();
            elasticClientCtrl.aliasSwitchIndex(Settings.ELASTIC_PYPI_LIBRARIES_ALIAS, currentSearchIndex, newIndex);
        }
        this.isIndexed = true;
        LOGGER.log(Level.INFO, "Finished indexing");
        for (String index : indicesToDelete) {
            DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(index);
            elasticClientCtrl.mngIndexDelete(deleteIndexRequest);
        }
    } catch (Exception ex) {
        LOGGER.log(Level.SEVERE, "Indexing pypi libraries failed", ex);
        scheduleTimer(errorRescheduleTimeout);
        if (elasticClientCtrl.mngIndexExists(newIndex)) {
            DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest().indices(newIndex);
            elasticClientCtrl.mngIndexDelete(deleteIndexRequest);
        }
        return;
    }
    String rawInterval = settings.getPyPiIndexerTimerInterval();
    Long intervalValue = settings.getConfTimeValue(rawInterval);
    TimeUnit intervalTimeunit = settings.getConfTimeTimeUnit(rawInterval);
    scheduleTimer(intervalTimeunit.toMillis(intervalValue));
}

Example 17 with ElasticException

use of io.hops.hopsworks.exceptions.ElasticException in project hopsworks by logicalclocks.

the class LibraryController method pipList.

/**
 * @param <R> parsed elastic item
 * @param <S1> intermediate result wrapped in Try
 * @param <S2> final result
 * @return
 * @throws ProvenanceException
 */
private <R, S1, S2> S2 pipList(String query, Integer offset, Integer limit, LibraryController.HandlerFactory<R, S1, S2> handlerFactory, String index) throws ServiceException {
    Pair<Long, Try<S1>> searchResult;
    try {
        CheckedSupplier<SearchRequest, ProvenanceException> srF = ElasticHelper.baseSearchRequest(index, settings.getElasticDefaultScrollPageSize()).andThen(ElasticHelper.withPagination(offset, limit, settings.getElasticDefaultScrollPageSize()));
        SearchRequest request = srF.get();
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        sourceBuilder.query(ElasticHelper.fullTextSearch("library", query));
        request.source(sourceBuilder);
        searchResult = provElasticController.search(request, handlerFactory.getHandler());
    } catch (ElasticException | ProvenanceException pe) {
        throw new ServiceException(RESTCodes.ServiceErrorCode.ANACONDA_LIST_LIB_ERROR, Level.FINE, "Unable to list python libraries", pe.getMessage(), pe);
    }
    return handlerFactory.checkedResult(searchResult);
}

Example 18 with ElasticException

use of io.hops.hopsworks.exceptions.ElasticException in project hopsworks by logicalclocks.

the class ProvAppController method provAppState.

public Map<String, Map<Provenance.AppState, ProvAppStateElastic>> provAppState(Map<ProvParser.Field, ProvParser.FilterVal> filterBy, List<Pair<ProvParser.Field, SortOrder>> sortBy, Integer offset, Integer limit) throws ProvenanceException {
    CheckedSupplier<SearchRequest, ProvenanceException> srF = ElasticHelper.scrollingSearchRequest(Settings.ELASTIC_INDEX_APP_PROVENANCE, settings.getElasticDefaultScrollPageSize()).andThen(provAppStateQB(filterBy)).andThen(ElasticHelper.sortBy(sortBy)).andThen(ElasticHelper.withPagination(offset, limit, settings.getElasticMaxScrollPageSize()));
    SearchRequest request = srF.get();
    Pair<Long, Try<ElasticAppStatesObj>> searchResult;
    try {
        searchResult = client.searchScrolling(request, ElasticAppStatesObj.getHandler());
    } catch (ElasticException e) {
        String msg = "provenance - elastic query problem";
        throw ProvHelper.fromElastic(e, msg, msg + " - app state");
    }
    return ElasticAppStatesObj.checkedResult(searchResult);
}

Example 19 with ElasticException

use of io.hops.hopsworks.exceptions.ElasticException in project hopsworks by logicalclocks.

the class ProvStateController method provFileState.

/**
 * @param <R> parsed elastic item
 * @param <S1> intermediate result wrapped in Try
 * @param <S2> final result
 * @return
 * @throws ProvenanceException
 */
private <R, S1, S2> S2 provFileState(Long projectIId, Map<ProvParser.Field, ProvParser.FilterVal> fileStateFilters, List<Pair<ProvParser.Field, SortOrder>> fileStateSortBy, Map<String, String> xAttrsFilters, Map<String, String> likeXAttrsFilters, Set<String> hasXAttrsFilters, List<ProvStateParamBuilder.SortE> xattrSortBy, Integer offset, Integer limit, HandlerFactory<R, S1, S2> handlerFactory) throws ProvenanceException {
    CheckedSupplier<SearchRequest, ProvenanceException> srF = ElasticHelper.baseSearchRequest(settings.getProvFileIndex(projectIId), settings.getElasticDefaultScrollPageSize()).andThen(filterByStateParams(fileStateFilters, xAttrsFilters, likeXAttrsFilters, hasXAttrsFilters)).andThen(ElasticHelper.withFileStateOrder(fileStateSortBy, xattrSortBy)).andThen(ElasticHelper.withPagination(offset, limit, settings.getElasticMaxScrollPageSize()));
    SearchRequest request = srF.get();
    Pair<Long, Try<S1>> searchResult;
    try {
        searchResult = client.search(request, handlerFactory.getHandler());
    } catch (ElasticException e) {
        String msg = "provenance - elastic query problem";
        throw ProvHelper.fromElastic(e, msg, msg + " - file state");
    }
    return handlerFactory.checkedResult(searchResult);
}

Example 20 with ElasticException

use of io.hops.hopsworks.exceptions.ElasticException in project hopsworks by logicalclocks.

the class ProvStateController method provFileStateCount.

private ProvStateDTO provFileStateCount(Long projectIId, Map<ProvParser.Field, ProvParser.FilterVal> fileStateFilters, Map<String, String> xAttrsFilters, Map<String, String> likeXAttrsFilters, Set<String> hasXAttrsFilters) throws ProvenanceException {
    CheckedSupplier<SearchRequest, ProvenanceException> srF = ElasticHelper.countSearchRequest(settings.getProvFileIndex(projectIId)).andThen(filterByStateParams(fileStateFilters, xAttrsFilters, likeXAttrsFilters, hasXAttrsFilters));
    SearchRequest request = srF.get();
    Long searchResult;
    try {
        searchResult = client.searchCount(request);
    } catch (ElasticException e) {
        String msg = "provenance - elastic query problem";
        throw ProvHelper.fromElastic(e, msg, msg + " - file state count");
    }
    ProvStateDTO container = new ProvStateDTO();
    container.setCount(searchResult);
    return container;
}