Examples with CollectionReference io.arlas.server.core.model.CollectionReference used on opensource projects

Example 21 with CollectionReference

use of io.arlas.server.core.model.CollectionReference in project ARLAS-server by gisaia.

the class HazelcastCacheManager method getCollectionReference.

@Override
public CollectionReference getCollectionReference(String ref) {
    CollectionReference c;
    try {
        c = (CollectionReference) this.instance.getReplicatedMap("collections").get(ref);
    } catch (HazelcastInstanceNotActiveException e) {
        // recover from unexpected shutdown
        init();
        c = (CollectionReference) this.instance.getReplicatedMap("collections").get(ref);
    }
    LOGGER.debug("Returning collection reference '" + ref + "' from cache with value " + (c == null ? "null" : c.collectionName));
    return c;
}

Example 22 with CollectionReference

use of io.arlas.server.core.model.CollectionReference in project ARLAS-server by gisaia.

the class ElasticCollectionReferenceService method getAllCollectionReferences.

@Override
public List<CollectionReference> getAllCollectionReferences(Optional<String> columnFilter) throws ArlasException {
    List<CollectionReference> collections = new ArrayList<>();
    try {
        QueryBuilder qb = QueryBuilders.matchAllQuery();
        SearchRequest request = new SearchRequest(arlasIndex);
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        // Exclude old include_fields for support old collection
        searchSourceBuilder.fetchSource(null, "include_fields").sort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC).query(qb).size(// max of 100 hits will be returned for each scroll
        100);
        request.source(searchSourceBuilder);
        request.scroll(new TimeValue(60000));
        SearchResponse scrollResp = client.search(request);
        Set<String> allowedCollections = ColumnFilterUtil.getAllowedCollections(columnFilter);
        do {
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                String source = hit.getSourceAsString();
                try {
                    for (String c : allowedCollections) {
                        if ((c.endsWith("*") && hit.getId().startsWith(c.substring(0, c.indexOf("*")))) || hit.getId().equals(c)) {
                            collections.add(new CollectionReference(hit.getId(), reader.readValue(source)));
                            break;
                        }
                    }
                } catch (IOException e) {
                    throw new InternalServerErrorException("Can not fetch collection", e);
                }
            }
            SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollResp.getScrollId());
            scrollRequest.scroll(new TimeValue(60000));
            scrollResp = client.searchScroll(scrollRequest);
        } while (// Zero hits mark the end of the scroll and the while loop.
        scrollResp.getHits().getHits().length != 0);
        ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
        clearScrollRequest.addScrollId(scrollResp.getScrollId());
        client.clearScroll(clearScrollRequest);
    } catch (IndexNotFoundException e) {
        throw new InternalServerErrorException("Unreachable collections", e);
    }
    return collections.stream().filter(c -> {
        try {
            return !getMapping(c.params.indexName).isEmpty();
        } catch (ArlasException e) {
            return false;
        }
    }).collect(Collectors.toList());
}

Example 23 with CollectionReference

use of io.arlas.server.core.model.CollectionReference in project ARLAS-server by gisaia.

the class ElasticExploreService method formatAggregationResult.

@SuppressWarnings({ "rawtypes", "unchecked" })
private AggregationResponse formatAggregationResult(MultiBucketsAggregation aggregation, AggregationResponse aggregationResponse, CollectionReference collection, List<Aggregation> aggregationsRequest, int aggTreeDepth) {
    aggregationResponse.name = aggregation.getName();
    if (aggregationResponse.name.equals(TERM_AGG)) {
        aggregationResponse.sumotherdoccounts = ((Terms) aggregation).getSumOfOtherDocCounts();
    }
    List<RawGeometry> rawGeometries = aggregationsRequest.size() > aggTreeDepth ? aggregationsRequest.get(aggTreeDepth).rawGeometries : null;
    List<AggregatedGeometryEnum> aggregatedGeometries = aggregationsRequest.size() > aggTreeDepth ? aggregationsRequest.get(aggTreeDepth).aggregatedGeometries : null;
    aggregationResponse.elements = new ArrayList<>();
    List<MultiBucketsAggregation.Bucket> buckets = (List<MultiBucketsAggregation.Bucket>) aggregation.getBuckets();
    buckets.forEach(bucket -> {
        AggregationResponse element = new AggregationResponse();
        element.keyAsString = bucket.getKeyAsString();
        // if it is a `geohash` aggregation type, we set the GEOHASHCENTER and GEOHASH aggregated geometries if they're requested
        if (aggregationResponse.name.equals(GEOHASH_AGG)) {
            GeoPoint geoPoint = getGeohashCentre(element.keyAsString.toString());
            element.key = geoPoint;
            if (!CollectionUtils.isEmpty(aggregatedGeometries)) {
                aggregatedGeometries.stream().filter(g -> g.isCellOrCellCenterAgg()).forEach(g -> {
                    ReturnedGeometry returnedGeometry = new ReturnedGeometry();
                    returnedGeometry.reference = g.value();
                    returnedGeometry.isRaw = false;
                    if (g.isCellAgg()) {
                        returnedGeometry.geometry = createPolygonFromGeohash(element.keyAsString.toString());
                    } else {
                        returnedGeometry.geometry = new Point(geoPoint.getLon(), geoPoint.getLat());
                    }
                    if (element.geometries == null) {
                        element.geometries = new ArrayList<>();
                    }
                    element.geometries.add(returnedGeometry);
                });
            }
        } else if (aggregationResponse.name.equals(GEOTILE_AGG)) {
            List<Integer> zxy = Stream.of(element.keyAsString.toString().split("/")).map(elem -> Integer.valueOf(elem)).collect(Collectors.toList());
            BoundingBox tile = GeoTileUtil.getBoundingBox(zxy.get(1), zxy.get(2), zxy.get(0));
            GeoPoint geoPoint = getTileCentre(tile);
            element.key = geoPoint;
            if (!CollectionUtils.isEmpty(aggregatedGeometries)) {
                aggregatedGeometries.stream().filter(g -> g.isCellOrCellCenterAgg()).forEach(g -> {
                    ReturnedGeometry returnedGeometry = new ReturnedGeometry();
                    returnedGeometry.reference = g.value();
                    returnedGeometry.isRaw = false;
                    if (g.isCellAgg()) {
                        returnedGeometry.geometry = createBox(tile);
                    } else {
                        returnedGeometry.geometry = new Point(geoPoint.getLon(), geoPoint.getLat());
                    }
                    if (element.geometries == null) {
                        element.geometries = new ArrayList<>();
                    }
                    element.geometries.add(returnedGeometry);
                });
            }
        } else if (aggregationResponse.name.equals(H3_AGG)) {
            GeoPoint geoPoint = getH3Centre(element.keyAsString.toString());
            element.key = geoPoint;
            if (!CollectionUtils.isEmpty(aggregatedGeometries)) {
                aggregatedGeometries.stream().filter(g -> g.isCellOrCellCenterAgg()).forEach(g -> {
                    ReturnedGeometry returnedGeometry = new ReturnedGeometry();
                    returnedGeometry.reference = g.value();
                    returnedGeometry.isRaw = false;
                    if (g.isCellAgg()) {
                        returnedGeometry.geometry = createPolygonFromH3(element.keyAsString.toString());
                    } else {
                        returnedGeometry.geometry = new Point(geoPoint.getLon(), geoPoint.getLat());
                    }
                    if (element.geometries == null) {
                        element.geometries = new ArrayList<>();
                    }
                    element.geometries.add(returnedGeometry);
                });
            }
        } else if (aggregationResponse.name.startsWith(DATEHISTOGRAM_AGG)) {
            element.key = ((ZonedDateTime) bucket.getKey()).withZoneSameInstant(ZoneOffset.UTC).toInstant().toEpochMilli();
        } else {
            element.key = bucket.getKey();
        }
        element.count = bucket.getDocCount();
        element.elements = new ArrayList<>();
        if (bucket.getAggregations().asList().size() == 0) {
            element.elements = null;
        } else {
            bucket.getAggregations().forEach(subAggregation -> {
                AggregationResponse subAggregationResponse = new AggregationResponse();
                subAggregationResponse.name = subAggregation.getName();
                if (subAggregationResponse.name.equals(TERM_AGG)) {
                    subAggregationResponse.sumotherdoccounts = ((Terms) subAggregation).getSumOfOtherDocCounts();
                }
                if (subAggregation.getName().equals(FETCH_HITS_AGG)) {
                    subAggregationResponse = null;
                    element.hits = Optional.ofNullable(((TopHits) subAggregation).getHits().getHits()).map(Arrays::asList).map(hitsList -> hitsList.stream().map(SearchHit::getSourceAsMap).collect(Collectors.toList())).orElse(new ArrayList());
                } else if (Arrays.asList(DATEHISTOGRAM_AGG, HISTOGRAM_AGG, TERM_AGG, GEOHASH_AGG, GEOTILE_AGG, H3_AGG).contains(subAggregation.getName())) {
                    subAggregationResponse = formatAggregationResult(((MultiBucketsAggregation) subAggregation), subAggregationResponse, collection, aggregationsRequest, aggTreeDepth + 1);
                } else if (isAggregatedGeometry(subAggregation.getName(), aggregatedGeometries)) {
                    subAggregationResponse = null;
                    ReturnedGeometry returnedGeometry = new ReturnedGeometry();
                    returnedGeometry.isRaw = false;
                    if (subAggregation.getName().equals(AggregatedGeometryEnum.BBOX.value() + AGGREGATED_GEOMETRY_SUFFIX)) {
                        Polygon box = createBox((GeoBounds) subAggregation);
                        returnedGeometry.reference = AggregatedGeometryEnum.BBOX.value();
                        returnedGeometry.geometry = box;
                    } else if (subAggregation.getName().equals(AggregatedGeometryEnum.CENTROID.value() + AGGREGATED_GEOMETRY_SUFFIX)) {
                        GeoPoint centroid = ((GeoCentroid) subAggregation).centroid();
                        GeoJsonObject g = new Point(centroid.getLon(), centroid.getLat());
                        returnedGeometry.reference = AggregatedGeometryEnum.CENTROID.value();
                        returnedGeometry.geometry = g;
                    }
                    if (element.geometries == null) {
                        element.geometries = new ArrayList<>();
                    }
                    element.geometries.add(returnedGeometry);
                } else if (subAggregation.getName().startsWith(RAW_GEOMETRY_SUFFIX)) {
                    String sort = subAggregation.getName().substring(RAW_GEOMETRY_SUFFIX.length());
                    subAggregationResponse = null;
                    long nbHits = ((TopHits) subAggregation).getHits().getTotalHits().value;
                    if (nbHits > 0) {
                        SearchHit[] hits = ((TopHits) subAggregation).getHits().getHits();
                        for (SearchHit hit : hits) {
                            Map source = hit.getSourceAsMap();
                            if (rawGeometries != null) {
                                List<String> geometries = rawGeometries.stream().filter(rg -> rg.sort.equals(sort)).map(rg -> rg.geometry).collect(Collectors.toList());
                                geometries.forEach(g -> {
                                    GeoJsonObject geometryGeoJson;
                                    try {
                                        CollectionReferenceManager.setCollectionGeometriesType(source, collection, String.join(",", geometries));
                                        GeoTypeEnum geometryType;
                                        Object geometry = MapExplorer.getObjectFromPath(g, source);
                                        ReturnedGeometry returnedGeometry = null;
                                        if (geometry != null) {
                                            returnedGeometry = new ReturnedGeometry();
                                            geometryType = collection.params.getGeometryType(g);
                                            returnedGeometry.reference = g;
                                        } else {
                                            geometry = MapExplorer.getObjectFromPath(collection.params.centroidPath, source);
                                            geometryType = collection.params.getGeometryType(collection.params.centroidPath);
                                            if (geometry != null) {
                                                returnedGeometry = new ReturnedGeometry();
                                                returnedGeometry.reference = collection.params.centroidPath;
                                            }
                                        }
                                        geometryGeoJson = geometry != null ? GeoTypeMapper.getGeoJsonObject(geometry, geometryType) : null;
                                        if (geometryGeoJson != null) {
                                            // "old-style crs member is not recommended" so we remove it
                                            geometryGeoJson.setCrs(null);
                                            returnedGeometry.geometry = geometryGeoJson;
                                            returnedGeometry.isRaw = true;
                                            returnedGeometry.sort = sort;
                                            if (element.geometries == null) {
                                                element.geometries = new ArrayList<>();
                                            }
                                            element.geometries.add(returnedGeometry);
                                        }
                                    } catch (ArlasException e) {
                                        e.printStackTrace();
                                    }
                                });
                            }
                        }
                    }
                    if (bucket.getAggregations().asList().size() == 1) {
                        element.metrics = null;
                        element.elements = null;
                    }
                } else {
                    if (element.metrics == null) {
                        element.metrics = new ArrayList<>();
                    }
                    subAggregationResponse = null;
                    AggregationMetric aggregationMetric = new AggregationMetric();
                    aggregationMetric.type = subAggregation.getName().split(":")[0];
                    aggregationMetric.field = subAggregation.getName().split(":")[1];
                    if (!aggregationMetric.type.equals(CollectionFunction.GEOCENTROID.name().toLowerCase()) && !aggregationMetric.type.equals(CollectionFunction.GEOBBOX.name().toLowerCase())) {
                        aggregationMetric.value = (((NumericMetricsAggregation.SingleValue) subAggregation).value());
                    } else {
                        FeatureCollection fc = new FeatureCollection();
                        Feature feature = new Feature();
                        if (aggregationMetric.type.equals(CollectionFunction.GEOBBOX.name().toLowerCase())) {
                            GeoJsonObject g = createBox((GeoBounds) subAggregation);
                            feature.setGeometry(g);
                            fc.add(feature);
                        } else if (aggregationMetric.type.equals(CollectionFunction.GEOCENTROID.name().toLowerCase())) {
                            GeoPoint centroid = ((GeoCentroid) subAggregation).centroid();
                            GeoJsonObject g = new Point(centroid.getLon(), centroid.getLat());
                            feature.setGeometry(g);
                            fc.add(feature);
                        }
                        aggregationMetric.value = fc;
                    }
                    element.metrics.add(aggregationMetric);
                }
                if (subAggregationResponse != null) {
                    element.elements.add(subAggregationResponse);
                }
            });
        }
        aggregationResponse.elements.add(element);
    });
    return aggregationResponse;
}

Example 24 with CollectionReference

use of io.arlas.server.core.model.CollectionReference in project ARLAS-server by gisaia.

the class JdbiCollectionReferenceService method getAllCollectionReferences.

@Override
public List<CollectionReference> getAllCollectionReferences(Optional<String> columnFilter) {
    Set<String> allowedCollections = ColumnFilterUtil.getAllowedCollections(columnFilter);
    List<CollectionReference> all = dao.selectAll(this.arlasIndex);
    List<CollectionReference> result = new ArrayList<>();
    for (CollectionReference cr : all) {
        for (String c : allowedCollections) {
            if ((c.endsWith("*") && cr.collectionName.startsWith(c.substring(0, c.indexOf("*")))) || cr.collectionName.equals(c)) {
                result.add(cr);
                break;
            }
        }
    }
    return result;
}

Example 25 with CollectionReference

use of io.arlas.server.core.model.CollectionReference in project ARLAS-server by gisaia.

the class CollectionReferenceService method getCollectionReference.

// -------
public CollectionReference getCollectionReference(String ref) throws ArlasException {
    CollectionReference collectionReference = cacheManager.getCollectionReference(ref);
    if (collectionReference == null) {
        collectionReference = getCollectionReferenceFromDao(ref);
        cacheManager.putCollectionReference(ref, collectionReference);
    }
    if (!getMapping(collectionReference.params.indexName).isEmpty()) {
        return collectionReference;
    } else {
        throw new ArlasException("Collection " + ref + " exists but can not be described. Check if index or template ".concat(collectionReference.params.indexName).concat(" exists"));
    }
}