Examples with OutputStreamToByteConsumer de.ii.xtraplatform.streams.domain.OutputStreamToByteConsumer used on opensource projects

Example 1 with OutputStreamToByteConsumer

use of de.ii.xtraplatform.streams.domain.OutputStreamToByteConsumer in project ldproxy by interactive-instruments.

the class TilesQueriesHandlerImpl method getMultiLayerTileResponse.

private Response getMultiLayerTileResponse(QueryInputTileMultiLayer queryInput, ApiRequestContext requestContext) {
    OgcApi api = requestContext.getApi();
    OgcApiDataV2 apiData = api.getData();
    Tile multiLayerTile = queryInput.getTile();
    List<String> collectionIds = multiLayerTile.getCollectionIds();
    Map<String, FeatureQuery> queryMap = queryInput.getQueryMap();
    Map<String, Tile> singleLayerTileMap = queryInput.getSingleLayerTileMap();
    FeatureProvider2 featureProvider = multiLayerTile.getFeatureProvider().get();
    TileMatrixSet tileMatrixSet = multiLayerTile.getTileMatrixSet();
    int tileLevel = multiLayerTile.getTileLevel();
    int tileRow = multiLayerTile.getTileRow();
    int tileCol = multiLayerTile.getTileCol();
    if (!(multiLayerTile.getOutputFormat() instanceof TileFormatWithQuerySupportExtension))
        throw new RuntimeException(String.format("Unexpected tile format without query support. Found: %s", multiLayerTile.getOutputFormat().getClass().getSimpleName()));
    TileFormatWithQuerySupportExtension outputFormat = (TileFormatWithQuerySupportExtension) multiLayerTile.getOutputFormat();
    // process parameters and generate query
    Optional<CrsTransformer> crsTransformer = Optional.empty();
    EpsgCrs targetCrs = tileMatrixSet.getCrs();
    if (featureProvider.supportsCrs()) {
        EpsgCrs sourceCrs = featureProvider.crs().getNativeCrs();
        crsTransformer = crsTransformerFactory.getTransformer(sourceCrs, targetCrs);
    }
    List<Link> links = new DefaultLinksGenerator().generateLinks(requestContext.getUriCustomizer(), requestContext.getMediaType(), requestContext.getAlternateMediaTypes(), i18n, requestContext.getLanguage());
    Map<String, ByteArrayOutputStream> byteArrayMap = new HashMap<>();
    for (String collectionId : collectionIds) {
        // TODO limitation of the current model: all layers have to come from the same feature provider and use the same CRS
        Tile tile = singleLayerTileMap.get(collectionId);
        if (!multiLayerTile.getTemporary()) {
            // use cached tile
            try {
                Optional<InputStream> tileContent = tileCache.getTile(tile);
                if (tileContent.isPresent()) {
                    ByteArrayOutputStream buffer = new ByteArrayOutputStream();
                    ByteStreams.copy(tileContent.get(), buffer);
                    byteArrayMap.put(collectionId, buffer);
                    continue;
                }
            } catch (SQLException | IOException e) {
            // could not read the cache, generate the tile
            }
        }
        String featureTypeId = apiData.getCollections().get(collectionId).getExtension(FeaturesCoreConfiguration.class).map(cfg -> cfg.getFeatureType().orElse(collectionId)).orElse(collectionId);
        FeatureQuery query = queryMap.get(collectionId);
        ImmutableFeatureTransformationContextTiles transformationContext;
        try {
            transformationContext = new ImmutableFeatureTransformationContextTiles.Builder().api(api).apiData(apiData).featureSchema(featureProvider.getData().getTypes().get(featureTypeId)).tile(tile).tileCache(tileCache).collectionId(collectionId).ogcApiRequest(requestContext).crsTransformer(crsTransformer).codelists(entityRegistry.getEntitiesForType(Codelist.class).stream().collect(Collectors.toMap(PersistentEntity::getId, c -> c))).defaultCrs(queryInput.getDefaultCrs()).links(links).isFeatureCollection(true).fields(query.getFields()).limit(query.getLimit()).offset(0).i18n(i18n).outputStream(new OutputStreamToByteConsumer()).build();
        } catch (Exception e) {
            throw new RuntimeException("Error building the tile transformation context.", e);
        }
        Optional<FeatureTokenEncoder<?>> encoder = outputFormat.getFeatureEncoder(transformationContext);
        if (outputFormat.supportsFeatureQuery() && encoder.isPresent()) {
            FeatureStream featureStream = featureProvider.queries().getFeatureStream(query);
            ResultReduced<byte[]> result = generateTile(featureStream, encoder.get(), transformationContext, outputFormat);
            if (result.isSuccess()) {
                byte[] bytes = result.reduced();
                ByteArrayOutputStream buffer = new ByteArrayOutputStream(bytes.length);
                buffer.write(bytes, 0, bytes.length);
                byteArrayMap.put(collectionId, buffer);
            }
        } else {
            throw new NotAcceptableException(MessageFormat.format("The requested media type {0} cannot be generated, because it does not support streaming.", requestContext.getMediaType().type()));
        }
    }
    TileFormatWithQuerySupportExtension.MultiLayerTileContent result;
    try {
        result = outputFormat.combineSingleLayerTilesToMultiLayerTile(tileMatrixSet, singleLayerTileMap, byteArrayMap);
    } catch (IOException e) {
        throw new RuntimeException("Error accessing the tile cache.", e);
    }
    // try to write/update tile in cache, if all collections have been processed
    if (result.isComplete) {
        try {
            tileCache.storeTile(multiLayerTile, result.byteArray);
        } catch (Throwable e) {
            String msg = "Failure to write the multi-layer file of tile {}/{}/{}/{} in dataset '{}', format '{}' to the cache";
            LogContext.errorAsInfo(LOGGER, e, msg, tileMatrixSet.getId(), tileLevel, tileRow, tileCol, api.getId(), outputFormat.getExtension());
        }
    }
    Date lastModified = null;
    EntityTag etag = getEtag(result.byteArray);
    Response.ResponseBuilder response = evaluatePreconditions(requestContext, lastModified, etag);
    if (Objects.nonNull(response))
        return response.build();
    return prepareSuccessResponse(requestContext, queryInput.getIncludeLinkHeader() ? links : null, lastModified, etag, queryInput.getCacheControl().orElse(null), queryInput.getExpires().orElse(null), null, true, String.format("%s_%d_%d_%d.%s", tileMatrixSet.getId(), tileLevel, tileRow, tileCol, outputFormat.getMediaType().fileExtension())).entity(result.byteArray).build();
}

Example 2 with OutputStreamToByteConsumer

use of de.ii.xtraplatform.streams.domain.OutputStreamToByteConsumer in project ldproxy by interactive-instruments.

the class FeatureEncoderHtml method onEnd.

@Override
public void onEnd(ModifiableContext context) {
    // TODO: FeatureTokenEncoderBytes.getOutputStream
    OutputStreamWriter writer = new OutputStreamWriter(new OutputStreamToByteConsumer(this::push));
    try {
        ((MustacheRenderer) transformationContext.mustacheRenderer()).render(transformationContext.collectionView(), writer);
        writer.flush();
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
}

Example 3 with OutputStreamToByteConsumer

use of de.ii.xtraplatform.streams.domain.OutputStreamToByteConsumer in project ldproxy by interactive-instruments.

the class TilesQueriesHandlerImpl method getSingleLayerTileResponse.

private Response getSingleLayerTileResponse(QueryInputTileSingleLayer queryInput, ApiRequestContext requestContext) {
    OgcApi api = requestContext.getApi();
    OgcApiDataV2 apiData = api.getData();
    Tile tile = queryInput.getTile();
    String collectionId = tile.getCollectionId();
    FeatureProvider2 featureProvider = tile.getFeatureProvider().get();
    FeatureQuery query = queryInput.getQuery();
    if (!(tile.getOutputFormat() instanceof TileFormatWithQuerySupportExtension))
        throw new RuntimeException(String.format("Unexpected tile format without query support. Found: %s", tile.getOutputFormat().getClass().getSimpleName()));
    TileFormatWithQuerySupportExtension outputFormat = (TileFormatWithQuerySupportExtension) tile.getOutputFormat();
    // process parameters and generate query
    Optional<CrsTransformer> crsTransformer = Optional.empty();
    EpsgCrs targetCrs = query.getCrs().orElse(queryInput.getDefaultCrs());
    if (featureProvider.supportsCrs()) {
        EpsgCrs sourceCrs = featureProvider.crs().getNativeCrs();
        crsTransformer = crsTransformerFactory.getTransformer(sourceCrs, targetCrs);
    }
    List<Link> links = new DefaultLinksGenerator().generateLinks(requestContext.getUriCustomizer(), requestContext.getMediaType(), requestContext.getAlternateMediaTypes(), i18n, requestContext.getLanguage());
    String featureTypeId = apiData.getCollections().get(collectionId).getExtension(FeaturesCoreConfiguration.class).map(cfg -> cfg.getFeatureType().orElse(collectionId)).orElse(collectionId);
    FeatureTransformationContextTiles transformationContext;
    try {
        transformationContext = new ImmutableFeatureTransformationContextTiles.Builder().api(api).apiData(apiData).featureSchema(featureProvider.getData().getTypes().get(featureTypeId)).tile(tile).tileCache(tileCache).collectionId(collectionId).ogcApiRequest(requestContext).crsTransformer(crsTransformer).codelists(entityRegistry.getEntitiesForType(Codelist.class).stream().collect(Collectors.toMap(PersistentEntity::getId, c -> c))).defaultCrs(queryInput.getDefaultCrs()).links(links).isFeatureCollection(true).fields(query.getFields()).limit(query.getLimit()).offset(0).i18n(i18n).outputStream(new OutputStreamToByteConsumer()).build();
    } catch (Exception e) {
        throw new RuntimeException("Error building the tile transformation context.", e);
    }
    Optional<FeatureTokenEncoder<?>> encoder = outputFormat.getFeatureEncoder(transformationContext);
    if (outputFormat.supportsFeatureQuery() && encoder.isPresent()) {
        FeatureStream featureStream = featureProvider.queries().getFeatureStream(query);
        ResultReduced<byte[]> result = generateTile(featureStream, encoder.get(), transformationContext, outputFormat);
        // internal processing, no need to process headers
        return prepareSuccessResponse(requestContext.getApi(), requestContext, null).entity(result.reduced()).build();
    } else {
        throw new NotAcceptableException(MessageFormat.format("The requested media type {0} cannot be generated, because it does not support streaming.", requestContext.getMediaType().type()));
    }
}

Example 4 with OutputStreamToByteConsumer

use of de.ii.xtraplatform.streams.domain.OutputStreamToByteConsumer in project ldproxy by interactive-instruments.

the class FeatureEncoderFlatgeobuf method onStart.

@Override
public void onStart(ModifiableContext context) {
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Start generating Flatgeobuf file for collection {}.", collectionId);
    }
    this.processingStart = System.nanoTime();
    if (outputStream instanceof OutputStreamToByteConsumer) {
        ((OutputStreamToByteConsumer) outputStream).setByteConsumer(this::push);
    }
    try {
        push(Constants.MAGIC_BYTES);
        // 0 = unknown
        headerMeta = getHeader(collectionId, context.metadata().getNumberReturned().orElse(0));
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        HeaderMeta.write(headerMeta, baos, builder);
        push(baos.toByteArray());
        builder.clear();
    } catch (IOException e) {
        throw new IllegalStateException("Could not write to Flatgeobuf output stream: " + e.getMessage(), e);
    }
}

Example 5 with OutputStreamToByteConsumer

use of de.ii.xtraplatform.streams.domain.OutputStreamToByteConsumer in project ldproxy by interactive-instruments.

the class QueryHandlerRoutesImpl method computeRoute.

private Response computeRoute(QueryInputComputeRoute queryInput, ApiRequestContext requestContext) {
    OgcApi api = requestContext.getApi();
    OgcApiDataV2 apiData = api.getData();
    RouteDefinition routeDefinition = queryInput.getDefinition();
    String routeId = queryInput.getRouteId();
    FeatureQuery query = queryInput.getQuery();
    FeatureProvider2 featureProvider = queryInput.getFeatureProvider();
    RoutingConfiguration config = apiData.getExtension(RoutingConfiguration.class).orElseThrow(() -> new IllegalStateException("No routing configuration found for the API."));
    RoutesConfiguration providerConfig = featureProvider.getData().getExtension(RoutesConfiguration.class).orElseThrow(() -> new IllegalStateException("No routing configuration found for the feature provider of this API."));
    RouteFormatExtension outputFormat = api.getOutputFormat(RouteFormatExtension.class, requestContext.getMediaType(), "/routes", Optional.empty()).orElseThrow(() -> new NotAcceptableException(MessageFormat.format("The requested media type ''{0}'' is not supported for this resource.", requestContext.getMediaType())));
    RouteDefinitionInputs inputs = routeDefinition.getInputs();
    ImmutableRouteQuery.Builder routeQueryBuilder = ImmutableRouteQuery.builder().start(routeDefinition.getStart()).end(routeDefinition.getEnd()).wayPoints(routeDefinition.getWaypoints());
    String preference = inputs.getPreference().orElse(config.getDefaultPreference());
    routeQueryBuilder = processPreference(preference, providerConfig.getPreferences(), routeQueryBuilder);
    String mode = inputs.getMode().orElse(config.getDefaultMode());
    routeQueryBuilder = processMode(mode, providerConfig.getModes(), routeQueryBuilder);
    ImmutableList.Builder<String> flagBuilder = new ImmutableList.Builder<>();
    for (String flag : inputs.getAdditionalFlags()) {
        flagBuilder = processFlag(flag, config.getAdditionalFlags(), flagBuilder);
    }
    routeQueryBuilder.flags(flagBuilder.build());
    if (!inputs.getWeight().isEmpty() && !config.supportsWeightRestrictions()) {
        throw new IllegalArgumentException("This API does not support weight restrictions as part of the definition of a route.");
    }
    routeQueryBuilder.weight(inputs.getWeight());
    if (!inputs.getHeight().isEmpty() && !config.supportsHeightRestrictions()) {
        throw new IllegalArgumentException("This API does not support weight restrictions as part of the definition of a route.");
    }
    routeQueryBuilder.height(inputs.getHeight());
    Optional<Geometry.MultiPolygon> obstacles = routeDefinition.getObstacles();
    if (!obstacles.isEmpty() && !config.supportsObstacles()) {
        throw new IllegalArgumentException("This API does not support obstacles as part of the definition of a route.");
    }
    routeQueryBuilder.obstacles(obstacles);
    EpsgCrs waypointCrs = routeDefinition.getWaypointsCrs();
    if (!crsSupport.isSupported(apiData, waypointCrs)) {
        throw new IllegalArgumentException(String.format("The parameter 'coordRefSys' in the route definition is invalid: the crs '%s' is not supported", waypointCrs.toUriString()));
    }
    if (!routeDefinition.getWaypoints().isEmpty() && !config.supportsIntermediateWaypoints()) {
        throw new IllegalArgumentException(String.format("This API does not support waypoints in addition to the start and end location. The following waypoints were provided: %s", routeDefinition.getWaypoints().stream().map(p -> p.getCoordinates().get(0).toString()).collect(Collectors.joining(","))));
    }
    RouteQuery routeQuery = routeQueryBuilder.build();
    LOGGER.debug("Route Query: {}", routeQuery);
    query = ImmutableFeatureQuery.builder().from(query).addExtensions(routeQuery).build();
    Optional<CrsTransformer> crsTransformer = Optional.empty();
    boolean swapCoordinates = false;
    EpsgCrs sourceCrs = null;
    EpsgCrs targetCrs = query.getCrs().orElse(queryInput.getDefaultCrs());
    if (featureProvider.supportsCrs()) {
        sourceCrs = featureProvider.crs().getNativeCrs();
        crsTransformer = crsTransformerFactory.getTransformer(sourceCrs, targetCrs);
    }
    List<ApiMediaType> alternateMediaTypes = requestContext.getAlternateMediaTypes();
    List<Link> links = ImmutableList.of();
    if (config.isManageRoutesEnabled()) {
        links = new RoutesLinksGenerator().generateRouteLinks(routeId, inputs.getName(), requestContext.getUriCustomizer(), requestContext.getMediaType(), i18n, requestContext.getLanguage());
    }
    FeatureTransformationContextRoutes transformationContext = ImmutableFeatureTransformationContextRoutes.builder().api(api).apiData(api.getData()).featureSchema(featureProvider.getData().getTypes().get(queryInput.getFeatureTypeId())).collectionId("not_applicable").ogcApiRequest(requestContext).crsTransformer(crsTransformer).codelists(entityRegistry.getEntitiesForType(Codelist.class).stream().collect(Collectors.toMap(PersistentEntity::getId, c -> c))).defaultCrs(queryInput.getDefaultCrs()).sourceCrs(Optional.ofNullable(sourceCrs)).crs(targetCrs).links(links).isFeatureCollection(true).isHitsOnly(query.hitsOnly()).isPropertyOnly(query.propertyOnly()).fields(query.getFields()).limit(query.getLimit()).offset(query.getOffset()).maxAllowableOffset(query.getMaxAllowableOffset()).geometryPrecision(query.getGeometryPrecision()).isHitsOnlyIfMore(false).showsFeatureSelfLink(false).name(inputs.getName()).format(outputFormat).outputStream(new OutputStreamToByteConsumer()).startTimeNano(System.nanoTime()).speedLimitUnit(queryInput.getSpeedLimitUnit()).elevationProfileSimplificationTolerance(queryInput.getElevationProfileSimplificationTolerance()).crsInfo(crsInfo).build();
    Optional<FeatureTokenEncoder<?>> encoder = outputFormat.getFeatureEncoder(transformationContext, Optional.empty());
    if (encoder.isEmpty()) {
        throw new NotAcceptableException(MessageFormat.format("The requested media type {0} cannot be generated, because it does not support streaming.", requestContext.getMediaType().type()));
    }
    FeatureStream featureStream = featureProvider.queries().getFeatureStream(query);
    StreamingOutput streamingOutput = stream(featureStream, encoder.get());
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try {
        streamingOutput.write(baos);
    } catch (IOException e) {
        throw new IllegalStateException("Feature stream error.", e);
    }
    byte[] result = baos.toByteArray();
    Date lastModified = null;
    EntityTag etag = getEtag(result);
    Response.ResponseBuilder response = evaluatePreconditions(requestContext, lastModified, etag);
    if (Objects.nonNull(response))
        return response.build();
    // write routeDefinition and route if managing routes is enabled
    if (config.isManageRoutesEnabled()) {
        try {
            List<Link> definitionLinks = new RoutesLinksGenerator().generateRouteDefinitionLinks(routeId, inputs.getName(), requestContext.getUriCustomizer(), RouteDefinitionFormatJson.MEDIA_TYPE, i18n, requestContext.getLanguage());
            routeRepository.writeRouteAndDefinition(apiData, routeId, outputFormat, result, routeDefinition, definitionLinks);
        } catch (IOException e) {
            LOGGER.error("Could not store route in route repository.", e);
            if (LOGGER.isDebugEnabled())
                LOGGER.debug("Stacktrace: ", e);
        }
    }
    return prepareSuccessResponse(requestContext, queryInput.getIncludeLinkHeader() ? links : null, lastModified, etag, queryInput.getCacheControl().orElse(null), queryInput.getExpires().orElse(null), targetCrs, true, String.format("%s.%s", inputs.getName().orElse("Route"), outputFormat.getMediaType().fileExtension())).entity(result).build();
}