Examples with DatasetFacet io.openlineage.client.OpenLineage.DatasetFacet used on opensource projects

Example 1 with DatasetFacet

use of io.openlineage.client.OpenLineage.DatasetFacet in project OpenLineage by OpenLineage.

the class OpenLineageRunEventBuilder method buildOutputDatasets.

private List<OpenLineage.OutputDataset> buildOutputDatasets(List<Object> nodes) {
    log.info("Visiting query plan {} with output dataset builders {}", openLineageContext.getQueryExecution(), outputDatasetBuilders);
    Function1<LogicalPlan, Collection<OutputDataset>> visitor = visitLogicalPlan(PlanUtils.merge(outputDatasetQueryPlanVisitors));
    List<OutputDataset> datasets = Stream.concat(buildDatasets(nodes, outputDatasetBuilders), openLineageContext.getQueryExecution().map(qe -> visitor.apply(qe.optimizedPlan())).map(Collection::stream).orElse(Stream.empty())).collect(Collectors.toList());
    OpenLineage openLineage = openLineageContext.getOpenLineage();
    if (!datasets.isEmpty()) {
        Map<String, OutputDatasetFacet> outputFacetsMap = new HashMap<>();
        nodes.forEach(event -> outputDatasetFacetBuilders.forEach(fn -> fn.accept(event, outputFacetsMap::put)));
        Map<String, DatasetFacet> datasetFacetsMap = new HashMap<>();
        nodes.forEach(event -> datasetFacetBuilders.forEach(fn -> fn.accept(event, datasetFacetsMap::put)));
        return datasets.stream().map(ds -> openLineage.newOutputDatasetBuilder().name(ds.getName()).namespace(ds.getNamespace()).outputFacets(mergeFacets(outputFacetsMap, ds.getOutputFacets(), OutputDatasetOutputFacets.class)).facets(mergeFacets(datasetFacetsMap, ds.getFacets(), DatasetFacets.class)).build()).collect(Collectors.toList());
    }
    return datasets;
}