Examples with OutputDatasetFacet io.openlineage.client.OpenLineage.OutputDatasetFacet used on opensource projects

Example 1 with OutputDatasetFacet

use of io.openlineage.client.OpenLineage.OutputDatasetFacet in project OpenLineage by OpenLineage.

the class OutputStatisticsOutputDatasetFacetBuilderTest method testBuild.

@Test
public void testBuild() {
    OutputStatisticsOutputDatasetFacetBuilder builder = new OutputStatisticsOutputDatasetFacetBuilder(OpenLineageContext.builder().openLineage(new OpenLineage(OpenLineageClient.OPEN_LINEAGE_CLIENT_URI)).sparkContext(sparkContext).build());
    JobMetricsHolder.getInstance().addJobStages(1, Collections.singleton(1));
    TaskMetrics taskMetrics = new TaskMetrics();
    taskMetrics.outputMetrics().setBytesWritten(10L);
    taskMetrics.outputMetrics().setRecordsWritten(100L);
    JobMetricsHolder.getInstance().addMetrics(1, taskMetrics);
    Map<String, OutputDatasetFacet> facetsMap = new HashMap<>();
    builder.build(new SparkListenerJobEnd(1, 1L, JobSucceeded$.MODULE$), facetsMap::put);
    assertThat(facetsMap).hasEntrySatisfying("outputStatistics", facet -> assertThat(facet).hasFieldOrPropertyWithValue("rowCount", 100L).hasFieldOrPropertyWithValue("size", 10L));
}

Example 2 with OutputDatasetFacet

use of io.openlineage.client.OpenLineage.OutputDatasetFacet in project OpenLineage by OpenLineage.

the class OpenLineageRunEventBuilder method buildOutputDatasets.

private List<OpenLineage.OutputDataset> buildOutputDatasets(List<Object> nodes) {
    log.info("Visiting query plan {} with output dataset builders {}", openLineageContext.getQueryExecution(), outputDatasetBuilders);
    Function1<LogicalPlan, Collection<OutputDataset>> visitor = visitLogicalPlan(PlanUtils.merge(outputDatasetQueryPlanVisitors));
    List<OutputDataset> datasets = Stream.concat(buildDatasets(nodes, outputDatasetBuilders), openLineageContext.getQueryExecution().map(qe -> visitor.apply(qe.optimizedPlan())).map(Collection::stream).orElse(Stream.empty())).collect(Collectors.toList());
    OpenLineage openLineage = openLineageContext.getOpenLineage();
    if (!datasets.isEmpty()) {
        Map<String, OutputDatasetFacet> outputFacetsMap = new HashMap<>();
        nodes.forEach(event -> outputDatasetFacetBuilders.forEach(fn -> fn.accept(event, outputFacetsMap::put)));
        Map<String, DatasetFacet> datasetFacetsMap = new HashMap<>();
        nodes.forEach(event -> datasetFacetBuilders.forEach(fn -> fn.accept(event, datasetFacetsMap::put)));
        return datasets.stream().map(ds -> openLineage.newOutputDatasetBuilder().name(ds.getName()).namespace(ds.getNamespace()).outputFacets(mergeFacets(outputFacetsMap, ds.getOutputFacets(), OutputDatasetOutputFacets.class)).facets(mergeFacets(datasetFacetsMap, ds.getFacets(), DatasetFacets.class)).build()).collect(Collectors.toList());
    }
    return datasets;
}