Search in sources :

Example 1 with DatabricksEnvironmentFacetBuilder

use of io.openlineage.spark.agent.facets.builder.DatabricksEnvironmentFacetBuilder in project OpenLineage by OpenLineage.

the class InternalEventHandlerFactory method createRunFacetBuilders.

@Override
public Collection<CustomFacetBuilder<?, ? extends RunFacet>> createRunFacetBuilders(OpenLineageContext context) {
    Builder<CustomFacetBuilder<?, ? extends RunFacet>> listBuilder;
    listBuilder = ImmutableList.<CustomFacetBuilder<?, ? extends RunFacet>>builder().addAll(generate(eventHandlerFactories, factory -> factory.createRunFacetBuilders((context)))).add(new ErrorFacetBuilder(), new LogicalPlanRunFacetBuilder(context), new SparkVersionFacetBuilder(context));
    if (DatabricksEnvironmentFacetBuilder.isDatabricksRuntime()) {
        listBuilder.add(new DatabricksEnvironmentFacetBuilder(context));
    }
    return listBuilder.build();
}
Also used : Spliterators(java.util.Spliterators) InputDataset(io.openlineage.client.OpenLineage.InputDataset) OutputDatasetFacet(io.openlineage.client.OpenLineage.OutputDatasetFacet) Function(java.util.function.Function) SparkVersionFacetBuilder(io.openlineage.spark.agent.facets.builder.SparkVersionFacetBuilder) ImmutableList(com.google.common.collect.ImmutableList) OutputDataset(io.openlineage.client.OpenLineage.OutputDataset) ErrorFacetBuilder(io.openlineage.spark.agent.facets.builder.ErrorFacetBuilder) OutputStatisticsOutputDatasetFacetBuilder(io.openlineage.spark.agent.facets.builder.OutputStatisticsOutputDatasetFacetBuilder) JobFacet(io.openlineage.client.OpenLineage.JobFacet) DatabricksEnvironmentFacetBuilder(io.openlineage.spark.agent.facets.builder.DatabricksEnvironmentFacetBuilder) StreamSupport(java.util.stream.StreamSupport) LogicalPlanRunFacetBuilder(io.openlineage.spark.agent.facets.builder.LogicalPlanRunFacetBuilder) LogicalPlan(org.apache.spark.sql.catalyst.plans.logical.LogicalPlan) PartialFunction(scala.PartialFunction) OpenLineageContext(io.openlineage.spark.api.OpenLineageContext) Collection(java.util.Collection) InputDatasetFacet(io.openlineage.client.OpenLineage.InputDatasetFacet) ServiceLoader(java.util.ServiceLoader) DatasetFacet(io.openlineage.client.OpenLineage.DatasetFacet) Collectors(java.util.stream.Collectors) List(java.util.List) OpenLineageEventHandlerFactory(io.openlineage.spark.api.OpenLineageEventHandlerFactory) CustomFacetBuilder(io.openlineage.spark.api.CustomFacetBuilder) Builder(com.google.common.collect.ImmutableList.Builder) Spliterator(java.util.Spliterator) RunFacet(io.openlineage.client.OpenLineage.RunFacet) ErrorFacetBuilder(io.openlineage.spark.agent.facets.builder.ErrorFacetBuilder) LogicalPlanRunFacetBuilder(io.openlineage.spark.agent.facets.builder.LogicalPlanRunFacetBuilder) DatabricksEnvironmentFacetBuilder(io.openlineage.spark.agent.facets.builder.DatabricksEnvironmentFacetBuilder) RunFacet(io.openlineage.client.OpenLineage.RunFacet) CustomFacetBuilder(io.openlineage.spark.api.CustomFacetBuilder) SparkVersionFacetBuilder(io.openlineage.spark.agent.facets.builder.SparkVersionFacetBuilder)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)1 Builder (com.google.common.collect.ImmutableList.Builder)1 DatasetFacet (io.openlineage.client.OpenLineage.DatasetFacet)1 InputDataset (io.openlineage.client.OpenLineage.InputDataset)1 InputDatasetFacet (io.openlineage.client.OpenLineage.InputDatasetFacet)1 JobFacet (io.openlineage.client.OpenLineage.JobFacet)1 OutputDataset (io.openlineage.client.OpenLineage.OutputDataset)1 OutputDatasetFacet (io.openlineage.client.OpenLineage.OutputDatasetFacet)1 RunFacet (io.openlineage.client.OpenLineage.RunFacet)1 DatabricksEnvironmentFacetBuilder (io.openlineage.spark.agent.facets.builder.DatabricksEnvironmentFacetBuilder)1 ErrorFacetBuilder (io.openlineage.spark.agent.facets.builder.ErrorFacetBuilder)1 LogicalPlanRunFacetBuilder (io.openlineage.spark.agent.facets.builder.LogicalPlanRunFacetBuilder)1 OutputStatisticsOutputDatasetFacetBuilder (io.openlineage.spark.agent.facets.builder.OutputStatisticsOutputDatasetFacetBuilder)1 SparkVersionFacetBuilder (io.openlineage.spark.agent.facets.builder.SparkVersionFacetBuilder)1 CustomFacetBuilder (io.openlineage.spark.api.CustomFacetBuilder)1 OpenLineageContext (io.openlineage.spark.api.OpenLineageContext)1 OpenLineageEventHandlerFactory (io.openlineage.spark.api.OpenLineageEventHandlerFactory)1 Collection (java.util.Collection)1 List (java.util.List)1 ServiceLoader (java.util.ServiceLoader)1