Examples with SparkListenerSQLExecutionEnd org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd used on opensource projects

Example 1 with SparkListenerSQLExecutionEnd

use of org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd in project OpenLineage by OpenLineage.

the class LogicalPlanRunFacetBuilderTest method testIsNotDefinedWithoutQueryExecution.

@Test
public void testIsNotDefinedWithoutQueryExecution() {
    LogicalPlanRunFacetBuilder builder = new LogicalPlanRunFacetBuilder(OpenLineageContext.builder().sparkContext(sparkContext).openLineage(new OpenLineage(OpenLineageClient.OPEN_LINEAGE_CLIENT_URI)).build());
    assertThat(builder.isDefinedAt(new SparkListenerSQLExecutionStart(1L, "", "", "", null, 1L))).isFalse();
    assertThat(builder.isDefinedAt(new SparkListenerSQLExecutionEnd(1L, 1L))).isFalse();
    assertThat(builder.isDefinedAt(new SparkListenerJobEnd(1, 1L, JobSucceeded$.MODULE$))).isFalse();
    assertThat(builder.isDefinedAt(new SparkListenerJobStart(1, 1L, Seq$.MODULE$.empty(), new Properties()))).isFalse();
}

Example 2 with SparkListenerSQLExecutionEnd

use of org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd in project OpenLineage by OpenLineage.

the class LogicalPlanRunFacetBuilderTest method testIsDefined.

@Test
public void testIsDefined() {
    LogicalPlanRunFacetBuilder builder = new LogicalPlanRunFacetBuilder(OpenLineageContext.builder().sparkContext(sparkContext).openLineage(new OpenLineage(OpenLineageClient.OPEN_LINEAGE_CLIENT_URI)).queryExecution(queryExecution).build());
    assertThat(builder.isDefinedAt(new SparkListenerSQLExecutionStart(1L, "", "", "", null, 1L))).isTrue();
    assertThat(builder.isDefinedAt(new SparkListenerSQLExecutionEnd(1L, 1L))).isTrue();
    assertThat(builder.isDefinedAt(new SparkListenerJobEnd(1, 1L, JobSucceeded$.MODULE$))).isTrue();
    assertThat(builder.isDefinedAt(new SparkListenerJobStart(1, 1L, Seq$.MODULE$.empty(), new Properties()))).isFalse();
}

Example 3 with SparkListenerSQLExecutionEnd

use of org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd in project OpenLineage by OpenLineage.

the class LogicalPlanRunFacetBuilderTest method testBuild.

@Test
public void testBuild() {
    LogicalPlanRunFacetBuilder builder = new LogicalPlanRunFacetBuilder(OpenLineageContext.builder().sparkContext(sparkContext).openLineage(new OpenLineage(OpenLineageClient.OPEN_LINEAGE_CLIENT_URI)).queryExecution(queryExecution).build());
    Map<String, RunFacet> facetMap = new HashMap<>();
    builder.build(new SparkListenerSQLExecutionEnd(1L, 1L), facetMap::put);
    assertThat(facetMap).hasEntrySatisfying("spark.logicalPlan", facet -> assertThat(facet).hasFieldOrPropertyWithValue("plan", queryExecution.optimizedPlan().toJSON()));
}

Example 4 with SparkListenerSQLExecutionEnd

use of org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd in project OpenLineage by OpenLineage.

the class SparkVersionFacetBuilderTest method testBuild.

@Test
public void testBuild() {
    SparkVersionFacetBuilder builder = new SparkVersionFacetBuilder(OpenLineageContext.builder().sparkContext(sparkContext).openLineage(new OpenLineage(OpenLineageClient.OPEN_LINEAGE_CLIENT_URI)).build());
    Map<String, RunFacet> runFacetMap = new HashMap<>();
    builder.build(new SparkListenerSQLExecutionEnd(1, 1L), runFacetMap::put);
    assertThat(runFacetMap).hasEntrySatisfying("spark_version", facet -> assertThat(facet).isInstanceOf(SparkVersionFacet.class).hasFieldOrPropertyWithValue("sparkVersion", sparkContext.version()));
}

Example 5 with SparkListenerSQLExecutionEnd

use of org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd in project OpenLineage by OpenLineage.

the class OutputStatisticsOutputDatasetFacetBuilderTest method testIsDefined.

@Test
public void testIsDefined() {
    OutputStatisticsOutputDatasetFacetBuilder builder = new OutputStatisticsOutputDatasetFacetBuilder(OpenLineageContext.builder().openLineage(new OpenLineage(OpenLineageClient.OPEN_LINEAGE_CLIENT_URI)).sparkContext(sparkContext).build());
    assertThat(builder.isDefinedAt(new SparkListenerJobEnd(1, 1L, JobSucceeded$.MODULE$))).isTrue();
    assertThat(builder.isDefinedAt(new SparkListenerSQLExecutionEnd(1L, 1L))).isFalse();
}