Examples with RunEvent io.openlineage.client.OpenLineage.RunEvent used on opensource projects

Example 1 with RunEvent

use of io.openlineage.client.OpenLineage.RunEvent in project OpenLineage by OpenLineage.

the class OpenLineageTest method jsonSerialization.

@Test
public void jsonSerialization() throws JsonProcessingException {
    ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
    URI producer = URI.create("producer");
    OpenLineage ol = new OpenLineage(producer);
    UUID runId = UUID.randomUUID();
    RunFacets runFacets = ol.newRunFacetsBuilder().nominalTime(ol.newNominalTimeRunFacet(now, now)).build();
    Run run = ol.newRun(runId, runFacets);
    String name = "jobName";
    String namespace = "namespace";
    JobFacets jobFacets = ol.newJobFacetsBuilder().build();
    Job job = ol.newJob(namespace, name, jobFacets);
    List<InputDataset> inputs = Arrays.asList(ol.newInputDataset("ins", "input", null, null));
    List<OutputDataset> outputs = Arrays.asList(ol.newOutputDataset("ons", "output", null, null));
    RunEvent runStateUpdate = ol.newRunEvent(OpenLineage.RunEvent.EventType.START, now, run, job, inputs, outputs);
    String json = mapper.writeValueAsString(runStateUpdate);
    RunEvent read = mapper.readValue(json, RunEvent.class);
    assertEquals(producer, read.getProducer());
    assertEquals(runId, read.getRun().getRunId());
    assertEquals(name, read.getJob().getName());
    assertEquals(namespace, read.getJob().getNamespace());
    assertEquals(runStateUpdate.getEventType(), read.getEventType());
    assertEquals(runStateUpdate.getEventTime(), read.getEventTime());
    assertEquals(1, runStateUpdate.getInputs().size());
    NominalTimeRunFacet nominalTime = runStateUpdate.getRun().getFacets().getNominalTime();
    assertEquals(now, nominalTime.getNominalStartTime());
    assertEquals(now, nominalTime.getNominalEndTime());
    InputDataset inputDataset = runStateUpdate.getInputs().get(0);
    assertEquals("ins", inputDataset.getNamespace());
    assertEquals("input", inputDataset.getName());
    assertEquals(1, runStateUpdate.getOutputs().size());
    OutputDataset outputDataset = runStateUpdate.getOutputs().get(0);
    assertEquals("ons", outputDataset.getNamespace());
    assertEquals("output", outputDataset.getName());
    assertEquals(roundTrip(json), roundTrip(mapper.writeValueAsString(read)));
}

Example 2 with RunEvent

use of io.openlineage.client.OpenLineage.RunEvent in project OpenLineage by OpenLineage.

the class OpenLineageTest method factory.

@Test
public void factory() throws JsonProcessingException {
    ZonedDateTime now = ZonedDateTime.now(ZoneId.of("UTC"));
    URI producer = URI.create("producer");
    OpenLineage ol = new OpenLineage(producer);
    UUID runId = UUID.randomUUID();
    RunFacets runFacets = ol.newRunFacetsBuilder().nominalTime(ol.newNominalTimeRunFacetBuilder().nominalStartTime(now).nominalEndTime(now).build()).build();
    Run run = ol.newRunBuilder().runId(runId).facets(runFacets).build();
    String name = "jobName";
    String namespace = "namespace";
    JobFacets jobFacets = ol.newJobFacetsBuilder().build();
    Job job = ol.newJobBuilder().namespace(namespace).name(name).facets(jobFacets).build();
    List<InputDataset> inputs = Arrays.asList(ol.newInputDatasetBuilder().namespace("ins").name("input").facets(ol.newDatasetFacetsBuilder().version(ol.newDatasetVersionDatasetFacet("input-version")).build()).inputFacets(ol.newInputDatasetInputFacetsBuilder().dataQualityMetrics(ol.newDataQualityMetricsInputDatasetFacetBuilder().rowCount(10L).bytes(20L).columnMetrics(ol.newDataQualityMetricsInputDatasetFacetColumnMetricsBuilder().put("mycol", ol.newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalBuilder().count(10D).distinctCount(10L).max(30D).min(5D).nullCount(1L).sum(3000D).quantiles(ol.newDataQualityMetricsInputDatasetFacetColumnMetricsAdditionalQuantilesBuilder().put("25", 52D).build()).build()).build()).build()).build()).build());
    List<OutputDataset> outputs = Arrays.asList(ol.newOutputDatasetBuilder().namespace("ons").name("output").facets(ol.newDatasetFacetsBuilder().version(ol.newDatasetVersionDatasetFacet("output-version")).build()).outputFacets(ol.newOutputDatasetOutputFacetsBuilder().outputStatistics(ol.newOutputStatisticsOutputDatasetFacet(10L, 20L)).build()).build());
    RunEvent runStateUpdate = ol.newRunEventBuilder().eventType(OpenLineage.RunEvent.EventType.START).eventTime(now).run(run).job(job).inputs(inputs).outputs(outputs).build();
    ObjectMapper mapper = new ObjectMapper();
    mapper.registerModule(new JavaTimeModule());
    mapper.setSerializationInclusion(Include.NON_NULL);
    mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
    mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
    String json = mapper.writeValueAsString(runStateUpdate);
    {
        RunEvent read = mapper.readValue(json, RunEvent.class);
        assertEquals(producer, read.getProducer());
        assertEquals(runId, read.getRun().getRunId());
        assertEquals(name, read.getJob().getName());
        assertEquals(namespace, read.getJob().getNamespace());
        assertEquals(runStateUpdate.getEventType(), read.getEventType());
        assertEquals(runStateUpdate.getEventTime(), read.getEventTime());
        assertEquals(1, runStateUpdate.getInputs().size());
        InputDataset inputDataset = runStateUpdate.getInputs().get(0);
        assertEquals("ins", inputDataset.getNamespace());
        assertEquals("input", inputDataset.getName());
        assertEquals("input-version", inputDataset.getFacets().getVersion().getDatasetVersion());
        DataQualityMetricsInputDatasetFacet dq = inputDataset.getInputFacets().getDataQualityMetrics();
        assertEquals((Long) 10L, dq.getRowCount());
        assertEquals((Long) 20L, dq.getBytes());
        DataQualityMetricsInputDatasetFacetColumnMetricsAdditional colMetrics = dq.getColumnMetrics().getAdditionalProperties().get("mycol");
        assertEquals((Double) 10D, colMetrics.getCount());
        assertEquals((Long) 10L, colMetrics.getDistinctCount());
        assertEquals((Double) 30D, colMetrics.getMax());
        assertEquals((Double) 5D, colMetrics.getMin());
        assertEquals((Long) 1L, colMetrics.getNullCount());
        assertEquals((Double) 3000D, colMetrics.getSum());
        assertEquals((Double) 52D, colMetrics.getQuantiles().getAdditionalProperties().get("25"));
        assertEquals(1, runStateUpdate.getOutputs().size());
        OutputDataset outputDataset = runStateUpdate.getOutputs().get(0);
        assertEquals("ons", outputDataset.getNamespace());
        assertEquals("output", outputDataset.getName());
        assertEquals("output-version", outputDataset.getFacets().getVersion().getDatasetVersion());
        assertEquals(roundTrip(json), roundTrip(mapper.writeValueAsString(read)));
        assertEquals((Long) 10L, outputDataset.getOutputFacets().getOutputStatistics().getRowCount());
        assertEquals((Long) 20L, outputDataset.getOutputFacets().getOutputStatistics().getSize());
        assertEquals(json, mapper.writeValueAsString(read));
    }
    {
        io.openlineage.server.OpenLineage.RunEvent readServer = mapper.readValue(json, io.openlineage.server.OpenLineage.RunEvent.class);
        assertEquals(producer, readServer.getProducer());
        assertEquals(runId, readServer.getRun().getRunId());
        assertEquals(name, readServer.getJob().getName());
        assertEquals(namespace, readServer.getJob().getNamespace());
        assertEquals(runStateUpdate.getEventType().name(), readServer.getEventType().name());
        assertEquals(runStateUpdate.getEventTime(), readServer.getEventTime());
        assertEquals(json, mapper.writeValueAsString(readServer));
    }
}

Example 3 with RunEvent

use of io.openlineage.client.OpenLineage.RunEvent in project OpenLineage by OpenLineage.

the class SparkSQLExecutionContext method end.

public void end(SparkListenerSQLExecutionEnd endEvent) {
    log.debug("SparkListenerSQLExecutionEnd - executionId: {}", endEvent.executionId());
    // Maybe use QueryExecutionListener?
    if (!olContext.getQueryExecution().isPresent()) {
        log.info("No execution info {}", olContext);
        return;
    }
    RunEvent event = runEventBuilder.buildRun(buildParentFacet(), openLineage.newRunEventBuilder().eventTime(toZonedTime(endEvent.time())), buildJob(olContext.getQueryExecution().get()), endEvent);
    log.debug("Posting event for end {}: {}", executionId, event);
    eventEmitter.emit(event);
}

Example 4 with RunEvent

use of io.openlineage.client.OpenLineage.RunEvent in project OpenLineage by OpenLineage.

the class SparkSQLExecutionContext method start.

public void start(SparkListenerSQLExecutionStart startEvent) {
    log.debug("SparkListenerSQLExecutionStart - executionId: {}", startEvent.executionId());
    if (!olContext.getQueryExecution().isPresent()) {
        log.info("No execution info {}", olContext);
        return;
    }
    RunEvent event = runEventBuilder.buildRun(buildParentFacet(), openLineage.newRunEventBuilder().eventTime(toZonedTime(startEvent.time())), buildJob(olContext.getQueryExecution().get()), startEvent);
    log.debug("Posting event for start {}: {}", executionId, event);
    eventEmitter.emit(event);
}

Example 5 with RunEvent

use of io.openlineage.client.OpenLineage.RunEvent in project OpenLineage by OpenLineage.

the class SparkSQLExecutionContext method start.

// TODO: not invoked until https://github.com/OpenLineage/OpenLineage/issues/470 is completed
@Override
public void start(SparkListenerStageSubmitted stageSubmitted) {
    if (!olContext.getQueryExecution().isPresent()) {
        log.info("No execution info {}", olContext);
        return;
    }
    RunEvent event = runEventBuilder.buildRun(buildParentFacet(), openLineage.newRunEventBuilder().eventTime(ZonedDateTime.now(ZoneOffset.UTC)), buildJob(olContext.getQueryExecution().get()), stageSubmitted);
    log.debug("Posting event for stage submitted {}: {}", executionId, event);
    eventEmitter.emit(event);
}