Examples with DeltaCatalog org.apache.spark.sql.delta.catalog.DeltaCatalog used on opensource projects

Example 1 with DeltaCatalog

use of org.apache.spark.sql.delta.catalog.DeltaCatalog in project OpenLineage by OpenLineage.

the class DeltaHandler method getDatasetVersion.

@SneakyThrows
public Optional<String> getDatasetVersion(TableCatalog tableCatalog, Identifier identifier, Map<String, String> properties) {
    DeltaCatalog deltaCatalog = (DeltaCatalog) tableCatalog;
    Table table = deltaCatalog.loadTable(identifier);
    if (table instanceof DeltaTableV2) {
        DeltaTableV2 deltaTable = (DeltaTableV2) table;
        return Optional.of(Long.toString(deltaTable.snapshot().version()));
    }
    return Optional.empty();
}

Example 2 with DeltaCatalog

use of org.apache.spark.sql.delta.catalog.DeltaCatalog in project OpenLineage by OpenLineage.

the class DeltaHandler method getDatasetIdentifier.

@Override
public DatasetIdentifier getDatasetIdentifier(SparkSession session, TableCatalog tableCatalog, Identifier identifier, Map<String, String> properties) {
    DeltaCatalog catalog = (DeltaCatalog) tableCatalog;
    Optional<String> location;
    if (catalog.isPathIdentifier(identifier)) {
        location = Optional.of(identifier.name());
    } else {
        location = Optional.ofNullable(properties.get("location"));
    }
    // Delta uses spark2 catalog when location isn't specified.
    Path path = new Path(location.orElse(session.sessionState().catalog().defaultTablePath(TableIdentifier.apply(identifier.name(), Option.apply(Arrays.stream(identifier.namespace()).reduce((x, y) -> y).orElse(null)))).toString()));
    log.info(path.toString());
    return PathUtils.fromPath(path, "file");
}

Example 3 with DeltaCatalog

use of org.apache.spark.sql.delta.catalog.DeltaCatalog in project OpenLineage by OpenLineage.

the class DeltaHandlerTest method testGetVersionString.

@Test
public void testGetVersionString() {
    DeltaCatalog deltaCatalog = mock(DeltaCatalog.class);
    DeltaTableV2 deltaTable = mock(DeltaTableV2.class, RETURNS_DEEP_STUBS);
    Identifier identifier = Identifier.of(new String[] { "database", "schema" }, "table");
    DeltaHandler deltaHandler = new DeltaHandler();
    when(deltaCatalog.loadTable(identifier)).thenReturn(deltaTable);
    when(deltaTable.snapshot().version()).thenReturn(2L);
    Optional<String> version = deltaHandler.getDatasetVersion(deltaCatalog, identifier, Collections.emptyMap());
    assertTrue(version.isPresent());
    assertEquals(version.get(), "2");
}