Examples with SparkCatalog org.apache.iceberg.spark.SparkCatalog used on opensource projects

Example 6 with SparkCatalog

use of org.apache.iceberg.spark.SparkCatalog in project iceberg by apache.

the class TestRemoveOrphanFilesAction3 method testSparkCatalogTable.

@Test
public void testSparkCatalogTable() throws Exception {
    spark.conf().set("spark.sql.catalog.mycat", "org.apache.iceberg.spark.SparkCatalog");
    spark.conf().set("spark.sql.catalog.mycat.type", "hadoop");
    spark.conf().set("spark.sql.catalog.mycat.warehouse", tableLocation);
    SparkCatalog cat = (SparkCatalog) spark.sessionState().catalogManager().catalog("mycat");
    String[] database = { "default" };
    Identifier id = Identifier.of(database, "table");
    Map<String, String> options = Maps.newHashMap();
    Transform[] transforms = {};
    cat.createTable(id, SparkSchemaUtil.convert(SCHEMA), transforms, options);
    SparkTable table = cat.loadTable(id);
    spark.sql("INSERT INTO mycat.default.table VALUES (1,1,1)");
    String location = table.table().location().replaceFirst("file:", "");
    new File(location + "/data/trashfile").createNewFile();
    DeleteOrphanFiles.Result results = SparkActions.get().deleteOrphanFiles(table.table()).olderThan(System.currentTimeMillis() + 1000).execute();
    Assert.assertTrue("trash file should be removed", StreamSupport.stream(results.orphanFileLocations().spliterator(), false).anyMatch(file -> file.contains("file:" + location + "/data/trashfile")));
}

Example 7 with SparkCatalog

use of org.apache.iceberg.spark.SparkCatalog in project iceberg by apache.

the class TestPathIdentifier method before.

@Before
public void before() throws IOException {
    tableLocation = temp.newFolder();
    identifier = new PathIdentifier(tableLocation.getAbsolutePath());
    sparkCatalog = new SparkCatalog();
    sparkCatalog.initialize("test", new CaseInsensitiveStringMap(ImmutableMap.of()));
}

Example 8 with SparkCatalog

use of org.apache.iceberg.spark.SparkCatalog in project iceberg by apache.

the class TestSparkCatalogCacheExpiration method testCacheEnabledAndExpirationDisabled.

@Test
public void testCacheEnabledAndExpirationDisabled() {
    SparkCatalog sparkCatalog = getSparkCatalog("expiration_disabled");
    Assertions.assertThat(sparkCatalog).extracting("cacheEnabled").isEqualTo(true);
    Assertions.assertThat(sparkCatalog).extracting("icebergCatalog").isInstanceOfSatisfying(CachingCatalog.class, icebergCatalog -> {
        Assertions.assertThat(icebergCatalog).extracting("expirationIntervalMillis").isEqualTo(-1L);
    });
}

Example 9 with SparkCatalog

use of org.apache.iceberg.spark.SparkCatalog in project iceberg by apache.

the class TestSparkCatalogCacheExpiration method testCacheDisabledImplicitly.

@Test
public void testCacheDisabledImplicitly() {
    SparkCatalog sparkCatalog = getSparkCatalog("cache_disabled_implicitly");
    Assertions.assertThat(sparkCatalog).extracting("cacheEnabled").isEqualTo(false);
    Assertions.assertThat(sparkCatalog).extracting("icebergCatalog").isInstanceOfSatisfying(Catalog.class, icebergCatalog -> Assertions.assertThat(icebergCatalog).isNotInstanceOf(CachingCatalog.class));
}

Example 10 with SparkCatalog

use of org.apache.iceberg.spark.SparkCatalog in project OpenLineage by OpenLineage.

the class IcebergHandlerTest method testGetVersionString.

@Test
public void testGetVersionString() throws NoSuchTableException {
    SparkCatalog sparkCatalog = mock(SparkCatalog.class);
    SparkTable sparkTable = mock(SparkTable.class, RETURNS_DEEP_STUBS);
    Identifier identifier = Identifier.of(new String[] { "database", "schema" }, "table");
    when(sparkCatalog.loadTable(identifier)).thenReturn(sparkTable);
    when(sparkTable.table().currentSnapshot().snapshotId()).thenReturn(1500100900L);
    Optional<String> version = icebergHandler.getDatasetVersion(sparkCatalog, identifier, Collections.emptyMap());
    assertTrue(version.isPresent());
    assertEquals(version.get(), "1500100900");
}