use of org.apache.iceberg.spark.source.SparkTable in project iceberg by apache.
the class AncestorsOfProcedure method call.
@Override
public InternalRow[] call(InternalRow args) {
Identifier tableIdent = toIdentifier(args.getString(0), PARAMETERS[0].name());
Long toSnapshotId = args.isNullAt(1) ? null : args.getLong(1);
SparkTable sparkTable = loadSparkTable(tableIdent);
Table icebergTable = sparkTable.table();
if (toSnapshotId == null) {
toSnapshotId = icebergTable.currentSnapshot() != null ? icebergTable.currentSnapshot().snapshotId() : -1;
}
List<Long> snapshotIds = Lists.newArrayList(SnapshotUtil.ancestorIdsBetween(toSnapshotId, null, icebergTable::snapshot));
return toOutputRow(icebergTable, snapshotIds);
}
use of org.apache.iceberg.spark.source.SparkTable in project iceberg by apache.
the class BaseProcedure method execute.
private <T> T execute(Identifier ident, boolean refreshSparkCache, Function<org.apache.iceberg.Table, T> func) {
SparkTable sparkTable = loadSparkTable(ident);
org.apache.iceberg.Table icebergTable = sparkTable.table();
T result = func.apply(icebergTable);
if (refreshSparkCache) {
refreshSparkCache(ident, sparkTable);
}
return result;
}
use of org.apache.iceberg.spark.source.SparkTable in project iceberg by apache.
the class TestRemoveOrphanFilesAction3 method testSparkCatalogNamedHadoopTable.
@Test
public void testSparkCatalogNamedHadoopTable() throws Exception {
spark.conf().set("spark.sql.catalog.hadoop", "org.apache.iceberg.spark.SparkCatalog");
spark.conf().set("spark.sql.catalog.hadoop.type", "hadoop");
spark.conf().set("spark.sql.catalog.hadoop.warehouse", tableLocation);
SparkCatalog cat = (SparkCatalog) spark.sessionState().catalogManager().catalog("hadoop");
String[] database = { "default" };
Identifier id = Identifier.of(database, "table");
Map<String, String> options = Maps.newHashMap();
Transform[] transforms = {};
cat.createTable(id, SparkSchemaUtil.convert(SCHEMA), transforms, options);
SparkTable table = cat.loadTable(id);
spark.sql("INSERT INTO hadoop.default.table VALUES (1,1,1)");
String location = table.table().location().replaceFirst("file:", "");
new File(location + "/data/trashfile").createNewFile();
DeleteOrphanFiles.Result results = SparkActions.get().deleteOrphanFiles(table.table()).olderThan(System.currentTimeMillis() + 1000).execute();
Assert.assertTrue("trash file should be removed", StreamSupport.stream(results.orphanFileLocations().spliterator(), false).anyMatch(file -> file.contains("file:" + location + "/data/trashfile")));
}
use of org.apache.iceberg.spark.source.SparkTable in project iceberg by apache.
the class TestRemoveOrphanFilesAction3 method testSparkCatalogTable.
@Test
public void testSparkCatalogTable() throws Exception {
spark.conf().set("spark.sql.catalog.mycat", "org.apache.iceberg.spark.SparkCatalog");
spark.conf().set("spark.sql.catalog.mycat.type", "hadoop");
spark.conf().set("spark.sql.catalog.mycat.warehouse", tableLocation);
SparkCatalog cat = (SparkCatalog) spark.sessionState().catalogManager().catalog("mycat");
String[] database = { "default" };
Identifier id = Identifier.of(database, "table");
Map<String, String> options = Maps.newHashMap();
Transform[] transforms = {};
cat.createTable(id, SparkSchemaUtil.convert(SCHEMA), transforms, options);
SparkTable table = cat.loadTable(id);
spark.sql("INSERT INTO mycat.default.table VALUES (1,1,1)");
String location = table.table().location().replaceFirst("file:", "");
new File(location + "/data/trashfile").createNewFile();
DeleteOrphanFiles.Result results = SparkActions.get().deleteOrphanFiles(table.table()).olderThan(System.currentTimeMillis() + 1000).execute();
Assert.assertTrue("trash file should be removed", StreamSupport.stream(results.orphanFileLocations().spliterator(), false).anyMatch(file -> file.contains("file:" + location + "/data/trashfile")));
}
use of org.apache.iceberg.spark.source.SparkTable in project iceberg by apache.
the class TestRemoveOrphanFilesAction3 method testSparkSessionCatalogHadoopTable.
@Test
public void testSparkSessionCatalogHadoopTable() throws Exception {
spark.conf().set("spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog");
spark.conf().set("spark.sql.catalog.spark_catalog.type", "hadoop");
spark.conf().set("spark.sql.catalog.spark_catalog.warehouse", tableLocation);
SparkSessionCatalog cat = (SparkSessionCatalog) spark.sessionState().catalogManager().v2SessionCatalog();
String[] database = { "default" };
Identifier id = Identifier.of(database, "table");
Map<String, String> options = Maps.newHashMap();
Transform[] transforms = {};
cat.createTable(id, SparkSchemaUtil.convert(SCHEMA), transforms, options);
SparkTable table = (SparkTable) cat.loadTable(id);
spark.sql("INSERT INTO default.table VALUES (1,1,1)");
String location = table.table().location().replaceFirst("file:", "");
new File(location + "/data/trashfile").createNewFile();
DeleteOrphanFiles.Result results = SparkActions.get().deleteOrphanFiles(table.table()).olderThan(System.currentTimeMillis() + 1000).execute();
Assert.assertTrue("trash file should be removed", StreamSupport.stream(results.orphanFileLocations().spliterator(), false).anyMatch(file -> file.contains("file:" + location + "/data/trashfile")));
}
Aggregations