Examples with CatalogManager org.apache.spark.sql.connector.catalog.CatalogManager used on opensource projects

Example 1 with CatalogManager

use of org.apache.spark.sql.connector.catalog.CatalogManager in project iceberg by apache.

the class Spark3Util method catalogAndIdentifier.

/**
 * A modified version of Spark's LookupCatalog.CatalogAndIdentifier.unapply
 * Attempts to find the catalog and identifier a multipart identifier represents
 * @param spark Spark session to use for resolution
 * @param nameParts Multipart identifier representing a table
 * @param defaultCatalog Catalog to use if none is specified
 * @return The CatalogPlugin and Identifier for the table
 */
public static CatalogAndIdentifier catalogAndIdentifier(SparkSession spark, List<String> nameParts, CatalogPlugin defaultCatalog) {
    CatalogManager catalogManager = spark.sessionState().catalogManager();
    String[] currentNamespace;
    if (defaultCatalog.equals(catalogManager.currentCatalog())) {
        currentNamespace = catalogManager.currentNamespace();
    } else {
        currentNamespace = defaultCatalog.defaultNamespace();
    }
    Pair<CatalogPlugin, Identifier> catalogIdentifier = SparkUtil.catalogAndIdentifier(nameParts, catalogName -> {
        try {
            return catalogManager.catalog(catalogName);
        } catch (Exception e) {
            return null;
        }
    }, Identifier::of, defaultCatalog, currentNamespace);
    return new CatalogAndIdentifier(catalogIdentifier);
}

Example 2 with CatalogManager

use of org.apache.spark.sql.connector.catalog.CatalogManager in project iceberg by apache.

the class IcebergSource method catalogAndIdentifier.

private Spark3Util.CatalogAndIdentifier catalogAndIdentifier(CaseInsensitiveStringMap options) {
    Preconditions.checkArgument(options.containsKey("path"), "Cannot open table: path is not set");
    SparkSession spark = SparkSession.active();
    setupDefaultSparkCatalog(spark);
    String path = options.get("path");
    Long snapshotId = propertyAsLong(options, SparkReadOptions.SNAPSHOT_ID);
    Long asOfTimestamp = propertyAsLong(options, SparkReadOptions.AS_OF_TIMESTAMP);
    Preconditions.checkArgument(asOfTimestamp == null || snapshotId == null, "Cannot specify both snapshot-id (%s) and as-of-timestamp (%s)", snapshotId, asOfTimestamp);
    String selector = null;
    if (snapshotId != null) {
        selector = SNAPSHOT_ID + snapshotId;
    }
    if (asOfTimestamp != null) {
        selector = AT_TIMESTAMP + asOfTimestamp;
    }
    CatalogManager catalogManager = spark.sessionState().catalogManager();
    if (path.contains("/")) {
        // contains a path. Return iceberg default catalog and a PathIdentifier
        String newPath = (selector == null) ? path : path + "#" + selector;
        return new Spark3Util.CatalogAndIdentifier(catalogManager.catalog(DEFAULT_CATALOG_NAME), new PathIdentifier(newPath));
    }
    final Spark3Util.CatalogAndIdentifier catalogAndIdentifier = Spark3Util.catalogAndIdentifier("path or identifier", spark, path);
    Identifier ident = identifierWithSelector(catalogAndIdentifier.identifier(), selector);
    if (catalogAndIdentifier.catalog().name().equals("spark_catalog") && !(catalogAndIdentifier.catalog() instanceof SparkSessionCatalog)) {
        // catalog is a session catalog but does not support Iceberg. Use Iceberg instead.
        return new Spark3Util.CatalogAndIdentifier(catalogManager.catalog(DEFAULT_CATALOG_NAME), ident);
    } else {
        return new Spark3Util.CatalogAndIdentifier(catalogAndIdentifier.catalog(), ident);
    }
}

Example 3 with CatalogManager

use of org.apache.spark.sql.connector.catalog.CatalogManager in project iceberg by apache.

the class TestSparkTable method testTableEquality.

@Test
public void testTableEquality() throws NoSuchTableException {
    CatalogManager catalogManager = spark.sessionState().catalogManager();
    TableCatalog catalog = (TableCatalog) catalogManager.catalog(catalogName);
    Identifier identifier = Identifier.of(tableIdent.namespace().levels(), tableIdent.name());
    SparkTable table1 = (SparkTable) catalog.loadTable(identifier);
    SparkTable table2 = (SparkTable) catalog.loadTable(identifier);
    // different instances pointing to the same table must be equivalent
    Assert.assertNotSame("References must be different", table1, table2);
    Assert.assertEquals("Tables must be equivalent", table1, table2);
}

Example 4 with CatalogManager

use of org.apache.spark.sql.connector.catalog.CatalogManager in project iceberg by apache.

the class TestAlterTablePartitionFields method sparkTable.

private SparkTable sparkTable() throws Exception {
    validationCatalog.loadTable(tableIdent).refresh();
    CatalogManager catalogManager = spark.sessionState().catalogManager();
    TableCatalog catalog = (TableCatalog) catalogManager.catalog(catalogName);
    Identifier identifier = Identifier.of(tableIdent.namespace().levels(), tableIdent.name());
    return (SparkTable) catalog.loadTable(identifier);
}