Examples with CatalogManager org.apache.flink.table.catalog.CatalogManager used on opensource projects

Example 1 with CatalogManager

use of org.apache.flink.table.catalog.CatalogManager in project flink by apache.

the class HiveParser method parse.

@Override
public List<Operation> parse(String statement) {
    CatalogManager catalogManager = getCatalogManager();
    Catalog currentCatalog = catalogManager.getCatalog(catalogManager.getCurrentCatalog()).orElse(null);
    if (!(currentCatalog instanceof HiveCatalog)) {
        LOG.warn("Current catalog is not HiveCatalog. Falling back to Flink's planner.");
        return super.parse(statement);
    }
    HiveConf hiveConf = new HiveConf(((HiveCatalog) currentCatalog).getHiveConf());
    hiveConf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
    hiveConf.set("hive.allow.udf.load.on.demand", "false");
    hiveConf.setVar(HiveConf.ConfVars.HIVE_EXECUTION_ENGINE, "mr");
    HiveShim hiveShim = HiveShimLoader.loadHiveShim(((HiveCatalog) currentCatalog).getHiveVersion());
    try {
        // creates SessionState
        startSessionState(hiveConf, catalogManager);
        // We override Hive's grouping function. Refer to the implementation for more details.
        hiveShim.registerTemporaryFunction("grouping", HiveGenericUDFGrouping.class);
        return processCmd(statement, hiveConf, hiveShim, (HiveCatalog) currentCatalog);
    } finally {
        clearSessionState();
    }
}

Example 2 with CatalogManager

use of org.apache.flink.table.catalog.CatalogManager in project flink by apache.

the class SessionContext method create.

// --------------------------------------------------------------------------------------------
// Helper method to create
// --------------------------------------------------------------------------------------------
public static SessionContext create(DefaultContext defaultContext, String sessionId) {
    // --------------------------------------------------------------------------------------------------------------
    // Init config
    // --------------------------------------------------------------------------------------------------------------
    Configuration configuration = defaultContext.getFlinkConfig().clone();
    // --------------------------------------------------------------------------------------------------------------
    // Init classloader
    // --------------------------------------------------------------------------------------------------------------
    URLClassLoader classLoader = ClientUtils.buildUserCodeClassLoader(defaultContext.getDependencies(), Collections.emptyList(), SessionContext.class.getClassLoader(), configuration);
    // --------------------------------------------------------------------------------------------------------------
    // Init session state
    // --------------------------------------------------------------------------------------------------------------
    ModuleManager moduleManager = new ModuleManager();
    final EnvironmentSettings settings = EnvironmentSettings.fromConfiguration(configuration);
    CatalogManager catalogManager = CatalogManager.newBuilder().classLoader(classLoader).config(configuration).defaultCatalog(settings.getBuiltInCatalogName(), new GenericInMemoryCatalog(settings.getBuiltInCatalogName(), settings.getBuiltInDatabaseName())).build();
    FunctionCatalog functionCatalog = new FunctionCatalog(configuration, catalogManager, moduleManager);
    SessionState sessionState = new SessionState(catalogManager, moduleManager, functionCatalog);
    // --------------------------------------------------------------------------------------------------------------
    // Init ExecutionContext
    // --------------------------------------------------------------------------------------------------------------
    ExecutionContext executionContext = new ExecutionContext(configuration, classLoader, sessionState);
    return new SessionContext(defaultContext, sessionId, configuration, classLoader, sessionState, executionContext);
}

Example 3 with CatalogManager

use of org.apache.flink.table.catalog.CatalogManager in project flink by apache.

the class DynamicTableSourceSpecSerdeTest method testDynamicTableSourceSpecSerde.

@ParameterizedTest
@MethodSource("testDynamicTableSinkSpecSerde")
void testDynamicTableSourceSpecSerde(DynamicTableSourceSpec spec) throws IOException {
    PlannerMocks plannerMocks = PlannerMocks.create();
    CatalogManager catalogManager = plannerMocks.getCatalogManager();
    catalogManager.createTable(spec.getContextResolvedTable().getResolvedTable(), spec.getContextResolvedTable().getIdentifier(), false);
    SerdeContext serdeCtx = configuredSerdeContext(catalogManager, plannerMocks.getTableConfig());
    // Re-init the spec to be permanent with correct catalog
    spec = new DynamicTableSourceSpec(ContextResolvedTable.permanent(spec.getContextResolvedTable().getIdentifier(), catalogManager.getCatalog(catalogManager.getCurrentCatalog()).get(), spec.getContextResolvedTable().getResolvedTable()), spec.getSourceAbilities());
    String actualJson = toJson(serdeCtx, spec);
    DynamicTableSourceSpec actual = toObject(serdeCtx, actualJson, DynamicTableSourceSpec.class);
    assertThat(actual.getContextResolvedTable()).isEqualTo(spec.getContextResolvedTable());
    assertThat(actual.getSourceAbilities()).isEqualTo(spec.getSourceAbilities());
    assertThat(actual.getScanTableSource(plannerMocks.getPlannerContext().getFlinkContext())).isNotNull();
}

Example 4 with CatalogManager

use of org.apache.flink.table.catalog.CatalogManager in project flink by apache.

the class LogicalTypeJsonSerdeTest method testIdentifierSerde.

@Test
public void testIdentifierSerde() throws IOException {
    final DataTypeFactoryMock dataTypeFactoryMock = new DataTypeFactoryMock();
    final TableConfig tableConfig = TableConfig.getDefault();
    final Configuration config = tableConfig.getConfiguration();
    final CatalogManager catalogManager = preparedCatalogManager().dataTypeFactory(dataTypeFactoryMock).build();
    final SerdeContext serdeContext = configuredSerdeContext(catalogManager, tableConfig);
    // minimal plan content
    config.set(TableConfigOptions.PLAN_COMPILE_CATALOG_OBJECTS, IDENTIFIER);
    final String minimalJson = toJson(serdeContext, STRUCTURED_TYPE);
    assertThat(minimalJson).isEqualTo("\"`default_catalog`.`default_database`.`MyType`\"");
    // catalog lookup with miss
    config.set(TableConfigOptions.PLAN_RESTORE_CATALOG_OBJECTS, TableConfigOptions.CatalogPlanRestore.IDENTIFIER);
    dataTypeFactoryMock.logicalType = Optional.empty();
    assertThatThrownBy(() -> toObject(serdeContext, minimalJson, LogicalType.class)).satisfies(anyCauseMatches(ValidationException.class, "No type found."));
    // catalog lookup
    config.set(TableConfigOptions.PLAN_RESTORE_CATALOG_OBJECTS, TableConfigOptions.CatalogPlanRestore.IDENTIFIER);
    dataTypeFactoryMock.logicalType = Optional.of(STRUCTURED_TYPE);
    assertThat(toObject(serdeContext, minimalJson, LogicalType.class)).isEqualTo(STRUCTURED_TYPE);
    // maximum plan content
    config.set(TableConfigOptions.PLAN_COMPILE_CATALOG_OBJECTS, ALL);
    final String maximumJson = toJson(serdeContext, STRUCTURED_TYPE);
    final ObjectMapper mapper = new ObjectMapper();
    final JsonNode maximumJsonNode = mapper.readTree(maximumJson);
    assertThat(maximumJsonNode.get(LogicalTypeJsonSerializer.FIELD_NAME_ATTRIBUTES)).isNotNull();
    assertThat(maximumJsonNode.get(LogicalTypeJsonSerializer.FIELD_NAME_DESCRIPTION).asText()).isEqualTo("My original type.");
    // catalog lookup with miss
    config.set(TableConfigOptions.PLAN_RESTORE_CATALOG_OBJECTS, TableConfigOptions.CatalogPlanRestore.IDENTIFIER);
    dataTypeFactoryMock.logicalType = Optional.empty();
    assertThatThrownBy(() -> toObject(serdeContext, maximumJson, LogicalType.class)).satisfies(anyCauseMatches(ValidationException.class, "No type found."));
    // catalog lookup
    config.set(TableConfigOptions.PLAN_RESTORE_CATALOG_OBJECTS, TableConfigOptions.CatalogPlanRestore.IDENTIFIER);
    dataTypeFactoryMock.logicalType = Optional.of(UPDATED_STRUCTURED_TYPE);
    assertThat(toObject(serdeContext, maximumJson, LogicalType.class)).isEqualTo(UPDATED_STRUCTURED_TYPE);
    // no lookup
    config.set(TableConfigOptions.PLAN_RESTORE_CATALOG_OBJECTS, TableConfigOptions.CatalogPlanRestore.ALL);
    dataTypeFactoryMock.logicalType = Optional.of(UPDATED_STRUCTURED_TYPE);
    assertThat(toObject(serdeContext, maximumJson, LogicalType.class)).isEqualTo(STRUCTURED_TYPE);
}

Example 5 with CatalogManager

use of org.apache.flink.table.catalog.CatalogManager in project flink by apache.

the class StreamTableEnvironmentImplTest method getStreamTableEnvironment.

private StreamTableEnvironmentImpl getStreamTableEnvironment(StreamExecutionEnvironment env, DataStreamSource<Integer> elements) {
    TableConfig tableConfig = new TableConfig();
    CatalogManager catalogManager = CatalogManagerMocks.createEmptyCatalogManager();
    ModuleManager moduleManager = new ModuleManager();
    return new StreamTableEnvironmentImpl(catalogManager, moduleManager, new FunctionCatalog(tableConfig, catalogManager, moduleManager), tableConfig, env, new TestPlanner(elements.getTransformation()), new ExecutorMock(), true, this.getClass().getClassLoader());
}