Search in sources :

Example 1 with InternalTypeManager

use of io.prestosql.type.InternalTypeManager in project hetu-core by openlookeng.

the class TestHiveWriterFactory method testSortingPath.

@Test
public void testSortingPath() {
    setUp();
    String targetPath = "/tmp";
    String writePath = "/tmp/table";
    Optional<WriteIdInfo> writeIdInfo = Optional.of(new WriteIdInfo(1, 1, 0));
    StorageFormat storageFormat = StorageFormat.fromHiveStorageFormat(ORC);
    Storage storage = new Storage(storageFormat, "", Optional.empty(), false, ImmutableMap.of());
    Table table = new Table("schema", "table", "user", "MANAGED_TABLE", storage, ImmutableList.of(new Column("col_1", HiveType.HIVE_INT, Optional.empty())), ImmutableList.of(), ImmutableMap.of("transactional", "true"), Optional.of("original"), Optional.of("expanded"));
    HiveConfig hiveConfig = getHiveConfig();
    HivePageSinkMetadata hivePageSinkMetadata = new HivePageSinkMetadata(new SchemaTableName("schema", "table"), Optional.of(table), ImmutableMap.of());
    PageSorter pageSorter = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false));
    Metadata metadata = createTestMetadataManager();
    TypeManager typeManager = new InternalTypeManager(metadata.getFunctionAndTypeManager());
    HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveConfig), ImmutableSet.of());
    HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveConfig, new NoHdfsAuthentication());
    LocationService locationService = new HiveLocationService(hdfsEnvironment);
    ConnectorSession session = newSession();
    HiveWriterFactory hiveWriterFactory = new HiveWriterFactory(getDefaultHiveFileWriterFactories(hiveConfig), "schema", "table", false, HiveACIDWriteType.DELETE, ImmutableList.of(new HiveColumnHandle("col_1", HiveType.HIVE_INT, new TypeSignature("integer", ImmutableList.of()), 0, HiveColumnHandle.ColumnType.REGULAR, Optional.empty())), ORC, ORC, ImmutableMap.of(), OptionalInt.empty(), ImmutableList.of(), new LocationHandle(targetPath, writePath, false, LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY, writeIdInfo), locationService, session.getQueryId(), new HivePageSinkMetadataProvider(hivePageSinkMetadata, CachingHiveMetastore.memoizeMetastore(metastore, 1000), new HiveIdentity(session)), typeManager, hdfsEnvironment, pageSorter, hiveConfig.getWriterSortBufferSize(), hiveConfig.getMaxOpenSortFiles(), false, UTC, session, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(hiveConfig, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(hiveConfig));
    HiveWriter hiveWriter = hiveWriterFactory.createWriter(ImmutableList.of(), OptionalInt.empty(), Optional.empty());
    assertEquals(((SortingFileWriter) hiveWriter.getFileWriter()).getTempFilePrefix().getName(), ".tmp-sort.bucket_00000");
}
Also used : HivePageSinkMetadataProvider(io.prestosql.plugin.hive.metastore.HivePageSinkMetadataProvider) HivePageSinkMetadata(io.prestosql.plugin.hive.metastore.HivePageSinkMetadata) Metadata(io.prestosql.metadata.Metadata) StorageFormat(io.prestosql.plugin.hive.metastore.StorageFormat) PagesIndex(io.prestosql.operator.PagesIndex) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) HiveIdentity(io.prestosql.plugin.hive.authentication.HiveIdentity) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) TypeSignature(io.prestosql.spi.type.TypeSignature) Column(io.prestosql.plugin.hive.metastore.Column) TestingNodeManager(io.prestosql.testing.TestingNodeManager) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) PageSorter(io.prestosql.spi.PageSorter) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) InternalTypeManager(io.prestosql.type.InternalTypeManager) Table(io.prestosql.plugin.hive.metastore.Table) HivePageSinkMetadata(io.prestosql.plugin.hive.metastore.HivePageSinkMetadata) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) Storage(io.prestosql.plugin.hive.metastore.Storage) InternalTypeManager(io.prestosql.type.InternalTypeManager) TypeManager(io.prestosql.spi.type.TypeManager) Test(org.testng.annotations.Test)

Example 2 with InternalTypeManager

use of io.prestosql.type.InternalTypeManager in project hetu-core by openlookeng.

the class ConnectorManager method createConnector.

public Connector createConnector(CatalogName catalogName, ConnectorFactory factory, Map<String, String> properties) {
    ConnectorContext context = new ConnectorContextInstance(new ConnectorAwareNodeManager(nodeManager, nodeInfo.getEnvironment(), catalogName), versionEmbedder, new InternalTypeManager(metadataManager.getFunctionAndTypeManager()), pageSorter, pageIndexerFactory, hetuMetaStoreManager.getHetuMetastore(), heuristicIndexerManager.getIndexClient(), new ConnectorRowExpressionService(domainTranslator, determinismEvaluator), metadataManager.getFunctionAndTypeManager(), new FunctionResolution(metadataManager.getFunctionAndTypeManager()), metadataManager.getFunctionAndTypeManager().getBlockEncodingSerde(), new ConnectorFilterStatsCalculatorService(filterStatsCalculator));
    try (ThreadContextClassLoader ignored = new ThreadContextClassLoader(factory.getClass().getClassLoader())) {
        Connector connector = factory.create(catalogName.getCatalogName(), properties, context);
        registerExternalFunctions(connector.getExternalFunctionHub(), metadataManager.getFunctionAndTypeManager());
        return connector;
    }
}
Also used : ConnectorFilterStatsCalculatorService(io.prestosql.cost.ConnectorFilterStatsCalculatorService) Connector(io.prestosql.spi.connector.Connector) InformationSchemaConnector(io.prestosql.connector.informationschema.InformationSchemaConnector) SystemConnector(io.prestosql.connector.system.SystemConnector) ConnectorRowExpressionService(io.prestosql.sql.relational.ConnectorRowExpressionService) ConnectorContext(io.prestosql.spi.connector.ConnectorContext) InternalTypeManager(io.prestosql.type.InternalTypeManager) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) ThreadContextClassLoader(io.prestosql.spi.classloader.ThreadContextClassLoader)

Example 3 with InternalTypeManager

use of io.prestosql.type.InternalTypeManager in project boostkit-bigdata by kunpengcompute.

the class TestHiveWriterFactory method testSortingPath.

@Test
public void testSortingPath() {
    setUp();
    String targetPath = "/tmp";
    String writePath = "/tmp/table";
    Optional<WriteIdInfo> writeIdInfo = Optional.of(new WriteIdInfo(1, 1, 0));
    StorageFormat storageFormat = StorageFormat.fromHiveStorageFormat(ORC);
    Storage storage = new Storage(storageFormat, "", Optional.empty(), false, ImmutableMap.of());
    Table table = new Table("schema", "table", "user", "MANAGED_TABLE", storage, ImmutableList.of(new Column("col_1", HiveType.HIVE_INT, Optional.empty())), ImmutableList.of(), ImmutableMap.of("transactional", "true"), Optional.of("original"), Optional.of("expanded"));
    HiveConfig hiveConfig = getHiveConfig();
    HivePageSinkMetadata hivePageSinkMetadata = new HivePageSinkMetadata(new SchemaTableName("schema", "table"), Optional.of(table), ImmutableMap.of());
    PageSorter pageSorter = new PagesIndexPageSorter(new PagesIndex.TestingFactory(false));
    Metadata metadata = createTestMetadataManager();
    TypeManager typeManager = new InternalTypeManager(metadata.getFunctionAndTypeManager());
    HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveConfig), ImmutableSet.of());
    HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveConfig, new NoHdfsAuthentication());
    LocationService locationService = new HiveLocationService(hdfsEnvironment);
    ConnectorSession session = newSession();
    HiveWriterFactory hiveWriterFactory = new HiveWriterFactory(getDefaultHiveFileWriterFactories(hiveConfig), "schema", "table", false, HiveACIDWriteType.DELETE, ImmutableList.of(new HiveColumnHandle("col_1", HiveType.HIVE_INT, new TypeSignature("integer", ImmutableList.of()), 0, HiveColumnHandle.ColumnType.REGULAR, Optional.empty())), ORC, ORC, ImmutableMap.of(), OptionalInt.empty(), ImmutableList.of(), new LocationHandle(targetPath, writePath, false, LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY, writeIdInfo), locationService, session.getQueryId(), new HivePageSinkMetadataProvider(hivePageSinkMetadata, CachingHiveMetastore.memoizeMetastore(metastore, 1000), new HiveIdentity(session)), typeManager, hdfsEnvironment, pageSorter, hiveConfig.getWriterSortBufferSize(), hiveConfig.getMaxOpenSortFiles(), false, UTC, session, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(hiveConfig, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(hiveConfig));
    HiveWriter hiveWriter = hiveWriterFactory.createWriter(ImmutableList.of(), OptionalInt.empty(), Optional.empty());
    assertEquals(((SortingFileWriter) hiveWriter.getFileWriter()).getTempFilePrefix().getName(), ".tmp-sort.bucket_00000");
}
Also used : HivePageSinkMetadataProvider(io.prestosql.plugin.hive.metastore.HivePageSinkMetadataProvider) HivePageSinkMetadata(io.prestosql.plugin.hive.metastore.HivePageSinkMetadata) Metadata(io.prestosql.metadata.Metadata) StorageFormat(io.prestosql.plugin.hive.metastore.StorageFormat) PagesIndex(io.prestosql.operator.PagesIndex) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) HiveIdentity(io.prestosql.plugin.hive.authentication.HiveIdentity) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) TypeSignature(io.prestosql.spi.type.TypeSignature) Column(io.prestosql.plugin.hive.metastore.Column) TestingNodeManager(io.prestosql.testing.TestingNodeManager) PagesIndexPageSorter(io.prestosql.PagesIndexPageSorter) PageSorter(io.prestosql.spi.PageSorter) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) InternalTypeManager(io.prestosql.type.InternalTypeManager) Table(io.prestosql.plugin.hive.metastore.Table) HivePageSinkMetadata(io.prestosql.plugin.hive.metastore.HivePageSinkMetadata) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) Storage(io.prestosql.plugin.hive.metastore.Storage) InternalTypeManager(io.prestosql.type.InternalTypeManager) TypeManager(io.prestosql.spi.type.TypeManager) Test(org.testng.annotations.Test)

Aggregations

InternalTypeManager (io.prestosql.type.InternalTypeManager)3 PagesIndexPageSorter (io.prestosql.PagesIndexPageSorter)2 Metadata (io.prestosql.metadata.Metadata)2 PagesIndex (io.prestosql.operator.PagesIndex)2 HiveIdentity (io.prestosql.plugin.hive.authentication.HiveIdentity)2 NoHdfsAuthentication (io.prestosql.plugin.hive.authentication.NoHdfsAuthentication)2 Column (io.prestosql.plugin.hive.metastore.Column)2 HivePageSinkMetadata (io.prestosql.plugin.hive.metastore.HivePageSinkMetadata)2 HivePageSinkMetadataProvider (io.prestosql.plugin.hive.metastore.HivePageSinkMetadataProvider)2 Storage (io.prestosql.plugin.hive.metastore.Storage)2 StorageFormat (io.prestosql.plugin.hive.metastore.StorageFormat)2 Table (io.prestosql.plugin.hive.metastore.Table)2 PageSorter (io.prestosql.spi.PageSorter)2 ConnectorSession (io.prestosql.spi.connector.ConnectorSession)2 SchemaTableName (io.prestosql.spi.connector.SchemaTableName)2 TypeManager (io.prestosql.spi.type.TypeManager)2 TypeSignature (io.prestosql.spi.type.TypeSignature)2 TestingConnectorSession (io.prestosql.testing.TestingConnectorSession)2 TestingNodeManager (io.prestosql.testing.TestingNodeManager)2 Test (org.testng.annotations.Test)2