Search in sources :

Example 1 with HivePageSourceFactory

use of io.prestosql.plugin.hive.HivePageSourceFactory in project carbondata by apache.

the class CarbondataModule method configure.

@Override
public void configure(Binder binder) {
    binder.install(new HiveHdfsModule());
    binder.bind(TypeTranslator.class).toInstance(new HiveTypeTranslator());
    binder.bind(CoercionPolicy.class).to(HiveCoercionPolicy.class).in(Scopes.SINGLETON);
    binder.bind(DirectoryLister.class).to(CachingDirectoryLister.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(HiveConfig.class);
    binder.bind(HiveSessionProperties.class).in(Scopes.SINGLETON);
    binder.bind(HiveTableProperties.class).in(Scopes.SINGLETON);
    binder.bind(HiveAnalyzeProperties.class).in(Scopes.SINGLETON);
    binder.bind(PrestoS3ClientFactory.class).in(Scopes.SINGLETON);
    binder.bind(CachingDirectoryLister.class).in(Scopes.SINGLETON);
    newExporter(binder).export(CachingDirectoryLister.class).withGeneratedName();
    binder.bind(HiveWriterStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(HiveWriterStats.class).withGeneratedName();
    newSetBinder(binder, EventClient.class).addBinding().to(HiveEventClient.class).in(Scopes.SINGLETON);
    binder.bind(HivePartitionManager.class).in(Scopes.SINGLETON);
    binder.bind(LocationService.class).to(HiveLocationService.class).in(Scopes.SINGLETON);
    binder.bind(HiveLocationService.class).to(CarbonDataLocationService.class).in(Scopes.SINGLETON);
    binder.bind(HiveMetadataFactory.class).to(CarbonMetadataFactory.class).in(Scopes.SINGLETON);
    binder.bind(TransactionalMetadataFactory.class).to(HiveMetadataFactory.class).in(Scopes.SINGLETON);
    binder.bind(HiveTransactionManager.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorSplitManager.class).to(CarbondataSplitManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ConnectorSplitManager.class).as(generator -> generator.generatedNameOf(CarbondataSplitManager.class));
    binder.bind(ConnectorPageSourceProvider.class).to(CarbondataPageSourceProvider.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorPageSinkProvider.class).to(CarbonDataPageSinkProvider.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorNodePartitioningProvider.class).to(HiveNodePartitioningProvider.class).in(Scopes.SINGLETON);
    jsonCodecBinder(binder).bindJsonCodec(PartitionUpdate.class);
    binder.bind(FileFormatDataSourceStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(FileFormatDataSourceStats.class).withGeneratedName();
    Multibinder<HivePageSourceFactory> pageSourceFactoryBinder = newSetBinder(binder, HivePageSourceFactory.class);
    pageSourceFactoryBinder.addBinding().to(OrcPageSourceFactory.class).in(Scopes.SINGLETON);
    pageSourceFactoryBinder.addBinding().to(ParquetPageSourceFactory.class).in(Scopes.SINGLETON);
    pageSourceFactoryBinder.addBinding().to(RcFilePageSourceFactory.class).in(Scopes.SINGLETON);
    Multibinder<HiveRecordCursorProvider> recordCursorProviderBinder = newSetBinder(binder, HiveRecordCursorProvider.class);
    recordCursorProviderBinder.addBinding().to(S3SelectRecordCursorProvider.class).in(Scopes.SINGLETON);
    binder.bind(GenericHiveRecordCursorProvider.class).in(Scopes.SINGLETON);
    Multibinder<HiveFileWriterFactory> fileWriterFactoryBinder = newSetBinder(binder, HiveFileWriterFactory.class);
    binder.bind(OrcFileWriterFactory.class).in(Scopes.SINGLETON);
    newExporter(binder).export(OrcFileWriterFactory.class).withGeneratedName();
    configBinder(binder).bindConfig(OrcReaderConfig.class);
    configBinder(binder).bindConfig(OrcWriterConfig.class);
    fileWriterFactoryBinder.addBinding().to(OrcFileWriterFactory.class).in(Scopes.SINGLETON);
    fileWriterFactoryBinder.addBinding().to(RcFileFileWriterFactory.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(ParquetReaderConfig.class);
    configBinder(binder).bindConfig(ParquetWriterConfig.class);
    fileWriterFactoryBinder.addBinding().to(CarbonDataFileWriterFactory.class).in(Scopes.SINGLETON);
    binder.bind(CarbonTableReader.class).in(Scopes.SINGLETON);
    jsonBinder(binder).addDeserializerBinding(Type.class).to(TypeDeserializer.class);
    newSetBinder(binder, SystemTable.class);
    // configure carbon properties
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.IS_QUERY_FROM_PRESTO, "true");
}
Also used : TypeTranslator(io.prestosql.plugin.hive.TypeTranslator) HiveTypeTranslator(io.prestosql.plugin.hive.HiveTypeTranslator) HiveMetadataFactory(io.prestosql.plugin.hive.HiveMetadataFactory) HivePartitionManager(io.prestosql.plugin.hive.HivePartitionManager) HiveCoercionPolicy(io.prestosql.plugin.hive.HiveCoercionPolicy) S3SelectRecordCursorProvider(io.prestosql.plugin.hive.s3select.S3SelectRecordCursorProvider) OrcPageSourceFactory(io.prestosql.plugin.hive.orc.OrcPageSourceFactory) HiveSessionProperties(io.prestosql.plugin.hive.HiveSessionProperties) HiveLocationService(io.prestosql.plugin.hive.HiveLocationService) HiveAnalyzeProperties(io.prestosql.plugin.hive.HiveAnalyzeProperties) ParquetPageSourceFactory(io.prestosql.plugin.hive.parquet.ParquetPageSourceFactory) CachingDirectoryLister(io.prestosql.plugin.hive.CachingDirectoryLister) HiveTableProperties(io.prestosql.plugin.hive.HiveTableProperties) HiveEventClient(io.prestosql.plugin.hive.HiveEventClient) HiveWriterStats(io.prestosql.plugin.hive.HiveWriterStats) FileFormatDataSourceStats(io.prestosql.plugin.hive.FileFormatDataSourceStats) GenericHiveRecordCursorProvider(io.prestosql.plugin.hive.GenericHiveRecordCursorProvider) HiveRecordCursorProvider(io.prestosql.plugin.hive.HiveRecordCursorProvider) EventClient(io.airlift.event.client.EventClient) HiveEventClient(io.prestosql.plugin.hive.HiveEventClient) OrcFileWriterFactory(io.prestosql.plugin.hive.orc.OrcFileWriterFactory) CarbonTableReader(org.apache.carbondata.presto.impl.CarbonTableReader) Type(io.prestosql.spi.type.Type) HiveTypeTranslator(io.prestosql.plugin.hive.HiveTypeTranslator) PrestoS3ClientFactory(io.prestosql.plugin.hive.s3select.PrestoS3ClientFactory) ConnectorSplitManager(io.prestosql.spi.connector.ConnectorSplitManager) GenericHiveRecordCursorProvider(io.prestosql.plugin.hive.GenericHiveRecordCursorProvider) HiveHdfsModule(io.prestosql.plugin.hive.HiveHdfsModule) RcFileFileWriterFactory(io.prestosql.plugin.hive.RcFileFileWriterFactory) HiveTransactionManager(io.prestosql.plugin.hive.HiveTransactionManager) HiveNodePartitioningProvider(io.prestosql.plugin.hive.HiveNodePartitioningProvider) HivePageSourceFactory(io.prestosql.plugin.hive.HivePageSourceFactory) HiveFileWriterFactory(io.prestosql.plugin.hive.HiveFileWriterFactory) RcFilePageSourceFactory(io.prestosql.plugin.hive.rcfile.RcFilePageSourceFactory)

Example 2 with HivePageSourceFactory

use of io.prestosql.plugin.hive.HivePageSourceFactory in project hetu-core by openlookeng.

the class CarbondataModule method configure.

@Override
public void configure(Binder binder) {
    binder.bind(TypeTranslator.class).toInstance(new HiveTypeTranslator());
    binder.bind(CoercionPolicy.class).to(HiveCoercionPolicy.class).in(Scopes.SINGLETON);
    binder.bind(HdfsConfigurationInitializer.class).in(Scopes.SINGLETON);
    newSetBinder(binder, DynamicConfigurationProvider.class);
    binder.bind(HdfsConfiguration.class).to(HiveHdfsConfiguration.class).in(Scopes.SINGLETON);
    binder.bind(HdfsEnvironment.class).in(Scopes.SINGLETON);
    binder.bind(DirectoryLister.class).to(CachingDirectoryLister.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(CarbondataConfig.class);
    binder.bind(HiveSessionProperties.class).in(Scopes.SINGLETON);
    binder.bind(CarbondataTableProperties.class).in(Scopes.SINGLETON);
    binder.bind(HiveAnalyzeProperties.class).in(Scopes.SINGLETON);
    binder.bind(NamenodeStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(NamenodeStats.class).as(generatedNameOf(NamenodeStats.class, connectorId));
    Multibinder<HiveRecordCursorProvider> recordCursorProviderBinder = newSetBinder(binder, HiveRecordCursorProvider.class);
    recordCursorProviderBinder.addBinding().to(GenericHiveRecordCursorProvider.class).in(Scopes.SINGLETON);
    binder.bind(HiveWriterStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(HiveWriterStats.class).as(generatedNameOf(HiveWriterStats.class, connectorId));
    newSetBinder(binder, EventClient.class).addBinding().to(HiveEventClient.class).in(Scopes.SINGLETON);
    binder.bind(HivePartitionManager.class).in(Scopes.SINGLETON);
    binder.bind(LocationService.class).to(CarbondataLocationService.class).in(Scopes.SINGLETON);
    binder.bind(HiveMetadataFactory.class).to(CarbondataMetadataFactory.class).in(Scopes.SINGLETON);
    binder.bind(new TypeLiteral<Supplier<TransactionalMetadata>>() {
    }).to(HiveMetadataFactory.class).in(Scopes.SINGLETON);
    binder.bind(HiveTransactionManager.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorSplitManager.class).to(CarbondataSplitManager.class).in(Scopes.SINGLETON);
    newExporter(binder).export(ConnectorSplitManager.class).as(generatedNameOf(HiveSplitManager.class, connectorId));
    binder.bind(ConnectorPageSourceProvider.class).to(CarbondataPageSourceProvider.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorPageSinkProvider.class).to(CarbondataPageSinkProvider.class).in(Scopes.SINGLETON);
    binder.bind(ConnectorNodePartitioningProvider.class).to(HiveNodePartitioningProvider.class).in(Scopes.SINGLETON);
    jsonCodecBinder(binder).bindJsonCodec(PartitionUpdate.class);
    jsonCodecBinder(binder).bindJsonCodec(CarbondataSegmentInfoUtil.class);
    binder.bind(FileFormatDataSourceStats.class).in(Scopes.SINGLETON);
    newExporter(binder).export(FileFormatDataSourceStats.class).as(generatedNameOf(FileFormatDataSourceStats.class, connectorId));
    Multibinder<HivePageSourceFactory> pageSourceFactoryBinder = newSetBinder(binder, HivePageSourceFactory.class);
    pageSourceFactoryBinder.addBinding().to(OrcPageSourceFactory.class).in(Scopes.SINGLETON);
    pageSourceFactoryBinder.addBinding().to(ParquetPageSourceFactory.class).in(Scopes.SINGLETON);
    pageSourceFactoryBinder.addBinding().to(RcFilePageSourceFactory.class).in(Scopes.SINGLETON);
    Multibinder<HiveFileWriterFactory> fileWriterFactoryBinder = newSetBinder(binder, HiveFileWriterFactory.class);
    binder.bind(OrcFileWriterFactory.class).in(Scopes.SINGLETON);
    newExporter(binder).export(OrcFileWriterFactory.class).as(generatedNameOf(OrcFileWriterFactory.class, connectorId));
    configBinder(binder).bindConfig(OrcFileWriterConfig.class);
    fileWriterFactoryBinder.addBinding().to(OrcFileWriterFactory.class).in(Scopes.SINGLETON);
    fileWriterFactoryBinder.addBinding().to(RcFileFileWriterFactory.class).in(Scopes.SINGLETON);
    fileWriterFactoryBinder.addBinding().to(CarbondataFileWriterFactory.class).in(Scopes.SINGLETON);
    binder.bind(CarbondataTableReader.class).in(Scopes.SINGLETON);
    configBinder(binder).bindConfig(ParquetFileWriterConfig.class);
}
Also used : HdfsConfigurationInitializer(io.prestosql.plugin.hive.HdfsConfigurationInitializer) TypeTranslator(io.prestosql.plugin.hive.TypeTranslator) HiveTypeTranslator(io.prestosql.plugin.hive.HiveTypeTranslator) HiveMetadataFactory(io.prestosql.plugin.hive.HiveMetadataFactory) HivePartitionManager(io.prestosql.plugin.hive.HivePartitionManager) HiveCoercionPolicy(io.prestosql.plugin.hive.HiveCoercionPolicy) OrcPageSourceFactory(io.prestosql.plugin.hive.orc.OrcPageSourceFactory) HiveSplitManager(io.prestosql.plugin.hive.HiveSplitManager) HiveSessionProperties(io.prestosql.plugin.hive.HiveSessionProperties) HdfsEnvironment(io.prestosql.plugin.hive.HdfsEnvironment) Supplier(java.util.function.Supplier) NamenodeStats(io.prestosql.plugin.hive.NamenodeStats) HiveAnalyzeProperties(io.prestosql.plugin.hive.HiveAnalyzeProperties) CarbondataTableReader(io.hetu.core.plugin.carbondata.impl.CarbondataTableReader) ParquetPageSourceFactory(io.prestosql.plugin.hive.parquet.ParquetPageSourceFactory) HiveHdfsConfiguration(io.prestosql.plugin.hive.HiveHdfsConfiguration) CachingDirectoryLister(io.prestosql.plugin.hive.CachingDirectoryLister) HiveEventClient(io.prestosql.plugin.hive.HiveEventClient) HiveWriterStats(io.prestosql.plugin.hive.HiveWriterStats) FileFormatDataSourceStats(io.prestosql.plugin.hive.FileFormatDataSourceStats) GenericHiveRecordCursorProvider(io.prestosql.plugin.hive.GenericHiveRecordCursorProvider) HiveRecordCursorProvider(io.prestosql.plugin.hive.HiveRecordCursorProvider) EventClient(io.airlift.event.client.EventClient) HiveEventClient(io.prestosql.plugin.hive.HiveEventClient) OrcFileWriterFactory(io.prestosql.plugin.hive.OrcFileWriterFactory) HiveTypeTranslator(io.prestosql.plugin.hive.HiveTypeTranslator) GenericHiveRecordCursorProvider(io.prestosql.plugin.hive.GenericHiveRecordCursorProvider) ConnectorSplitManager(io.prestosql.spi.connector.ConnectorSplitManager) RcFileFileWriterFactory(io.prestosql.plugin.hive.RcFileFileWriterFactory) HiveTransactionManager(io.prestosql.plugin.hive.HiveTransactionManager) HiveNodePartitioningProvider(io.prestosql.plugin.hive.HiveNodePartitioningProvider) HivePageSourceFactory(io.prestosql.plugin.hive.HivePageSourceFactory) HiveFileWriterFactory(io.prestosql.plugin.hive.HiveFileWriterFactory) RcFilePageSourceFactory(io.prestosql.plugin.hive.rcfile.RcFilePageSourceFactory)

Aggregations

EventClient (io.airlift.event.client.EventClient)2 CachingDirectoryLister (io.prestosql.plugin.hive.CachingDirectoryLister)2 FileFormatDataSourceStats (io.prestosql.plugin.hive.FileFormatDataSourceStats)2 GenericHiveRecordCursorProvider (io.prestosql.plugin.hive.GenericHiveRecordCursorProvider)2 HiveAnalyzeProperties (io.prestosql.plugin.hive.HiveAnalyzeProperties)2 HiveCoercionPolicy (io.prestosql.plugin.hive.HiveCoercionPolicy)2 HiveEventClient (io.prestosql.plugin.hive.HiveEventClient)2 HiveFileWriterFactory (io.prestosql.plugin.hive.HiveFileWriterFactory)2 HiveMetadataFactory (io.prestosql.plugin.hive.HiveMetadataFactory)2 HiveNodePartitioningProvider (io.prestosql.plugin.hive.HiveNodePartitioningProvider)2 HivePageSourceFactory (io.prestosql.plugin.hive.HivePageSourceFactory)2 HivePartitionManager (io.prestosql.plugin.hive.HivePartitionManager)2 HiveRecordCursorProvider (io.prestosql.plugin.hive.HiveRecordCursorProvider)2 HiveSessionProperties (io.prestosql.plugin.hive.HiveSessionProperties)2 HiveTransactionManager (io.prestosql.plugin.hive.HiveTransactionManager)2 HiveTypeTranslator (io.prestosql.plugin.hive.HiveTypeTranslator)2 HiveWriterStats (io.prestosql.plugin.hive.HiveWriterStats)2 RcFileFileWriterFactory (io.prestosql.plugin.hive.RcFileFileWriterFactory)2 TypeTranslator (io.prestosql.plugin.hive.TypeTranslator)2 OrcPageSourceFactory (io.prestosql.plugin.hive.orc.OrcPageSourceFactory)2