Search in sources :

Example 1 with METASTORE_CONTEXT

use of com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT in project presto by prestodb.

the class AbstractTestHiveClient method createDummyPartitionedTable.

protected void createDummyPartitionedTable(SchemaTableName tableName, List<ColumnMetadata> columns) throws Exception {
    doCreateEmptyTable(tableName, ORC, columns);
    ExtendedHiveMetastore metastoreClient = getMetastoreClient();
    Table table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    List<String> firstPartitionValues = ImmutableList.of("2016-01-01");
    List<String> secondPartitionValues = ImmutableList.of("2016-01-02");
    String firstPartitionName = makePartName(ImmutableList.of("ds"), firstPartitionValues);
    String secondPartitionName = makePartName(ImmutableList.of("ds"), secondPartitionValues);
    List<PartitionWithStatistics> partitions = ImmutableList.of(firstPartitionName, secondPartitionName).stream().map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName), partitionName, PartitionStatistics.empty())).collect(toImmutableList());
    metastoreClient.addPartitions(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), partitions);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), firstPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), secondPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) SkipException(org.testng.SkipException) WarningCollector(com.facebook.presto.spi.WarningCollector) CharType.createCharType(com.facebook.presto.common.type.CharType.createCharType) DateTimeZone(org.joda.time.DateTimeZone) SORTED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) FileSystem(org.apache.hadoop.fs.FileSystem) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) CounterStat(com.facebook.airlift.stats.CounterStat) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) FileStatus(org.apache.hadoop.fs.FileStatus) MoreFutures.getFutureValue(com.facebook.airlift.concurrent.MoreFutures.getFutureValue) MAX_PARTITION_KEY_COLUMN_INDEX(com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX) NOT_PARTITIONED(com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) Slices(io.airlift.slice.Slices) Map(java.util.Map) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) NullableValue(com.facebook.presto.common.predicate.NullableValue) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) SqlFunctionProperties(com.facebook.presto.common.function.SqlFunctionProperties) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) TABLE(com.facebook.presto.hive.CacheQuotaScope.TABLE) ZoneId(java.time.ZoneId) HiveColumnHandle.bucketColumnHandle(com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) SqlTimestamp(com.facebook.presto.common.type.SqlTimestamp) Predicate(com.google.common.base.Predicate) HiveColumnStatistics.createBinaryColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) StandardTypes(com.facebook.presto.common.type.StandardTypes) Table(com.facebook.presto.hive.metastore.Table) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveTestUtils.arrayType(com.facebook.presto.hive.HiveTestUtils.arrayType) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) REAL(com.facebook.presto.common.type.RealType.REAL) PAGEFILE(com.facebook.presto.hive.HiveStorageFormat.PAGEFILE) OptionalLong(java.util.OptionalLong) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) GIGABYTE(io.airlift.units.DataSize.Unit.GIGABYTE) HIVE_BYTE(com.facebook.presto.hive.HiveType.HIVE_BYTE) SqlVarbinary(com.facebook.presto.common.type.SqlVarbinary) Assertions.assertInstanceOf(com.facebook.airlift.testing.Assertions.assertInstanceOf) CSV(com.facebook.presto.hive.HiveStorageFormat.CSV) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) MetastoreCacheScope(com.facebook.presto.hive.metastore.CachingHiveMetastore.MetastoreCacheScope) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) AfterClass(org.testng.annotations.AfterClass) FileUtils.makePartName(org.apache.hadoop.hive.common.FileUtils.makePartName) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) HostAndPort(com.google.common.net.HostAndPort) Domain(com.facebook.presto.common.predicate.Domain) FUNCTION_AND_TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.FUNCTION_AND_TYPE_MANAGER) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) SqlFunctionId(com.facebook.presto.spi.function.SqlFunctionId) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) SplitSchedulingContext(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingContext) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) HiveColumnStatistics.createDecimalColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) RowType(com.facebook.presto.common.type.RowType) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveFilterPushdown.pushdownFilter(com.facebook.presto.hive.rule.HiveFilterPushdown.pushdownFilter) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) HIVE_FLOAT(com.facebook.presto.hive.HiveType.HIVE_FLOAT) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) TypeProvider(com.facebook.presto.sql.planner.TypeProvider) Locale(java.util.Locale) UNGROUPED_SCHEDULING(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) Thread.sleep(java.lang.Thread.sleep) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) PageFilePageSource(com.facebook.presto.hive.pagefile.PageFilePageSource) ImmutableSet(com.google.common.collect.ImmutableSet) OFFLINE_DATA_DEBUG_MODE_ENABLED(com.facebook.presto.hive.HiveSessionProperties.OFFLINE_DATA_DEBUG_MODE_ENABLED) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) Range(com.facebook.presto.common.predicate.Range) Objects(java.util.Objects) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) MetastoreUtil.toPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues) HiveColumnStatistics.createBooleanColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics) METADATA(com.facebook.presto.hive.HiveTestUtils.METADATA) HiveTestUtils.getDefaultHiveBatchPageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories) IntStream(java.util.stream.IntStream) MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) MapType(com.facebook.presto.common.type.MapType) FILTER_STATS_CALCULATOR_SERVICE(com.facebook.presto.hive.HiveTestUtils.FILTER_STATS_CALCULATOR_SERVICE) Assert.assertNull(org.testng.Assert.assertNull) HiveColumnStatistics.createIntegerColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) OptionalDouble(java.util.OptionalDouble) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) OptionalInt(java.util.OptionalInt) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) HIVE_DOUBLE(com.facebook.presto.hive.HiveType.HIVE_DOUBLE) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) HiveTestUtils.mapType(com.facebook.presto.hive.HiveTestUtils.mapType) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) Type(com.facebook.presto.common.type.Type) ExecutorService(java.util.concurrent.ExecutorService) NamedTypeSignature(com.facebook.presto.common.type.NamedTypeSignature) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) DEFAULT_COLUMN_CONVERTER_PROVIDER(com.facebook.presto.hive.HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER) USER(com.facebook.presto.spi.security.PrincipalType.USER) HiveTestUtils.getDefaultHiveSelectivePageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories) HiveColumnStatistics.createStringColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createStringColumnStatistics) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) PageSinkContext(com.facebook.presto.spi.PageSinkContext) OutputStreamDataSinkFactory(com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory) MetastoreUtil.createDirectory(com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory) TRANSACTION_CONFLICT(com.facebook.presto.spi.StandardErrorCode.TRANSACTION_CONFLICT) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) Hashing.sha256(com.google.common.hash.Hashing.sha256) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPartitioningMetadata(com.facebook.presto.spi.connector.ConnectorPartitioningMetadata) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Assert.assertTrue(org.testng.Assert.assertTrue) PlanBuilder.expression(com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) DecimalType.createDecimalType(com.facebook.presto.common.type.DecimalType.createDecimalType) HIVE_COMPATIBLE(com.facebook.presto.hive.BucketFunctionType.HIVE_COMPATIBLE) Page(com.facebook.presto.common.Page) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ColumnStatistics(com.facebook.presto.spi.statistics.ColumnStatistics) HIVE_SHORT(com.facebook.presto.hive.HiveType.HIVE_SHORT) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) SqlInvokedFunction(com.facebook.presto.spi.function.SqlInvokedFunction) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) HiveColumnStatistics.createDoubleColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics) BigDecimal(java.math.BigDecimal) TupleDomain.withColumnDomains(com.facebook.presto.common.predicate.TupleDomain.withColumnDomains) Sets.difference(com.google.common.collect.Sets.difference) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) PRESTO_VERSION_NAME(com.facebook.presto.hive.HiveMetadata.PRESTO_VERSION_NAME) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TestingRowExpressionTranslator(com.facebook.presto.sql.TestingRowExpressionTranslator) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Executors(java.util.concurrent.Executors) ConnectorSession(com.facebook.presto.spi.ConnectorSession) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) Assertions.assertGreaterThan(com.facebook.airlift.testing.Assertions.assertGreaterThan) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) OrcSelectivePageSource(com.facebook.presto.hive.orc.OrcSelectivePageSource) STAGE_AND_MOVE_TO_TARGET_DIRECTORY(com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY) VariablesExtractor.extractUnique(com.facebook.presto.sql.planner.VariablesExtractor.extractUnique) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Slice(io.airlift.slice.Slice) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) TINYINT(com.facebook.presto.common.type.TinyintType.TINYINT) Assertions.assertGreaterThanOrEqual(com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) WriteInfo(com.facebook.presto.hive.LocationService.WriteInfo) HYPER_LOG_LOG(com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG) HiveBasicStatistics.createEmptyStatistics(com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics) DATE(com.facebook.presto.common.type.DateType.DATE) FUNCTION_RESOLUTION(com.facebook.presto.hive.HiveTestUtils.FUNCTION_RESOLUTION) ArrayList(java.util.ArrayList) ROW_EXPRESSION_SERVICE(com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) SqlDate(com.facebook.presto.common.type.SqlDate) NON_CACHEABLE(com.facebook.presto.spi.SplitContext.NON_CACHEABLE) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) TableHandle(com.facebook.presto.spi.TableHandle) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) HiveTestUtils.rowType(com.facebook.presto.hive.HiveTestUtils.rowType) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) LongStream(java.util.stream.LongStream) Executor(java.util.concurrent.Executor) Constraint(com.facebook.presto.spi.Constraint) UTC_KEY(com.facebook.presto.common.type.TimeZoneKey.UTC_KEY) UTC(org.joda.time.DateTimeZone.UTC) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SMALLINT(com.facebook.presto.common.type.SmallintType.SMALLINT) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Assertions.assertLessThanOrEqual(com.facebook.airlift.testing.Assertions.assertLessThanOrEqual) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) HIVE_BOOLEAN(com.facebook.presto.hive.HiveType.HIVE_BOOLEAN) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) HiveColumnStatistics.createDateColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics) ValueSet(com.facebook.presto.common.predicate.ValueSet) SORTED_WRITE_TO_TEMP_PATH_ENABLED(com.facebook.presto.hive.HiveSessionProperties.SORTED_WRITE_TO_TEMP_PATH_ENABLED) MetadataManager(com.facebook.presto.metadata.MetadataManager) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) TableStatistics(com.facebook.presto.spi.statistics.TableStatistics) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) CacheConfig(com.facebook.presto.cache.CacheConfig) HiveTestUtils.getDefaultOrcFileWriterFactory(com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) MoreCollectors.onlyElement(com.google.common.collect.MoreCollectors.onlyElement) OrcBatchPageSource(com.facebook.presto.hive.orc.OrcBatchPageSource) Iterables.concat(com.google.common.collect.Iterables.concat) MANAGED_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) Path(org.apache.hadoop.fs.Path) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) METASTORE_CONTEXT(com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableMap(com.google.common.collect.ImmutableMap) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) Threads.daemonThreadsNamed(com.facebook.airlift.concurrent.Threads.daemonThreadsNamed) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) PAGE_SORTER(com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER) RecordCursor(com.facebook.presto.spi.RecordCursor) DataSize(io.airlift.units.DataSize) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) Logger(com.facebook.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) DateTimeTestingUtils.sqlTimestampOf(com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) AtomicReference(java.util.concurrent.atomic.AtomicReference) VarcharType.createVarcharType(com.facebook.presto.common.type.VarcharType.createVarcharType) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Verify.verify(com.google.common.base.Verify.verify) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Objects.requireNonNull(java.util.Objects.requireNonNull) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) RowExpression(com.facebook.presto.spi.relation.RowExpression) Assertions.assertEqualsIgnoreOrder(com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) ConnectorIdentity(com.facebook.presto.spi.security.ConnectorIdentity) HiveBasicStatistics.createZeroStatistics(com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) RowFieldName(com.facebook.presto.common.type.RowFieldName) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Block(com.facebook.presto.common.block.Block) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) SECONDS(java.util.concurrent.TimeUnit.SECONDS) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Table(com.facebook.presto.hive.metastore.Table) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore)

Example 2 with METASTORE_CONTEXT

use of com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT in project presto by prestodb.

the class AbstractTestHiveClient method testStorePartitionWithStatistics.

protected void testStorePartitionWithStatistics(List<ColumnMetadata> columns, PartitionStatistics statsForAllColumns1, PartitionStatistics statsForAllColumns2, PartitionStatistics statsForSubsetOfColumns, PartitionStatistics emptyStatistics, Duration delayBetweenAlters) throws Exception {
    SchemaTableName tableName = temporaryTable("store_partition_with_statistics");
    try {
        doCreateEmptyTable(tableName, ORC, columns);
        ExtendedHiveMetastore metastoreClient = getMetastoreClient();
        Table table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
        List<String> partitionValues = ImmutableList.of("2016-01-01");
        String partitionName = makePartName(ImmutableList.of("ds"), partitionValues);
        Partition partition = createDummyPartition(table, partitionName);
        // create partition with stats for all columns
        metastoreClient.addPartitions(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), ImmutableList.of(new PartitionWithStatistics(partition, partitionName, statsForAllColumns1)));
        assertEquals(metastoreClient.getPartition(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), partitionValues).get().getStorage().getStorageFormat(), fromHiveStorageFormat(ORC));
        assertThat(metastoreClient.getPartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, statsForAllColumns1));
        sleep(delayBetweenAlters.toMillis());
        // alter the partition into one with other stats
        Partition modifiedPartition = Partition.builder(partition).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(DWRF)).setLocation(partitionTargetPath(tableName, partitionName))).build();
        metastoreClient.alterPartition(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), new PartitionWithStatistics(modifiedPartition, partitionName, statsForAllColumns2));
        assertEquals(metastoreClient.getPartition(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), partitionValues).get().getStorage().getStorageFormat(), fromHiveStorageFormat(DWRF));
        assertThat(metastoreClient.getPartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, statsForAllColumns2));
        sleep(delayBetweenAlters.toMillis());
        // alter the partition into one with stats for only subset of columns
        modifiedPartition = Partition.builder(partition).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE)).setLocation(partitionTargetPath(tableName, partitionName))).build();
        metastoreClient.alterPartition(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), new PartitionWithStatistics(modifiedPartition, partitionName, statsForSubsetOfColumns));
        assertThat(metastoreClient.getPartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, statsForSubsetOfColumns));
        sleep(delayBetweenAlters.toMillis());
        // alter the partition into one without stats
        modifiedPartition = Partition.builder(partition).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE)).setLocation(partitionTargetPath(tableName, partitionName))).build();
        metastoreClient.alterPartition(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), new PartitionWithStatistics(modifiedPartition, partitionName, emptyStatistics));
        assertThat(metastoreClient.getPartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, emptyStatistics));
    } finally {
        dropTable(tableName);
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) SkipException(org.testng.SkipException) WarningCollector(com.facebook.presto.spi.WarningCollector) CharType.createCharType(com.facebook.presto.common.type.CharType.createCharType) DateTimeZone(org.joda.time.DateTimeZone) SORTED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) FileSystem(org.apache.hadoop.fs.FileSystem) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) CounterStat(com.facebook.airlift.stats.CounterStat) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) FileStatus(org.apache.hadoop.fs.FileStatus) MoreFutures.getFutureValue(com.facebook.airlift.concurrent.MoreFutures.getFutureValue) MAX_PARTITION_KEY_COLUMN_INDEX(com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX) NOT_PARTITIONED(com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) Slices(io.airlift.slice.Slices) Map(java.util.Map) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) NullableValue(com.facebook.presto.common.predicate.NullableValue) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) SqlFunctionProperties(com.facebook.presto.common.function.SqlFunctionProperties) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) TABLE(com.facebook.presto.hive.CacheQuotaScope.TABLE) ZoneId(java.time.ZoneId) HiveColumnHandle.bucketColumnHandle(com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) SqlTimestamp(com.facebook.presto.common.type.SqlTimestamp) Predicate(com.google.common.base.Predicate) HiveColumnStatistics.createBinaryColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) StandardTypes(com.facebook.presto.common.type.StandardTypes) Table(com.facebook.presto.hive.metastore.Table) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveTestUtils.arrayType(com.facebook.presto.hive.HiveTestUtils.arrayType) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) REAL(com.facebook.presto.common.type.RealType.REAL) PAGEFILE(com.facebook.presto.hive.HiveStorageFormat.PAGEFILE) OptionalLong(java.util.OptionalLong) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) GIGABYTE(io.airlift.units.DataSize.Unit.GIGABYTE) HIVE_BYTE(com.facebook.presto.hive.HiveType.HIVE_BYTE) SqlVarbinary(com.facebook.presto.common.type.SqlVarbinary) Assertions.assertInstanceOf(com.facebook.airlift.testing.Assertions.assertInstanceOf) CSV(com.facebook.presto.hive.HiveStorageFormat.CSV) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) MetastoreCacheScope(com.facebook.presto.hive.metastore.CachingHiveMetastore.MetastoreCacheScope) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) AfterClass(org.testng.annotations.AfterClass) FileUtils.makePartName(org.apache.hadoop.hive.common.FileUtils.makePartName) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) HostAndPort(com.google.common.net.HostAndPort) Domain(com.facebook.presto.common.predicate.Domain) FUNCTION_AND_TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.FUNCTION_AND_TYPE_MANAGER) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) SqlFunctionId(com.facebook.presto.spi.function.SqlFunctionId) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) SplitSchedulingContext(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingContext) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) HiveColumnStatistics.createDecimalColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) RowType(com.facebook.presto.common.type.RowType) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveFilterPushdown.pushdownFilter(com.facebook.presto.hive.rule.HiveFilterPushdown.pushdownFilter) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) HIVE_FLOAT(com.facebook.presto.hive.HiveType.HIVE_FLOAT) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) TypeProvider(com.facebook.presto.sql.planner.TypeProvider) Locale(java.util.Locale) UNGROUPED_SCHEDULING(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) Thread.sleep(java.lang.Thread.sleep) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) PageFilePageSource(com.facebook.presto.hive.pagefile.PageFilePageSource) ImmutableSet(com.google.common.collect.ImmutableSet) OFFLINE_DATA_DEBUG_MODE_ENABLED(com.facebook.presto.hive.HiveSessionProperties.OFFLINE_DATA_DEBUG_MODE_ENABLED) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) Range(com.facebook.presto.common.predicate.Range) Objects(java.util.Objects) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) MetastoreUtil.toPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues) HiveColumnStatistics.createBooleanColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics) METADATA(com.facebook.presto.hive.HiveTestUtils.METADATA) HiveTestUtils.getDefaultHiveBatchPageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories) IntStream(java.util.stream.IntStream) MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) MapType(com.facebook.presto.common.type.MapType) FILTER_STATS_CALCULATOR_SERVICE(com.facebook.presto.hive.HiveTestUtils.FILTER_STATS_CALCULATOR_SERVICE) Assert.assertNull(org.testng.Assert.assertNull) HiveColumnStatistics.createIntegerColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) OptionalDouble(java.util.OptionalDouble) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) OptionalInt(java.util.OptionalInt) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) HIVE_DOUBLE(com.facebook.presto.hive.HiveType.HIVE_DOUBLE) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) HiveTestUtils.mapType(com.facebook.presto.hive.HiveTestUtils.mapType) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) Type(com.facebook.presto.common.type.Type) ExecutorService(java.util.concurrent.ExecutorService) NamedTypeSignature(com.facebook.presto.common.type.NamedTypeSignature) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) DEFAULT_COLUMN_CONVERTER_PROVIDER(com.facebook.presto.hive.HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER) USER(com.facebook.presto.spi.security.PrincipalType.USER) HiveTestUtils.getDefaultHiveSelectivePageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories) HiveColumnStatistics.createStringColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createStringColumnStatistics) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) PageSinkContext(com.facebook.presto.spi.PageSinkContext) OutputStreamDataSinkFactory(com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory) MetastoreUtil.createDirectory(com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory) TRANSACTION_CONFLICT(com.facebook.presto.spi.StandardErrorCode.TRANSACTION_CONFLICT) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) Hashing.sha256(com.google.common.hash.Hashing.sha256) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPartitioningMetadata(com.facebook.presto.spi.connector.ConnectorPartitioningMetadata) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Assert.assertTrue(org.testng.Assert.assertTrue) PlanBuilder.expression(com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) DecimalType.createDecimalType(com.facebook.presto.common.type.DecimalType.createDecimalType) HIVE_COMPATIBLE(com.facebook.presto.hive.BucketFunctionType.HIVE_COMPATIBLE) Page(com.facebook.presto.common.Page) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ColumnStatistics(com.facebook.presto.spi.statistics.ColumnStatistics) HIVE_SHORT(com.facebook.presto.hive.HiveType.HIVE_SHORT) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) SqlInvokedFunction(com.facebook.presto.spi.function.SqlInvokedFunction) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) HiveColumnStatistics.createDoubleColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics) BigDecimal(java.math.BigDecimal) TupleDomain.withColumnDomains(com.facebook.presto.common.predicate.TupleDomain.withColumnDomains) Sets.difference(com.google.common.collect.Sets.difference) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) PRESTO_VERSION_NAME(com.facebook.presto.hive.HiveMetadata.PRESTO_VERSION_NAME) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TestingRowExpressionTranslator(com.facebook.presto.sql.TestingRowExpressionTranslator) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Executors(java.util.concurrent.Executors) ConnectorSession(com.facebook.presto.spi.ConnectorSession) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) Assertions.assertGreaterThan(com.facebook.airlift.testing.Assertions.assertGreaterThan) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) OrcSelectivePageSource(com.facebook.presto.hive.orc.OrcSelectivePageSource) STAGE_AND_MOVE_TO_TARGET_DIRECTORY(com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY) VariablesExtractor.extractUnique(com.facebook.presto.sql.planner.VariablesExtractor.extractUnique) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Slice(io.airlift.slice.Slice) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) TINYINT(com.facebook.presto.common.type.TinyintType.TINYINT) Assertions.assertGreaterThanOrEqual(com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) WriteInfo(com.facebook.presto.hive.LocationService.WriteInfo) HYPER_LOG_LOG(com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG) HiveBasicStatistics.createEmptyStatistics(com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics) DATE(com.facebook.presto.common.type.DateType.DATE) FUNCTION_RESOLUTION(com.facebook.presto.hive.HiveTestUtils.FUNCTION_RESOLUTION) ArrayList(java.util.ArrayList) ROW_EXPRESSION_SERVICE(com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) SqlDate(com.facebook.presto.common.type.SqlDate) NON_CACHEABLE(com.facebook.presto.spi.SplitContext.NON_CACHEABLE) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) TableHandle(com.facebook.presto.spi.TableHandle) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) HiveTestUtils.rowType(com.facebook.presto.hive.HiveTestUtils.rowType) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) LongStream(java.util.stream.LongStream) Executor(java.util.concurrent.Executor) Constraint(com.facebook.presto.spi.Constraint) UTC_KEY(com.facebook.presto.common.type.TimeZoneKey.UTC_KEY) UTC(org.joda.time.DateTimeZone.UTC) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SMALLINT(com.facebook.presto.common.type.SmallintType.SMALLINT) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Assertions.assertLessThanOrEqual(com.facebook.airlift.testing.Assertions.assertLessThanOrEqual) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) HIVE_BOOLEAN(com.facebook.presto.hive.HiveType.HIVE_BOOLEAN) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) HiveColumnStatistics.createDateColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics) ValueSet(com.facebook.presto.common.predicate.ValueSet) SORTED_WRITE_TO_TEMP_PATH_ENABLED(com.facebook.presto.hive.HiveSessionProperties.SORTED_WRITE_TO_TEMP_PATH_ENABLED) MetadataManager(com.facebook.presto.metadata.MetadataManager) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) TableStatistics(com.facebook.presto.spi.statistics.TableStatistics) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) CacheConfig(com.facebook.presto.cache.CacheConfig) HiveTestUtils.getDefaultOrcFileWriterFactory(com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) MoreCollectors.onlyElement(com.google.common.collect.MoreCollectors.onlyElement) OrcBatchPageSource(com.facebook.presto.hive.orc.OrcBatchPageSource) Iterables.concat(com.google.common.collect.Iterables.concat) MANAGED_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) Path(org.apache.hadoop.fs.Path) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) METASTORE_CONTEXT(com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableMap(com.google.common.collect.ImmutableMap) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) Threads.daemonThreadsNamed(com.facebook.airlift.concurrent.Threads.daemonThreadsNamed) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) PAGE_SORTER(com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER) RecordCursor(com.facebook.presto.spi.RecordCursor) DataSize(io.airlift.units.DataSize) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) Logger(com.facebook.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) DateTimeTestingUtils.sqlTimestampOf(com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) AtomicReference(java.util.concurrent.atomic.AtomicReference) VarcharType.createVarcharType(com.facebook.presto.common.type.VarcharType.createVarcharType) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Verify.verify(com.google.common.base.Verify.verify) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Objects.requireNonNull(java.util.Objects.requireNonNull) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) RowExpression(com.facebook.presto.spi.relation.RowExpression) Assertions.assertEqualsIgnoreOrder(com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) ConnectorIdentity(com.facebook.presto.spi.security.ConnectorIdentity) HiveBasicStatistics.createZeroStatistics(com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) RowFieldName(com.facebook.presto.common.type.RowFieldName) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Block(com.facebook.presto.common.block.Block) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) SECONDS(java.util.concurrent.TimeUnit.SECONDS) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Partition(com.facebook.presto.hive.metastore.Partition) Table(com.facebook.presto.hive.metastore.Table) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) SchemaTableName(com.facebook.presto.spi.SchemaTableName)

Example 3 with METASTORE_CONTEXT

use of com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT in project presto by prestodb.

the class TestHiveClientFileMetastore method createDummyPartitionedTable.

private void createDummyPartitionedTable(SchemaTableName tableName, List<ColumnMetadata> columns, Map<String, String> dynamicPartitionParameters) throws Exception {
    doCreateEmptyTable(tableName, ORC, columns);
    ExtendedHiveMetastore metastoreClient = getMetastoreClient();
    Table table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    List<String> firstPartitionValues = ImmutableList.of("2020-01-01");
    List<String> secondPartitionValues = ImmutableList.of("2020-01-02");
    List<String> thirdPartitionValues = ImmutableList.of("2020-01-03");
    String firstPartitionName = makePartName(ImmutableList.of("ds"), firstPartitionValues);
    String secondPartitionName = makePartName(ImmutableList.of("ds"), secondPartitionValues);
    String thirdPartitionName = makePartName(ImmutableList.of("ds"), thirdPartitionValues);
    List<PartitionWithStatistics> partitions = ImmutableList.of(firstPartitionName, secondPartitionName).stream().map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName), partitionName, PartitionStatistics.empty())).collect(toImmutableList());
    metastoreClient.addPartitions(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), partitions);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), firstPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), secondPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
    List<PartitionWithStatistics> partitionsNotReadable = ImmutableList.of(thirdPartitionName).stream().map(partitionName -> new PartitionWithStatistics(createDummyPartition(table, partitionName, dynamicPartitionParameters), partitionName, PartitionStatistics.empty())).collect(toImmutableList());
    metastoreClient.addPartitions(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), partitionsNotReadable);
    metastoreClient.updatePartitionStatistics(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), thirdPartitionName, currentStatistics -> EMPTY_TABLE_STATISTICS);
}
Also used : SkipException(org.testng.SkipException) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) Table(com.facebook.presto.hive.metastore.Table) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) Test(org.testng.annotations.Test) Partition(com.facebook.presto.hive.metastore.Partition) SchemaTableName(com.facebook.presto.spi.SchemaTableName) WarningCollectorConfig(com.facebook.presto.execution.warnings.WarningCollectorConfig) ImmutableList(com.google.common.collect.ImmutableList) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) PRESTO_VERSION_NAME(com.facebook.presto.hive.HiveMetadata.PRESTO_VERSION_NAME) Map(java.util.Map) UNGROUPED_SCHEDULING(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) OBJECT_NOT_READABLE(com.facebook.presto.hive.HiveSplitManager.OBJECT_NOT_READABLE) METASTORE_CONTEXT(com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) FileUtils.makePartName(org.apache.hadoop.hive.common.FileUtils.makePartName) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Assert.fail(org.testng.Assert.fail) Constraint(com.facebook.presto.spi.Constraint) Assert.assertNotNull(org.testng.Assert.assertNotNull) WarningHandlingLevel(com.facebook.presto.execution.warnings.WarningHandlingLevel) File(java.io.File) ConnectorSession(com.facebook.presto.spi.ConnectorSession) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) FileHiveMetastore(com.facebook.presto.hive.metastore.file.FileHiveMetastore) ColumnHandle(com.facebook.presto.spi.ColumnHandle) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) MetastoreUtil.toPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) DefaultWarningCollector(com.facebook.presto.execution.warnings.DefaultWarningCollector) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) SplitSchedulingContext(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingContext) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Table(com.facebook.presto.hive.metastore.Table) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore)

Example 4 with METASTORE_CONTEXT

use of com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT in project presto by prestodb.

the class AbstractTestHiveClient method testDropColumn.

@Test
public void testDropColumn() throws Exception {
    SchemaTableName tableName = temporaryTable("test_drop_column");
    try {
        doCreateEmptyTable(tableName, ORC, CREATE_TABLE_COLUMNS);
        ExtendedHiveMetastore metastoreClient = getMetastoreClient();
        metastoreClient.dropColumn(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName(), CREATE_TABLE_COLUMNS.get(0).getName());
        Optional<Table> table = metastoreClient.getTable(METASTORE_CONTEXT, tableName.getSchemaName(), tableName.getTableName());
        assertTrue(table.isPresent());
        List<Column> columns = table.get().getDataColumns();
        assertEquals(columns.get(0).getName(), CREATE_TABLE_COLUMNS.get(1).getName());
        assertFalse(columns.stream().map(Column::getName).anyMatch(colName -> colName.equals(CREATE_TABLE_COLUMNS.get(0).getName())));
    } finally {
        dropTable(tableName);
    }
}
Also used : RecordPageSource(com.facebook.presto.spi.RecordPageSource) SkipException(org.testng.SkipException) WarningCollector(com.facebook.presto.spi.WarningCollector) CharType.createCharType(com.facebook.presto.common.type.CharType.createCharType) DateTimeZone(org.joda.time.DateTimeZone) SORTED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.SORTED_BY_PROPERTY) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) FileSystem(org.apache.hadoop.fs.FileSystem) PrestoPrincipal(com.facebook.presto.spi.security.PrestoPrincipal) CounterStat(com.facebook.airlift.stats.CounterStat) Test(org.testng.annotations.Test) HIVE_PARTITION_SCHEMA_MISMATCH(com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH) FileStatus(org.apache.hadoop.fs.FileStatus) MoreFutures.getFutureValue(com.facebook.airlift.concurrent.MoreFutures.getFutureValue) MAX_PARTITION_KEY_COLUMN_INDEX(com.facebook.presto.hive.HiveColumnHandle.MAX_PARTITION_KEY_COLUMN_INDEX) NOT_PARTITIONED(com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) Slices(io.airlift.slice.Slices) Map(java.util.Map) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) NullableValue(com.facebook.presto.common.predicate.NullableValue) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) SqlFunctionProperties(com.facebook.presto.common.function.SqlFunctionProperties) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) TABLE(com.facebook.presto.hive.CacheQuotaScope.TABLE) ZoneId(java.time.ZoneId) HiveColumnHandle.bucketColumnHandle(com.facebook.presto.hive.HiveColumnHandle.bucketColumnHandle) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) SqlTimestamp(com.facebook.presto.common.type.SqlTimestamp) Predicate(com.google.common.base.Predicate) HiveColumnStatistics.createBinaryColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBinaryColumnStatistics) HivePrivilegeInfo(com.facebook.presto.hive.metastore.HivePrivilegeInfo) StandardTypes(com.facebook.presto.common.type.StandardTypes) Table(com.facebook.presto.hive.metastore.Table) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) BUCKET_COLUMN_NAME(com.facebook.presto.hive.HiveColumnHandle.BUCKET_COLUMN_NAME) HiveTestUtils.arrayType(com.facebook.presto.hive.HiveTestUtils.arrayType) HiveUtil.columnExtraInfo(com.facebook.presto.hive.HiveUtil.columnExtraInfo) HiveColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) REAL(com.facebook.presto.common.type.RealType.REAL) PAGEFILE(com.facebook.presto.hive.HiveStorageFormat.PAGEFILE) OptionalLong(java.util.OptionalLong) ROLLBACK_AFTER_SINK_FINISH(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_SINK_FINISH) GIGABYTE(io.airlift.units.DataSize.Unit.GIGABYTE) HIVE_BYTE(com.facebook.presto.hive.HiveType.HIVE_BYTE) SqlVarbinary(com.facebook.presto.common.type.SqlVarbinary) Assertions.assertInstanceOf(com.facebook.airlift.testing.Assertions.assertInstanceOf) CSV(com.facebook.presto.hive.HiveStorageFormat.CSV) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) MetastoreCacheScope(com.facebook.presto.hive.metastore.CachingHiveMetastore.MetastoreCacheScope) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) AfterClass(org.testng.annotations.AfterClass) FileUtils.makePartName(org.apache.hadoop.hive.common.FileUtils.makePartName) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) HostAndPort(com.google.common.net.HostAndPort) Domain(com.facebook.presto.common.predicate.Domain) FUNCTION_AND_TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.FUNCTION_AND_TYPE_MANAGER) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) SqlFunctionId(com.facebook.presto.spi.function.SqlFunctionId) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) SchemaTablePrefix(com.facebook.presto.spi.SchemaTablePrefix) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) SplitSchedulingContext(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingContext) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) HiveColumnStatistics.createDecimalColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDecimalColumnStatistics) ConnectorViewDefinition(com.facebook.presto.spi.ConnectorViewDefinition) RowType(com.facebook.presto.common.type.RowType) ViewNotFoundException(com.facebook.presto.spi.ViewNotFoundException) ORC(com.facebook.presto.hive.HiveStorageFormat.ORC) ROLLBACK_AFTER_FINISH_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_FINISH_INSERT) HiveFilterPushdown.pushdownFilter(com.facebook.presto.hive.rule.HiveFilterPushdown.pushdownFilter) Duration(io.airlift.units.Duration) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) HIVE_FLOAT(com.facebook.presto.hive.HiveType.HIVE_FLOAT) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) TypeProvider(com.facebook.presto.sql.planner.TypeProvider) Locale(java.util.Locale) UNGROUPED_SCHEDULING(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) BUCKETED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKETED_BY_PROPERTY) Thread.sleep(java.lang.Thread.sleep) DiscretePredicates(com.facebook.presto.spi.DiscretePredicates) TEXTFILE(com.facebook.presto.hive.HiveStorageFormat.TEXTFILE) PageFilePageSource(com.facebook.presto.hive.pagefile.PageFilePageSource) ImmutableSet(com.google.common.collect.ImmutableSet) OFFLINE_DATA_DEBUG_MODE_ENABLED(com.facebook.presto.hive.HiveSessionProperties.OFFLINE_DATA_DEBUG_MODE_ENABLED) TimeZone(java.util.TimeZone) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) DWRF(com.facebook.presto.hive.HiveStorageFormat.DWRF) UUID(java.util.UUID) Assert.assertNotNull(org.testng.Assert.assertNotNull) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) Range(com.facebook.presto.common.predicate.Range) Objects(java.util.Objects) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) MetastoreUtil.toPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues) HiveColumnStatistics.createBooleanColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createBooleanColumnStatistics) METADATA(com.facebook.presto.hive.HiveTestUtils.METADATA) HiveTestUtils.getDefaultHiveBatchPageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories) IntStream(java.util.stream.IntStream) MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) MapType(com.facebook.presto.common.type.MapType) FILTER_STATS_CALCULATOR_SERVICE(com.facebook.presto.hive.HiveTestUtils.FILTER_STATS_CALCULATOR_SERVICE) Assert.assertNull(org.testng.Assert.assertNull) HiveColumnStatistics.createIntegerColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createIntegerColumnStatistics) RcFilePageSource(com.facebook.presto.hive.rcfile.RcFilePageSource) ConnectorTableLayoutHandle(com.facebook.presto.spi.ConnectorTableLayoutHandle) OptionalDouble(java.util.OptionalDouble) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) OptionalInt(java.util.OptionalInt) HashSet(java.util.HashSet) ROLLBACK_AFTER_APPEND_PAGE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_APPEND_PAGE) Subfield(com.facebook.presto.common.Subfield) ImmutableList(com.google.common.collect.ImmutableList) HIVE_DOUBLE(com.facebook.presto.hive.HiveType.HIVE_DOUBLE) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) HiveTestUtils.mapType(com.facebook.presto.hive.HiveTestUtils.mapType) Math.toIntExact(java.lang.Math.toIntExact) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) Type(com.facebook.presto.common.type.Type) ExecutorService(java.util.concurrent.ExecutorService) NamedTypeSignature(com.facebook.presto.common.type.NamedTypeSignature) ConnectorInsertTableHandle(com.facebook.presto.spi.ConnectorInsertTableHandle) DEFAULT_COLUMN_CONVERTER_PROVIDER(com.facebook.presto.hive.HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER) USER(com.facebook.presto.spi.security.PrincipalType.USER) HiveTestUtils.getDefaultHiveSelectivePageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories) HiveColumnStatistics.createStringColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createStringColumnStatistics) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) UTF_8(java.nio.charset.StandardCharsets.UTF_8) ConnectorTableLayout(com.facebook.presto.spi.ConnectorTableLayout) Assert.fail(org.testng.Assert.fail) DateTime(org.joda.time.DateTime) PageSinkContext(com.facebook.presto.spi.PageSinkContext) OutputStreamDataSinkFactory(com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory) MetastoreUtil.createDirectory(com.facebook.presto.hive.metastore.MetastoreUtil.createDirectory) TRANSACTION_CONFLICT(com.facebook.presto.spi.StandardErrorCode.TRANSACTION_CONFLICT) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) Hashing.sha256(com.google.common.hash.Hashing.sha256) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPartitioningMetadata(com.facebook.presto.spi.connector.ConnectorPartitioningMetadata) BUCKET_COUNT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.BUCKET_COUNT_PROPERTY) Assert.assertTrue(org.testng.Assert.assertTrue) PlanBuilder.expression(com.facebook.presto.sql.planner.iterative.rule.test.PlanBuilder.expression) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) DecimalType.createDecimalType(com.facebook.presto.common.type.DecimalType.createDecimalType) HIVE_COMPATIBLE(com.facebook.presto.hive.BucketFunctionType.HIVE_COMPATIBLE) Page(com.facebook.presto.common.Page) Arrays(java.util.Arrays) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ColumnStatistics(com.facebook.presto.spi.statistics.ColumnStatistics) HIVE_SHORT(com.facebook.presto.hive.HiveType.HIVE_SHORT) PartitionWithStatistics(com.facebook.presto.hive.metastore.PartitionWithStatistics) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) Maps.uniqueIndex(com.google.common.collect.Maps.uniqueIndex) SqlInvokedFunction(com.facebook.presto.spi.function.SqlInvokedFunction) ROLLBACK_AFTER_BEGIN_INSERT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_BEGIN_INSERT) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) HiveColumnStatistics.createDoubleColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDoubleColumnStatistics) BigDecimal(java.math.BigDecimal) TupleDomain.withColumnDomains(com.facebook.presto.common.predicate.TupleDomain.withColumnDomains) Sets.difference(com.google.common.collect.Sets.difference) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) ROLLBACK_AFTER_DELETE(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_AFTER_DELETE) PRESTO_VERSION_NAME(com.facebook.presto.hive.HiveMetadata.PRESTO_VERSION_NAME) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Varchars.isVarcharType(com.facebook.presto.common.type.Varchars.isVarcharType) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TestingRowExpressionTranslator(com.facebook.presto.sql.TestingRowExpressionTranslator) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) Executors(java.util.concurrent.Executors) ConnectorSession(com.facebook.presto.spi.ConnectorSession) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) Assertions.assertGreaterThan(com.facebook.airlift.testing.Assertions.assertGreaterThan) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) ParquetPageSource(com.facebook.presto.hive.parquet.ParquetPageSource) OrcSelectivePageSource(com.facebook.presto.hive.orc.OrcSelectivePageSource) STAGE_AND_MOVE_TO_TARGET_DIRECTORY(com.facebook.presto.hive.LocationHandle.WriteMode.STAGE_AND_MOVE_TO_TARGET_DIRECTORY) VariablesExtractor.extractUnique(com.facebook.presto.sql.planner.VariablesExtractor.extractUnique) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) Slice(io.airlift.slice.Slice) Chars.isCharType(com.facebook.presto.common.type.Chars.isCharType) TINYINT(com.facebook.presto.common.type.TinyintType.TINYINT) Assertions.assertGreaterThanOrEqual(com.facebook.airlift.testing.Assertions.assertGreaterThanOrEqual) TIMESTAMP(com.facebook.presto.common.type.TimestampType.TIMESTAMP) WriteInfo(com.facebook.presto.hive.LocationService.WriteInfo) HYPER_LOG_LOG(com.facebook.presto.common.type.HyperLogLogType.HYPER_LOG_LOG) HiveBasicStatistics.createEmptyStatistics(com.facebook.presto.hive.HiveBasicStatistics.createEmptyStatistics) DATE(com.facebook.presto.common.type.DateType.DATE) FUNCTION_RESOLUTION(com.facebook.presto.hive.HiveTestUtils.FUNCTION_RESOLUTION) ArrayList(java.util.ArrayList) ROW_EXPRESSION_SERVICE(com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) SqlDate(com.facebook.presto.common.type.SqlDate) NON_CACHEABLE(com.facebook.presto.spi.SplitContext.NON_CACHEABLE) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) RCTEXT(com.facebook.presto.hive.HiveStorageFormat.RCTEXT) BOOLEAN(com.facebook.presto.common.type.BooleanType.BOOLEAN) ArrayType(com.facebook.presto.common.type.ArrayType) TableHandle(com.facebook.presto.spi.TableHandle) JSON(com.facebook.presto.hive.HiveStorageFormat.JSON) HiveTestUtils.rowType(com.facebook.presto.hive.HiveTestUtils.rowType) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) LongStream(java.util.stream.LongStream) Executor(java.util.concurrent.Executor) Constraint(com.facebook.presto.spi.Constraint) UTC_KEY(com.facebook.presto.common.type.TimeZoneKey.UTC_KEY) UTC(org.joda.time.DateTimeZone.UTC) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) RCBINARY(com.facebook.presto.hive.HiveStorageFormat.RCBINARY) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) HivePrivilege(com.facebook.presto.hive.metastore.HivePrivilegeInfo.HivePrivilege) SMALLINT(com.facebook.presto.common.type.SmallintType.SMALLINT) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Assertions.assertLessThanOrEqual(com.facebook.airlift.testing.Assertions.assertLessThanOrEqual) ROLLBACK_RIGHT_AWAY(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.ROLLBACK_RIGHT_AWAY) HiveMetadata.convertToPredicate(com.facebook.presto.hive.HiveMetadata.convertToPredicate) HIVE_BOOLEAN(com.facebook.presto.hive.HiveType.HIVE_BOOLEAN) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) HiveColumnStatistics.createDateColumnStatistics(com.facebook.presto.hive.metastore.HiveColumnStatistics.createDateColumnStatistics) ValueSet(com.facebook.presto.common.predicate.ValueSet) SORTED_WRITE_TO_TEMP_PATH_ENABLED(com.facebook.presto.hive.HiveSessionProperties.SORTED_WRITE_TO_TEMP_PATH_ENABLED) MetadataManager(com.facebook.presto.metadata.MetadataManager) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) TableStatistics(com.facebook.presto.spi.statistics.TableStatistics) HiveType.toHiveType(com.facebook.presto.hive.HiveType.toHiveType) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) CacheConfig(com.facebook.presto.cache.CacheConfig) HiveTestUtils.getDefaultOrcFileWriterFactory(com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) MoreCollectors.onlyElement(com.google.common.collect.MoreCollectors.onlyElement) OrcBatchPageSource(com.facebook.presto.hive.orc.OrcBatchPageSource) Iterables.concat(com.google.common.collect.Iterables.concat) MANAGED_TABLE(com.facebook.presto.hive.metastore.PrestoTableType.MANAGED_TABLE) AVRO(com.facebook.presto.hive.HiveStorageFormat.AVRO) Path(org.apache.hadoop.fs.Path) KILOBYTE(io.airlift.units.DataSize.Unit.KILOBYTE) METASTORE_CONTEXT(com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableMap(com.google.common.collect.ImmutableMap) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) String.format(java.lang.String.format) COMMIT(com.facebook.presto.hive.AbstractTestHiveClient.TransactionDeleteInsertTestTag.COMMIT) Preconditions.checkState(com.google.common.base.Preconditions.checkState) Threads.daemonThreadsNamed(com.facebook.airlift.concurrent.Threads.daemonThreadsNamed) STORAGE_FORMAT_PROPERTY(com.facebook.presto.hive.HiveTableProperties.STORAGE_FORMAT_PROPERTY) PAGE_SORTER(com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER) RecordCursor(com.facebook.presto.spi.RecordCursor) DataSize(io.airlift.units.DataSize) List(java.util.List) PrestoTableType(com.facebook.presto.hive.metastore.PrestoTableType) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) PARTITION_KEY(com.facebook.presto.hive.HiveColumnHandle.ColumnType.PARTITION_KEY) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) Logger(com.facebook.airlift.log.Logger) Column(com.facebook.presto.hive.metastore.Column) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) DateTimeTestingUtils.sqlTimestampOf(com.facebook.presto.testing.DateTimeTestingUtils.sqlTimestampOf) HIVE_INVALID_PARTITION_VALUE(com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) AtomicReference(java.util.concurrent.atomic.AtomicReference) VarcharType.createVarcharType(com.facebook.presto.common.type.VarcharType.createVarcharType) PARQUET(com.facebook.presto.hive.HiveStorageFormat.PARQUET) Partition(com.facebook.presto.hive.metastore.Partition) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) PARTITIONED_BY_PROPERTY(com.facebook.presto.hive.HiveTableProperties.PARTITIONED_BY_PROPERTY) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Verify.verify(com.google.common.base.Verify.verify) SESSION(com.facebook.presto.hive.HiveTestUtils.SESSION) Objects.requireNonNull(java.util.Objects.requireNonNull) SEQUENCEFILE(com.facebook.presto.hive.HiveStorageFormat.SEQUENCEFILE) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) RowExpression(com.facebook.presto.spi.relation.RowExpression) Assertions.assertEqualsIgnoreOrder(com.facebook.airlift.testing.Assertions.assertEqualsIgnoreOrder) VARBINARY(com.facebook.presto.common.type.VarbinaryType.VARBINARY) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) ConnectorIdentity(com.facebook.presto.spi.security.ConnectorIdentity) HiveBasicStatistics.createZeroStatistics(com.facebook.presto.hive.HiveBasicStatistics.createZeroStatistics) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) StorageFormat.fromHiveStorageFormat(com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat) RowFieldName(com.facebook.presto.common.type.RowFieldName) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) Block(com.facebook.presto.common.block.Block) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Table(com.facebook.presto.hive.metastore.Table) Column(com.facebook.presto.hive.metastore.Column) SortingColumn(com.facebook.presto.hive.metastore.SortingColumn) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) SchemaTableName(com.facebook.presto.spi.SchemaTableName) Test(org.testng.annotations.Test)

Aggregations

MoreFutures.getFutureValue (com.facebook.airlift.concurrent.MoreFutures.getFutureValue)3 Threads.daemonThreadsNamed (com.facebook.airlift.concurrent.Threads.daemonThreadsNamed)3 Logger (com.facebook.airlift.log.Logger)3 PRESTO_VERSION_NAME (com.facebook.presto.hive.HiveMetadata.PRESTO_VERSION_NAME)3 METASTORE_CONTEXT (com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT)3 ORC (com.facebook.presto.hive.HiveStorageFormat.ORC)3 NoHdfsAuthentication (com.facebook.presto.hive.authentication.NoHdfsAuthentication)3 ExtendedHiveMetastore (com.facebook.presto.hive.metastore.ExtendedHiveMetastore)3 PRESTO_QUERY_ID_NAME (com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME)3 MetastoreUtil.toPartitionValues (com.facebook.presto.hive.metastore.MetastoreUtil.toPartitionValues)3 Partition (com.facebook.presto.hive.metastore.Partition)3 PartitionStatistics (com.facebook.presto.hive.metastore.PartitionStatistics)3 PartitionWithStatistics (com.facebook.presto.hive.metastore.PartitionWithStatistics)3 StorageFormat.fromHiveStorageFormat (com.facebook.presto.hive.metastore.StorageFormat.fromHiveStorageFormat)3 Table (com.facebook.presto.hive.metastore.Table)3 ColumnHandle (com.facebook.presto.spi.ColumnHandle)3 ColumnMetadata (com.facebook.presto.spi.ColumnMetadata)3 ConnectorSession (com.facebook.presto.spi.ConnectorSession)3 ConnectorTableHandle (com.facebook.presto.spi.ConnectorTableHandle)3 ConnectorTableLayout (com.facebook.presto.spi.ConnectorTableLayout)3