Search in sources :

Example 1 with BOOLEAN

use of com.facebook.presto.common.type.StandardTypes.BOOLEAN in project presto by prestodb.

the class SyncPartitionMetadataProcedure method doSyncPartitionMetadata.

private void doSyncPartitionMetadata(ConnectorSession session, String schemaName, String tableName, String mode, boolean caseSensitive) {
    SyncMode syncMode = toSyncMode(mode);
    SemiTransactionalHiveMetastore metastore = hiveMetadataFactory.get().getMetastore();
    SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
    Table table = metastore.getTable(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), schemaName, tableName).orElseThrow(() -> new TableNotFoundException(schemaTableName));
    if (table.getPartitionColumns().isEmpty()) {
        throw new PrestoException(INVALID_PROCEDURE_ARGUMENT, "Table is not partitioned: " + schemaTableName);
    }
    Path tableLocation = new Path(table.getStorage().getLocation());
    HdfsContext context = new HdfsContext(session, schemaName, tableName, table.getStorage().getLocation(), false);
    Set<String> partitionsToAdd;
    Set<String> partitionsToDrop;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, tableLocation);
        List<String> partitionsInMetastore = metastore.getPartitionNames(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), schemaName, tableName).orElseThrow(() -> new TableNotFoundException(schemaTableName));
        List<String> partitionsInFileSystem = listDirectory(fileSystem, fileSystem.getFileStatus(tableLocation), table.getPartitionColumns(), table.getPartitionColumns().size(), caseSensitive).stream().map(fileStatus -> fileStatus.getPath().toUri()).map(uri -> tableLocation.toUri().relativize(uri).getPath()).collect(toImmutableList());
        // partitions in file system but not in metastore
        partitionsToAdd = difference(partitionsInFileSystem, partitionsInMetastore);
        // partitions in metastore but not in file system
        partitionsToDrop = difference(partitionsInMetastore, partitionsInFileSystem);
    } catch (IOException e) {
        throw new PrestoException(HIVE_FILESYSTEM_ERROR, e);
    }
    syncPartitions(partitionsToAdd, partitionsToDrop, syncMode, metastore, session, table);
}
Also used : Path(org.apache.hadoop.fs.Path) MethodHandle(java.lang.invoke.MethodHandle) Table(com.facebook.presto.hive.metastore.Table) Provider(javax.inject.Provider) Column(com.facebook.presto.hive.metastore.Column) FileSystem(org.apache.hadoop.fs.FileSystem) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) PrestoException(com.facebook.presto.spi.PrestoException) FileStatus(org.apache.hadoop.fs.FileStatus) Supplier(java.util.function.Supplier) Partition(com.facebook.presto.hive.metastore.Partition) Inject(javax.inject.Inject) HashSet(java.util.HashSet) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) Path(org.apache.hadoop.fs.Path) ThreadContextClassLoader(com.facebook.presto.spi.classloader.ThreadContextClassLoader) ENGLISH(java.util.Locale.ENGLISH) Argument(com.facebook.presto.spi.procedure.Procedure.Argument) BOOLEAN(com.facebook.presto.common.type.StandardTypes.BOOLEAN) ImmutableMap(com.google.common.collect.ImmutableMap) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) IOException(java.io.IOException) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) Sets(com.google.common.collect.Sets) ConnectorSession(com.facebook.presto.spi.ConnectorSession) VARCHAR(com.facebook.presto.common.type.StandardTypes.VARCHAR) INVALID_PROCEDURE_ARGUMENT(com.facebook.presto.spi.StandardErrorCode.INVALID_PROCEDURE_ARGUMENT) MethodHandleUtil.methodHandle(com.facebook.presto.common.block.MethodHandleUtil.methodHandle) Procedure(com.facebook.presto.spi.procedure.Procedure) List(java.util.List) HIVE_FILESYSTEM_ERROR(com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) Stream(java.util.stream.Stream) MetastoreUtil.extractPartitionValues(com.facebook.presto.hive.metastore.MetastoreUtil.extractPartitionValues) PartitionStatistics(com.facebook.presto.hive.metastore.PartitionStatistics) MetastoreUtil.getMetastoreHeaders(com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders) MetastoreUtil.isUserDefinedTypeEncodingEnabled(com.facebook.presto.hive.metastore.MetastoreUtil.isUserDefinedTypeEncodingEnabled) PRESTO_QUERY_ID_NAME(com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME) TRUE(java.lang.Boolean.TRUE) Table(com.facebook.presto.hive.metastore.Table) SemiTransactionalHiveMetastore(com.facebook.presto.hive.metastore.SemiTransactionalHiveMetastore) MetastoreContext(com.facebook.presto.hive.metastore.MetastoreContext) PrestoException(com.facebook.presto.spi.PrestoException) IOException(java.io.IOException) SchemaTableName(com.facebook.presto.spi.SchemaTableName) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) FileSystem(org.apache.hadoop.fs.FileSystem)

Example 2 with BOOLEAN

use of com.facebook.presto.common.type.StandardTypes.BOOLEAN in project presto by prestodb.

the class TestPolymorphicScalarFunction method testSelectsMultipleChoiceWithBlockPosition.

@Test
public void testSelectsMultipleChoiceWithBlockPosition() throws Throwable {
    Signature signature = SignatureBuilder.builder().kind(SCALAR).operatorType(IS_DISTINCT_FROM).argumentTypes(DECIMAL_SIGNATURE, DECIMAL_SIGNATURE).returnType(parseTypeSignature(BOOLEAN)).build();
    SqlScalarFunction function = SqlScalarFunction.builder(TestMethods.class, IS_DISTINCT_FROM).signature(signature).deterministic(true).choice(choice -> choice.argumentProperties(valueTypeArgumentProperty(USE_NULL_FLAG), valueTypeArgumentProperty(USE_NULL_FLAG)).implementation(methodsGroup -> methodsGroup.methods("shortShort", "longLong"))).choice(choice -> choice.argumentProperties(valueTypeArgumentProperty(BLOCK_AND_POSITION), valueTypeArgumentProperty(BLOCK_AND_POSITION)).implementation(methodsGroup -> methodsGroup.methodWithExplicitJavaTypes("blockPositionLongLong", asList(Optional.of(Slice.class), Optional.of(Slice.class))).methodWithExplicitJavaTypes("blockPositionShortShort", asList(Optional.of(long.class), Optional.of(long.class))))).build();
    BuiltInScalarFunctionImplementation functionImplementation = function.specialize(SHORT_DECIMAL_BOUND_VARIABLES, 2, FUNCTION_AND_TYPE_MANAGER);
    assertEquals(functionImplementation.getAllChoices().size(), 2);
    assertEquals(functionImplementation.getAllChoices().get(0).getArgumentProperties(), Collections.nCopies(2, valueTypeArgumentProperty(USE_NULL_FLAG)));
    assertEquals(functionImplementation.getAllChoices().get(1).getArgumentProperties(), Collections.nCopies(2, valueTypeArgumentProperty(BLOCK_AND_POSITION)));
    Block block1 = new LongArrayBlock(0, Optional.empty(), new long[0]);
    Block block2 = new LongArrayBlock(0, Optional.empty(), new long[0]);
    assertFalse((boolean) functionImplementation.getAllChoices().get(1).getMethodHandle().invoke(block1, 0, block2, 0));
    functionImplementation = function.specialize(LONG_DECIMAL_BOUND_VARIABLES, 2, FUNCTION_AND_TYPE_MANAGER);
    assertTrue((boolean) functionImplementation.getAllChoices().get(1).getMethodHandle().invoke(block1, 0, block2, 0));
}
Also used : StandardTypes(com.facebook.presto.common.type.StandardTypes) Slice(io.airlift.slice.Slice) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) IS_DISTINCT_FROM(com.facebook.presto.common.function.OperatorType.IS_DISTINCT_FROM) FunctionAndTypeManager.createTestFunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager.createTestFunctionAndTypeManager) TypeSignature(com.facebook.presto.common.type.TypeSignature) MAX_SHORT_PRECISION(com.facebook.presto.common.type.Decimals.MAX_SHORT_PRECISION) ADD(com.facebook.presto.common.function.OperatorType.ADD) ImmutableList(com.google.common.collect.ImmutableList) Arrays.asList(java.util.Arrays.asList) Slices(io.airlift.slice.Slices) Math.toIntExact(java.lang.Math.toIntExact) VARCHAR_TO_VARCHAR_RETURN_VALUE(com.facebook.presto.metadata.TestPolymorphicScalarFunction.TestMethods.VARCHAR_TO_VARCHAR_RETURN_VALUE) Assert.assertFalse(org.testng.Assert.assertFalse) BOOLEAN(com.facebook.presto.common.type.StandardTypes.BOOLEAN) BuiltInScalarFunctionImplementation(com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation) ImmutableSet(com.google.common.collect.ImmutableSet) Signature.comparableWithVariadicBound(com.facebook.presto.spi.function.Signature.comparableWithVariadicBound) ImmutableMap(com.google.common.collect.ImmutableMap) LongArrayBlock(com.facebook.presto.common.block.LongArrayBlock) BLOCK_AND_POSITION(com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.BLOCK_AND_POSITION) USE_NULL_FLAG(com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.NullConvention.USE_NULL_FLAG) SCALAR(com.facebook.presto.spi.function.FunctionKind.SCALAR) VARCHAR(com.facebook.presto.common.type.StandardTypes.VARCHAR) ArgumentProperty.valueTypeArgumentProperty(com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) VARCHAR_TO_BIGINT_RETURN_VALUE(com.facebook.presto.metadata.TestPolymorphicScalarFunction.TestMethods.VARCHAR_TO_BIGINT_RETURN_VALUE) Signature(com.facebook.presto.spi.function.Signature) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) Block(com.facebook.presto.common.block.Block) Collections(java.util.Collections) LongArrayBlock(com.facebook.presto.common.block.LongArrayBlock) BuiltInScalarFunctionImplementation(com.facebook.presto.operator.scalar.BuiltInScalarFunctionImplementation) TypeSignature(com.facebook.presto.common.type.TypeSignature) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) Signature(com.facebook.presto.spi.function.Signature) LongArrayBlock(com.facebook.presto.common.block.LongArrayBlock) Block(com.facebook.presto.common.block.Block) Test(org.testng.annotations.Test)

Aggregations

BOOLEAN (com.facebook.presto.common.type.StandardTypes.BOOLEAN)2 VARCHAR (com.facebook.presto.common.type.StandardTypes.VARCHAR)2 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 Block (com.facebook.presto.common.block.Block)1 LongArrayBlock (com.facebook.presto.common.block.LongArrayBlock)1 MethodHandleUtil.methodHandle (com.facebook.presto.common.block.MethodHandleUtil.methodHandle)1 ADD (com.facebook.presto.common.function.OperatorType.ADD)1 IS_DISTINCT_FROM (com.facebook.presto.common.function.OperatorType.IS_DISTINCT_FROM)1 MAX_SHORT_PRECISION (com.facebook.presto.common.type.Decimals.MAX_SHORT_PRECISION)1 StandardTypes (com.facebook.presto.common.type.StandardTypes)1 TypeSignature (com.facebook.presto.common.type.TypeSignature)1 TypeSignature.parseTypeSignature (com.facebook.presto.common.type.TypeSignature.parseTypeSignature)1 HIVE_FILESYSTEM_ERROR (com.facebook.presto.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR)1 Column (com.facebook.presto.hive.metastore.Column)1 MetastoreContext (com.facebook.presto.hive.metastore.MetastoreContext)1 PRESTO_QUERY_ID_NAME (com.facebook.presto.hive.metastore.MetastoreUtil.PRESTO_QUERY_ID_NAME)1 MetastoreUtil.extractPartitionValues (com.facebook.presto.hive.metastore.MetastoreUtil.extractPartitionValues)1 MetastoreUtil.getMetastoreHeaders (com.facebook.presto.hive.metastore.MetastoreUtil.getMetastoreHeaders)1 MetastoreUtil.isUserDefinedTypeEncodingEnabled (com.facebook.presto.hive.metastore.MetastoreUtil.isUserDefinedTypeEncodingEnabled)1