Examples with MetastoreContext com.facebook.presto.hive.metastore.MetastoreContext used on opensource projects

Example 86 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HiveMetadata method getMaterializedViewStatus.

@Override
public MaterializedViewStatus getMaterializedViewStatus(ConnectorSession session, SchemaTableName materializedViewName) {
    MetastoreContext metastoreContext = getMetastoreContext(session);
    ConnectorMaterializedViewDefinition viewDefinition = getMaterializedView(session, materializedViewName).orElseThrow(() -> new MaterializedViewNotFoundException(materializedViewName));
    List<Table> baseTables = viewDefinition.getBaseTables().stream().map(baseTableName -> metastore.getTable(metastoreContext, baseTableName.getSchemaName(), baseTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(baseTableName))).collect(toImmutableList());
    baseTables.forEach(table -> checkState(table.getTableType().equals(MANAGED_TABLE), format("base table %s is not a managed table", table.getTableName())));
    Table materializedViewTable = metastore.getTable(metastoreContext, materializedViewName.getSchemaName(), materializedViewName.getTableName()).orElseThrow(() -> new MaterializedViewNotFoundException(materializedViewName));
    checkState(materializedViewTable.getTableType().equals(MATERIALIZED_VIEW), format("materialized view table %s is not a materialized view", materializedViewTable.getTableName()));
    validateMaterializedViewPartitionColumns(metastore, metastoreContext, materializedViewTable, viewDefinition);
    Map<String, Map<SchemaTableName, String>> directColumnMappings = viewDefinition.getDirectColumnMappingsAsMap();
    Map<SchemaTableName, Map<String, String>> viewToBasePartitionMap = getViewToBasePartitionMap(materializedViewTable, baseTables, directColumnMappings);
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(metastore, metastoreContext, typeManager, materializedViewTable, timeZone);
    if (materializedDataPredicates.getPredicateDisjuncts().isEmpty()) {
        return new MaterializedViewStatus(NOT_MATERIALIZED);
    }
    // Partitions to keep track of for materialized view freshness are the partitions of every base table
    // that are not available/updated to the materialized view yet.
    Map<SchemaTableName, MaterializedDataPredicates> partitionsFromBaseTables = baseTables.stream().collect(toImmutableMap(baseTable -> new SchemaTableName(baseTable.getDatabaseName(), baseTable.getTableName()), baseTable -> {
        MaterializedDataPredicates baseTableMaterializedPredicates = getMaterializedDataPredicates(metastore, metastoreContext, typeManager, baseTable, timeZone);
        SchemaTableName schemaTableName = new SchemaTableName(baseTable.getDatabaseName(), baseTable.getTableName());
        Map<String, String> viewToBaseIndirectMappedColumns = viewToBaseTableOnOuterJoinSideIndirectMappedPartitions(viewDefinition, baseTable).orElse(ImmutableMap.of());
        return differenceDataPredicates(baseTableMaterializedPredicates, materializedDataPredicates, viewToBasePartitionMap.getOrDefault(schemaTableName, ImmutableMap.of()), viewToBaseIndirectMappedColumns);
    }));
    for (MaterializedDataPredicates dataPredicates : partitionsFromBaseTables.values()) {
        if (!dataPredicates.getPredicateDisjuncts().isEmpty()) {
            if (dataPredicates.getPredicateDisjuncts().stream().mapToInt(tupleDomain -> tupleDomain.getDomains().isPresent() ? tupleDomain.getDomains().get().size() : 0).sum() > HiveSessionProperties.getMaterializedViewMissingPartitionsThreshold(session)) {
                return new MaterializedViewStatus(TOO_MANY_PARTITIONS_MISSING, partitionsFromBaseTables);
            }
            return new MaterializedViewStatus(PARTIALLY_MATERIALIZED, partitionsFromBaseTables);
        }
    }
    return new MaterializedViewStatus(FULLY_MATERIALIZED);
}

Example 87 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HiveWriteUtils method getTableDefaultLocation.

public static Path getTableDefaultLocation(ConnectorSession session, SemiTransactionalHiveMetastore metastore, HdfsEnvironment hdfsEnvironment, String schemaName, String tableName) {
    MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider());
    Optional<String> location = getDatabase(session.getIdentity(), metastoreContext, metastore, schemaName).getLocation();
    if (!location.isPresent() || location.get().isEmpty()) {
        throw new PrestoException(HIVE_DATABASE_LOCATION_ERROR, format("Database '%s' location is not set", schemaName));
    }
    HdfsContext context = new HdfsContext(session, schemaName, tableName, location.get(), true);
    Path databasePath = new Path(location.get());
    if (!isS3FileSystem(context, hdfsEnvironment, databasePath)) {
        if (!pathExists(context, hdfsEnvironment, databasePath)) {
            throw new PrestoException(HIVE_DATABASE_LOCATION_ERROR, format("Database '%s' location does not exist: %s", schemaName, databasePath));
        }
        if (!isDirectory(context, hdfsEnvironment, databasePath)) {
            throw new PrestoException(HIVE_DATABASE_LOCATION_ERROR, format("Database '%s' location is not a directory: %s", schemaName, databasePath));
        }
    }
    return new Path(databasePath, tableName);
}

Example 88 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class HivePartitionManager method createPartitionPredicates.

private Map<Column, Domain> createPartitionPredicates(SemiTransactionalHiveMetastore metastore, ConnectorSession session, TupleDomain<ColumnHandle> effectivePredicateColumnHandles, List<HiveColumnHandle> partitionColumns, boolean assumeCanonicalPartitionKeys) {
    Optional<Map<ColumnHandle, Domain>> domains = effectivePredicateColumnHandles.getDomains();
    if (domains.isPresent()) {
        Map<ColumnHandle, Domain> columnHandleDomainMap = domains.get();
        ImmutableMap.Builder<Column, Domain> partitionPredicateBuilder = ImmutableMap.builder();
        MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider());
        for (HiveColumnHandle partitionColumn : partitionColumns) {
            Column key = new Column(partitionColumn.getName(), partitionColumn.getHiveType(), partitionColumn.getComment(), metastoreContext.getColumnConverter().getTypeMetadata(partitionColumn.getHiveType(), partitionColumn.getTypeSignature()));
            if (columnHandleDomainMap.containsKey(partitionColumn)) {
                if (assumeCanonicalPartitionKeys) {
                    partitionPredicateBuilder.put(key, columnHandleDomainMap.get(partitionColumn));
                } else {
                    Type type = typeManager.getType(partitionColumn.getTypeSignature());
                    if (type instanceof VarcharType || type instanceof CharType) {
                        partitionPredicateBuilder.put(key, columnHandleDomainMap.get(partitionColumn));
                    } else {
                        Domain allDomain = Domain.all(typeManager.getType(partitionColumn.getTypeSignature()));
                        partitionPredicateBuilder.put(key, allDomain);
                    }
                }
            } else {
                Domain allDomain = Domain.all(typeManager.getType(partitionColumn.getTypeSignature()));
                partitionPredicateBuilder.put(key, allDomain);
            }
        }
        return partitionPredicateBuilder.build();
    } else {
        return ImmutableMap.of();
    }
}

Example 89 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class SyncPartitionMetadataProcedure method doSyncPartitionMetadata.

private void doSyncPartitionMetadata(ConnectorSession session, String schemaName, String tableName, String mode, boolean caseSensitive) {
    SyncMode syncMode = toSyncMode(mode);
    SemiTransactionalHiveMetastore metastore = hiveMetadataFactory.get().getMetastore();
    SchemaTableName schemaTableName = new SchemaTableName(schemaName, tableName);
    Table table = metastore.getTable(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), schemaName, tableName).orElseThrow(() -> new TableNotFoundException(schemaTableName));
    if (table.getPartitionColumns().isEmpty()) {
        throw new PrestoException(INVALID_PROCEDURE_ARGUMENT, "Table is not partitioned: " + schemaTableName);
    }
    Path tableLocation = new Path(table.getStorage().getLocation());
    HdfsContext context = new HdfsContext(session, schemaName, tableName, table.getStorage().getLocation(), false);
    Set<String> partitionsToAdd;
    Set<String> partitionsToDrop;
    try {
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, tableLocation);
        List<String> partitionsInMetastore = metastore.getPartitionNames(new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider()), schemaName, tableName).orElseThrow(() -> new TableNotFoundException(schemaTableName));
        List<String> partitionsInFileSystem = listDirectory(fileSystem, fileSystem.getFileStatus(tableLocation), table.getPartitionColumns(), table.getPartitionColumns().size(), caseSensitive).stream().map(fileStatus -> fileStatus.getPath().toUri()).map(uri -> tableLocation.toUri().relativize(uri).getPath()).collect(toImmutableList());
        // partitions in file system but not in metastore
        partitionsToAdd = difference(partitionsInFileSystem, partitionsInMetastore);
        // partitions in metastore but not in file system
        partitionsToDrop = difference(partitionsInMetastore, partitionsInFileSystem);
    } catch (IOException e) {
        throw new PrestoException(HIVE_FILESYSTEM_ERROR, e);
    }
    syncPartitions(partitionsToAdd, partitionsToDrop, syncMode, metastore, session, table);
}

Example 90 with MetastoreContext

use of com.facebook.presto.hive.metastore.MetastoreContext in project presto by prestodb.

the class AbstractTestHiveClient method createEmptyTable.

protected Table createEmptyTable(SchemaTableName schemaTableName, HiveStorageFormat hiveStorageFormat, List<Column> columns, List<Column> partitionColumns, Optional<HiveBucketProperty> bucketProperty, Map<String, String> parameters) throws Exception {
    Path targetPath;
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        String tableOwner = session.getUser();
        String schemaName = schemaTableName.getSchemaName();
        String tableName = schemaTableName.getTableName();
        LocationService locationService = getLocationService();
        LocationHandle locationHandle = locationService.forNewTable(transaction.getMetastore(), session, schemaName, tableName, false);
        targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
        Table.Builder tableBuilder = Table.builder().setDatabaseName(schemaName).setTableName(tableName).setOwner(tableOwner).setTableType(MANAGED_TABLE).setParameters(ImmutableMap.<String, String>builder().put(PRESTO_VERSION_NAME, TEST_SERVER_VERSION).put(PRESTO_QUERY_ID_NAME, session.getQueryId()).putAll(parameters).build()).setDataColumns(columns).setPartitionColumns(partitionColumns);
        tableBuilder.getStorageBuilder().setLocation(targetPath.toString()).setStorageFormat(StorageFormat.create(hiveStorageFormat.getSerDe(), hiveStorageFormat.getInputFormat(), hiveStorageFormat.getOutputFormat())).setBucketProperty(bucketProperty).setSerdeParameters(ImmutableMap.of());
        PrincipalPrivileges principalPrivileges = testingPrincipalPrivilege(tableOwner, session.getUser());
        transaction.getMetastore().createTable(session, tableBuilder.build(), principalPrivileges, Optional.empty(), true, EMPTY_TABLE_STATISTICS);
        transaction.commit();
    }
    HdfsContext context = new HdfsContext(newSession(), schemaTableName.getSchemaName(), schemaTableName.getTableName(), targetPath.toString(), false);
    List<String> targetDirectoryList = listDirectory(context, targetPath);
    assertEquals(targetDirectoryList, ImmutableList.of());
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), false, DEFAULT_COLUMN_CONVERTER_PROVIDER);
        return transaction.getMetastore().getTable(metastoreContext, schemaTableName.getSchemaName(), schemaTableName.getTableName()).get();
    }
}