Search in sources :

Example 1 with Database

use of com.facebook.presto.hive.metastore.Database in project presto by prestodb.

the class FileHiveMetastore method getAllTables.

@Override
public synchronized Optional<List<String>> getAllTables(String databaseName) {
    requireNonNull(databaseName, "databaseName is null");
    Optional<Database> database = getDatabase(databaseName);
    if (!database.isPresent()) {
        return Optional.empty();
    }
    Path databaseMetadataDirectory = getDatabaseMetadataDirectory(databaseName);
    List<String> tables = getChildSchemaDirectories(databaseMetadataDirectory).stream().map(Path::getName).collect(toList());
    return Optional.of(ImmutableList.copyOf(tables));
}
Also used : Path(org.apache.hadoop.fs.Path) Database(com.facebook.presto.hive.metastore.Database)

Example 2 with Database

use of com.facebook.presto.hive.metastore.Database in project presto by prestodb.

the class HiveMetadata method createSchema.

@Override
public void createSchema(ConnectorSession session, String schemaName, Map<String, Object> properties) {
    Optional<String> location = HiveSchemaProperties.getLocation(properties).map(locationUri -> {
        try {
            hdfsEnvironment.getFileSystem(session.getUser(), new Path(locationUri));
        } catch (IOException e) {
            throw new PrestoException(INVALID_SCHEMA_PROPERTY, "Invalid location URI: " + locationUri, e);
        }
        return locationUri;
    });
    Database database = Database.builder().setDatabaseName(schemaName).setLocation(location).setOwnerType(USER).setOwnerName(session.getUser()).build();
    metastore.createDatabase(database);
}
Also used : Path(org.apache.hadoop.fs.Path) Database(com.facebook.presto.hive.metastore.Database) PrestoException(com.facebook.presto.spi.PrestoException) IOException(java.io.IOException)

Example 3 with Database

use of com.facebook.presto.hive.metastore.Database in project presto by prestodb.

the class FileHiveMetastore method isDatabaseOwner.

private boolean isDatabaseOwner(String user, String databaseName) {
    // all users are "owners" of the default database
    if (DEFAULT_DATABASE_NAME.equalsIgnoreCase(databaseName)) {
        return true;
    }
    Optional<Database> databaseMetadata = getDatabase(databaseName);
    if (!databaseMetadata.isPresent()) {
        return false;
    }
    Database database = databaseMetadata.get();
    // a database can be owned by a user or role
    if (database.getOwnerType() == USER && user.equals(database.getOwnerName())) {
        return true;
    }
    if (database.getOwnerType() == ROLE && getRoles(user).contains(database.getOwnerName())) {
        return true;
    }
    return false;
}
Also used : Database(com.facebook.presto.hive.metastore.Database)

Example 4 with Database

use of com.facebook.presto.hive.metastore.Database in project presto by prestodb.

the class AbstractTestHiveClientS3 method doCreateTable.

private void doCreateTable(SchemaTableName tableName, HiveStorageFormat storageFormat) throws Exception {
    List<ColumnMetadata> columns = ImmutableList.<ColumnMetadata>builder().add(new ColumnMetadata("id", BIGINT)).build();
    MaterializedResult data = MaterializedResult.resultBuilder(newSession(), BIGINT).row(1L).row(3L).row(2L).build();
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // begin creating the table
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(tableName, columns, createTableProperties(storageFormat));
        ConnectorOutputTableHandle outputHandle = metadata.beginCreateTable(session, tableMetadata, Optional.empty());
        // write the records
        ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, outputHandle);
        sink.appendPage(data.toPage());
        Collection<Slice> fragments = getFutureValue(sink.finish());
        // commit the table
        metadata.finishCreateTable(session, outputHandle, fragments);
        transaction.commit();
        // Hack to work around the metastore not being configured for S3.
        // The metastore tries to validate the location when creating the
        // table, which fails without explicit configuration for S3.
        // We work around that by using a dummy location when creating the
        // table and update it here to the correct S3 location.
        metastoreClient.updateTableLocation(database, tableName.getTableName(), locationService.writePathRoot(((HiveOutputTableHandle) outputHandle).getLocationHandle()).get().toString());
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        // load the new table
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        // verify the metadata
        ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, getTableHandle(metadata, tableName));
        assertEquals(filterNonHiddenColumnMetadata(tableMetadata.getColumns()), columns);
        // verify the data
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.all(), bindings -> true), Optional.empty());
        HiveTableLayoutHandle layoutHandle = (HiveTableLayoutHandle) getOnlyElement(tableLayoutResults).getTableLayout().getHandle();
        assertEquals(layoutHandle.getPartitions().get().size(), 1);
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, layoutHandle);
        ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
        try (ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, columnHandles)) {
            MaterializedResult result = materializeSourceDataStream(session, pageSource, getTypes(columnHandles));
            assertEqualsIgnoreOrder(result.getMaterializedRows(), data.getMaterializedRows());
        }
    }
}
Also used : HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) FileSystem(org.apache.hadoop.fs.FileSystem) TypeRegistry(com.facebook.presto.type.TypeRegistry) Test(org.testng.annotations.Test) AbstractTestHiveClient.getAllSplits(com.facebook.presto.hive.AbstractTestHiveClient.getAllSplits) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) MaterializedResult.materializeSourceDataStream(com.facebook.presto.testing.MaterializedResult.materializeSourceDataStream) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) Map(java.util.Map) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Path(org.apache.hadoop.fs.Path) ConnectorPageSourceProvider(com.facebook.presto.spi.connector.ConnectorPageSourceProvider) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) AbstractTestHiveClient.filterNonHiddenColumnHandles(com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnHandles) ConnectorSplitManager(com.facebook.presto.spi.connector.ConnectorSplitManager) ImmutableMap(com.google.common.collect.ImmutableMap) PrincipalPrivileges(com.facebook.presto.hive.metastore.PrincipalPrivileges) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) UUID(java.util.UUID) String.format(java.lang.String.format) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) List(java.util.List) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) BridgingHiveMetastore(com.facebook.presto.hive.metastore.BridgingHiveMetastore) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) JsonCodec(io.airlift.json.JsonCodec) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) Table(com.facebook.presto.hive.metastore.Table) Slice(io.airlift.slice.Slice) HiveTransaction(com.facebook.presto.hive.AbstractTestHiveClient.HiveTransaction) Database(com.facebook.presto.hive.metastore.Database) HiveTestUtils.getTypes(com.facebook.presto.hive.HiveTestUtils.getTypes) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) ImmutableList(com.google.common.collect.ImmutableList) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ConnectorPageSinkProvider(com.facebook.presto.spi.connector.ConnectorPageSinkProvider) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) AbstractTestHiveClient.createTableProperties(com.facebook.presto.hive.AbstractTestHiveClient.createTableProperties) ExecutorService(java.util.concurrent.ExecutorService) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata) AfterClass(org.testng.annotations.AfterClass) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) Transaction(com.facebook.presto.hive.AbstractTestHiveClient.Transaction) Throwables(com.google.common.base.Throwables) Constraint(com.facebook.presto.spi.Constraint) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) HostAndPort(com.google.common.net.HostAndPort) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TableNotFoundException(com.facebook.presto.spi.TableNotFoundException) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) MaterializedRow(com.facebook.presto.testing.MaterializedRow) AbstractTestHiveClient.filterNonHiddenColumnMetadata(com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnMetadata) Assert.assertTrue(org.testng.Assert.assertTrue) HadoopFileStatus.isDirectory(com.facebook.presto.hadoop.HadoopFileStatus.isDirectory) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) ThriftHiveMetastore(com.facebook.presto.hive.metastore.ThriftHiveMetastore) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ColumnMetadata(com.facebook.presto.spi.ColumnMetadata) AbstractTestHiveClient.filterNonHiddenColumnMetadata(com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnMetadata) ConnectorSplitSource(com.facebook.presto.spi.ConnectorSplitSource) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) ConnectorOutputTableHandle(com.facebook.presto.spi.ConnectorOutputTableHandle) HiveTransaction(com.facebook.presto.hive.AbstractTestHiveClient.HiveTransaction) Transaction(com.facebook.presto.hive.AbstractTestHiveClient.Transaction) ConnectorTableLayoutResult(com.facebook.presto.spi.ConnectorTableLayoutResult) Slice(io.airlift.slice.Slice) ConnectorSession(com.facebook.presto.spi.ConnectorSession) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) ConnectorMetadata(com.facebook.presto.spi.connector.ConnectorMetadata) MaterializedResult(com.facebook.presto.testing.MaterializedResult) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) ConnectorTableMetadata(com.facebook.presto.spi.ConnectorTableMetadata)

Aggregations

Database (com.facebook.presto.hive.metastore.Database)4 Path (org.apache.hadoop.fs.Path)3 GroupByHashPageIndexerFactory (com.facebook.presto.GroupByHashPageIndexerFactory)1 HadoopFileStatus.isDirectory (com.facebook.presto.hadoop.HadoopFileStatus.isDirectory)1 HiveTransaction (com.facebook.presto.hive.AbstractTestHiveClient.HiveTransaction)1 Transaction (com.facebook.presto.hive.AbstractTestHiveClient.Transaction)1 AbstractTestHiveClient.createTableProperties (com.facebook.presto.hive.AbstractTestHiveClient.createTableProperties)1 AbstractTestHiveClient.filterNonHiddenColumnHandles (com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnHandles)1 AbstractTestHiveClient.filterNonHiddenColumnMetadata (com.facebook.presto.hive.AbstractTestHiveClient.filterNonHiddenColumnMetadata)1 AbstractTestHiveClient.getAllSplits (com.facebook.presto.hive.AbstractTestHiveClient.getAllSplits)1 TYPE_MANAGER (com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER)1 HiveTestUtils.getDefaultHiveDataStreamFactories (com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories)1 HiveTestUtils.getDefaultHiveFileWriterFactories (com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories)1 HiveTestUtils.getDefaultHiveRecordCursorProvider (com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider)1 HiveTestUtils.getTypes (com.facebook.presto.hive.HiveTestUtils.getTypes)1 NoHdfsAuthentication (com.facebook.presto.hive.authentication.NoHdfsAuthentication)1 BridgingHiveMetastore (com.facebook.presto.hive.metastore.BridgingHiveMetastore)1 CachingHiveMetastore (com.facebook.presto.hive.metastore.CachingHiveMetastore)1 ExtendedHiveMetastore (com.facebook.presto.hive.metastore.ExtendedHiveMetastore)1 PrincipalPrivileges (com.facebook.presto.hive.metastore.PrincipalPrivileges)1