Examples with PrincipalPrivileges io.prestosql.plugin.hive.metastore.PrincipalPrivileges used on opensource projects

Example 1 with PrincipalPrivileges

use of io.prestosql.plugin.hive.metastore.PrincipalPrivileges in project hetu-core by openlookeng.

the class CarbondataMetadata method createTable.

@Override
public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, boolean ignoreExisting) {
    SchemaTableName localSchemaTableName = tableMetadata.getTable();
    String localSchemaName = localSchemaTableName.getSchemaName();
    String tableName = localSchemaTableName.getTableName();
    this.user = session.getUser();
    this.schemaName = localSchemaName;
    currentState = State.CREATE_TABLE;
    List<String> partitionedBy = new ArrayList<String>();
    List<SortingColumn> sortBy = new ArrayList<SortingColumn>();
    List<HiveColumnHandle> columnHandles = new ArrayList<HiveColumnHandle>();
    Map<String, String> tableProperties = new HashMap<String, String>();
    getParametersForCreateTable(session, tableMetadata, partitionedBy, sortBy, columnHandles, tableProperties);
    metastore.getDatabase(localSchemaName).orElseThrow(() -> new SchemaNotFoundException(localSchemaName));
    BaseStorageFormat hiveStorageFormat = CarbondataTableProperties.getCarbondataStorageFormat(tableMetadata.getProperties());
    // it will get final path to create carbon table
    LocationHandle locationHandle = getCarbonDataTableCreationPath(session, tableMetadata, HiveWriteUtils.OpertionType.CREATE_TABLE);
    Path targetPath = locationService.getQueryWriteInfo(locationHandle).getTargetPath();
    AbsoluteTableIdentifier finalAbsoluteTableIdentifier = AbsoluteTableIdentifier.from(targetPath.toString(), new CarbonTableIdentifier(localSchemaName, tableName, UUID.randomUUID().toString()));
    hdfsEnvironment.doAs(session.getUser(), () -> {
        initialConfiguration = ConfigurationUtils.toJobConf(this.hdfsEnvironment.getConfiguration(new HdfsEnvironment.HdfsContext(session, localSchemaName, tableName), new Path(locationHandle.getJsonSerializableTargetPath())));
        CarbondataMetadataUtils.createMetaDataFolderSchemaFile(hdfsEnvironment, session, columnHandles, finalAbsoluteTableIdentifier, partitionedBy, sortBy.stream().map(s -> s.getColumnName().toLowerCase(Locale.ENGLISH)).collect(toList()), targetPath.toString(), initialConfiguration);
        this.tableStorageLocation = Optional.of(targetPath.toString());
        try {
            Map<String, String> serdeParameters = initSerDeProperties(tableName);
            Table localTable = buildTableObject(session.getQueryId(), localSchemaName, tableName, session.getUser(), columnHandles, hiveStorageFormat, partitionedBy, Optional.empty(), tableProperties, targetPath, // carbon table is set as external table
            true, prestoVersion, serdeParameters);
            PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(localTable.getOwner());
            HiveBasicStatistics basicStatistics = localTable.getPartitionColumns().isEmpty() ? HiveBasicStatistics.createZeroStatistics() : HiveBasicStatistics.createEmptyStatistics();
            metastore.createTable(session, localTable, principalPrivileges, Optional.empty(), ignoreExisting, new PartitionStatistics(basicStatistics, ImmutableMap.of()));
        } catch (RuntimeException ex) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("Error: creating table: %s ", ex.getMessage()), ex);
        }
    });
}

Example 2 with PrincipalPrivileges

use of io.prestosql.plugin.hive.metastore.PrincipalPrivileges in project hetu-core by openlookeng.

the class AbstractTestHive method alterBucketProperty.

private void alterBucketProperty(SchemaTableName schemaTableName, Optional<HiveBucketProperty> bucketProperty) {
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        String tableOwner = session.getUser();
        String schemaName = schemaTableName.getSchemaName();
        String tableName = schemaTableName.getTableName();
        Optional<Table> table = transaction.getMetastore(schemaName).getTable(new HiveIdentity(session), schemaName, tableName);
        Table.Builder tableBuilder = Table.builder(table.get());
        tableBuilder.getStorageBuilder().setBucketProperty(bucketProperty);
        PrincipalPrivileges principalPrivileges = testingPrincipalPrivilege(tableOwner, session.getUser());
        // hack: replaceView can be used as replaceTable despite its name
        transaction.getMetastore(schemaName).replaceView(new HiveIdentity(session), schemaName, tableName, tableBuilder.build(), principalPrivileges);
        transaction.commit();
    }
}

Example 3 with PrincipalPrivileges

use of io.prestosql.plugin.hive.metastore.PrincipalPrivileges in project hetu-core by openlookeng.

the class HiveMetadata method createView.

@Override
public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, boolean replace) {
    HiveIdentity identity = new HiveIdentity(session);
    Map<String, String> properties = ImmutableMap.<String, String>builder().put(TABLE_COMMENT, "Presto View").put(PRESTO_VIEW_FLAG, "true").put(PRESTO_VERSION_NAME, prestoVersion).put(PRESTO_QUERY_ID_NAME, session.getQueryId()).build();
    Column dummyColumn = new Column("dummy", HiveType.HIVE_STRING, Optional.empty());
    Table.Builder tableBuilder = Table.builder().setDatabaseName(viewName.getSchemaName()).setTableName(viewName.getTableName()).setOwner(session.getUser()).setTableType(TableType.VIRTUAL_VIEW.name()).setDataColumns(ImmutableList.of(dummyColumn)).setPartitionColumns(ImmutableList.of()).setParameters(properties).setViewOriginalText(Optional.of(encodeViewData(definition))).setViewExpandedText(Optional.of("/* Presto View */"));
    tableBuilder.getStorageBuilder().setStorageFormat(StorageFormat.VIEW_STORAGE_FORMAT).setLocation("");
    Table table = tableBuilder.build();
    PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(session.getUser());
    Optional<Table> existing = metastore.getTable(identity, viewName.getSchemaName(), viewName.getTableName());
    if (existing.isPresent()) {
        if (!replace || !HiveUtil.isPrestoView(existing.get())) {
            throw new ViewAlreadyExistsException(viewName);
        }
        metastore.replaceView(identity, viewName.getSchemaName(), viewName.getTableName(), table, principalPrivileges);
        return;
    }
    try {
        metastore.createTable(session, table, principalPrivileges, Optional.empty(), false, new PartitionStatistics(HiveBasicStatistics.createEmptyStatistics(), ImmutableMap.of()));
    } catch (TableAlreadyExistsException e) {
        throw new ViewAlreadyExistsException(e.getTableName());
    }
}

Example 4 with PrincipalPrivileges

use of io.prestosql.plugin.hive.metastore.PrincipalPrivileges in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHive method testTableCreationIgnoreExisting.

@Test
public void testTableCreationIgnoreExisting() {
    List<Column> columns = ImmutableList.of(new Column("dummy", HiveType.valueOf("uniontype<smallint,tinyint>"), Optional.empty()));
    SchemaTableName schemaTableName = temporaryTable("create");
    ConnectorSession session = newSession();
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    PrincipalPrivileges privileges = testingPrincipalPrivilege(session);
    Path targetPath;
    try {
        try (Transaction transaction = newTransaction()) {
            LocationService service = getLocationService();
            LocationHandle locationHandle = service.forNewTable(transaction.getMetastore(schemaName), session, schemaName, tableName, Optional.empty(), Optional.empty(), HiveWriteUtils.OpertionType.CREATE_TABLE);
            targetPath = service.getQueryWriteInfo(locationHandle).getTargetPath();
            Table table = createSimpleTable(schemaTableName, columns, session, targetPath, "q1");
            transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), false, EMPTY_TABLE_STATISTICS);
            Optional<Table> tableHandle = transaction.getMetastore(schemaName).getTable(new HiveIdentity(session), schemaName, tableName);
            assertTrue(tableHandle.isPresent());
            transaction.commit();
        }
        // try creating it again from another transaction with ignoreExisting=false
        try (Transaction transaction = newTransaction()) {
            Table table = createSimpleTable(schemaTableName, columns, session, targetPath.suffix("_2"), "q2");
            transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), false, EMPTY_TABLE_STATISTICS);
            transaction.commit();
            fail("Expected exception");
        } catch (PrestoException e) {
            assertInstanceOf(e, TableAlreadyExistsException.class);
        }
        // try creating it again from another transaction with ignoreExisting=true
        try (Transaction transaction = newTransaction()) {
            Table table = createSimpleTable(schemaTableName, columns, session, targetPath.suffix("_3"), "q3");
            transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), true, EMPTY_TABLE_STATISTICS);
            transaction.commit();
        }
        // at this point the table should exist, now try creating the table again with a different table definition
        columns = ImmutableList.of(new Column("new_column", HiveType.valueOf("string"), Optional.empty()));
        try (Transaction transaction = newTransaction()) {
            Table table = createSimpleTable(schemaTableName, columns, session, targetPath.suffix("_4"), "q4");
            transaction.getMetastore(schemaName).createTable(session, table, privileges, Optional.empty(), true, EMPTY_TABLE_STATISTICS);
            transaction.commit();
            fail("Expected exception");
        } catch (PrestoException e) {
            assertEquals(e.getErrorCode(), TRANSACTION_CONFLICT.toErrorCode());
            assertEquals(e.getMessage(), format("Table already exists with a different schema: '%s'", schemaTableName.getTableName()));
        }
    } finally {
        dropTable(schemaTableName);
    }
}

Example 5 with PrincipalPrivileges

use of io.prestosql.plugin.hive.metastore.PrincipalPrivileges in project boostkit-bigdata by kunpengcompute.

the class HiveMetadata method finishCreateTable.

public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection<Slice> fragments, Collection<ComputedStatistics> computedStatistics, Map<String, String> serdeParameters) {
    HiveOutputTableHandle handle = (HiveOutputTableHandle) tableHandle;
    List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes).map(partitionUpdateCodec::fromJson).collect(toList());
    LocationService.WriteInfo writeInfo = locationService.getQueryWriteInfo(handle.getLocationHandle());
    Table table = buildTableObject(session.getQueryId(), handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), handle.getInputColumns(), handle.getTableStorageFormat(), handle.getPartitionedBy(), handle.getBucketProperty(), handle.getAdditionalTableParameters(), writeInfo.getTargetPath(), externalTable, prestoVersion, serdeParameters);
    PrincipalPrivileges principalPrivileges = MetastoreUtil.buildInitialPrivilegeSet(handle.getTableOwner());
    partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
    if (session.isSnapshotEnabled()) {
        Set<String> mergedFileNames = collectMergedFileNames(partitionUpdates);
        updateSnapshotFiles(session, handle, false, mergedFileNames, OptionalLong.empty());
        // Remove suffix from file names in partition updates
        partitionUpdates = updateSnapshotFileNames(partitionUpdates, session.getQueryId());
    }
    if (handle.getBucketProperty().isPresent() && HiveSessionProperties.isCreateEmptyBucketFiles(session)) {
        List<PartitionUpdate> partitionUpdatesForMissingBuckets = computePartitionUpdatesForMissingBuckets(session, handle, table, partitionUpdates);
        // replace partitionUpdates before creating the empty files so that those files will be cleaned up if we end up rollback
        partitionUpdates = PartitionUpdate.mergePartitionUpdates(concat(partitionUpdates, partitionUpdatesForMissingBuckets));
        for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) {
            Optional<Partition> partition = table.getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(buildPartitionObject(session, table, partitionUpdate));
            createEmptyFiles(session, partitionUpdate.getWritePath(), table, partition, partitionUpdate.getFileNames());
        }
    }
    Map<String, Type> columnTypes = handle.getInputColumns().stream().collect(toImmutableMap(HiveColumnHandle::getName, column -> column.getHiveType().getType(typeManager)));
    Map<List<String>, ComputedStatistics> partitionComputedStatistics = Statistics.createComputedStatisticsToPartitionMap(computedStatistics, handle.getPartitionedBy(), columnTypes);
    PartitionStatistics tableStatistics;
    if (table.getPartitionColumns().isEmpty()) {
        HiveBasicStatistics basicStatistics = partitionUpdates.stream().map(PartitionUpdate::getStatistics).reduce((first, second) -> Statistics.reduce(first, second, Statistics.ReduceOperator.ADD)).orElse(HiveBasicStatistics.createZeroStatistics());
        tableStatistics = createPartitionStatistics(session, basicStatistics, columnTypes, getColumnStatistics(partitionComputedStatistics, ImmutableList.of()));
    } else {
        tableStatistics = new PartitionStatistics(HiveBasicStatistics.createEmptyStatistics(), ImmutableMap.of());
    }
    metastore.createTable(session, table, principalPrivileges, Optional.of(writeInfo.getWritePath()), false, tableStatistics);
    if (!handle.getPartitionedBy().isEmpty()) {
        if (HiveSessionProperties.isRespectTableFormat(session)) {
            verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat());
        }
        List<? extends Future<?>> futures = partitionUpdates.stream().map(update -> hiveMetastoreClientService.submit(() -> {
            Partition partition = buildPartitionObject(session, table, update);
            PartitionStatistics partitionStatistics = createPartitionStatistics(session, update.getStatistics(), columnTypes, getColumnStatistics(partitionComputedStatistics, partition.getValues()));
            metastore.addPartition(session, handle.getSchemaName(), handle.getTableName(), buildPartitionObject(session, table, update), update.getWritePath(), partitionStatistics, HiveACIDWriteType.NONE);
        })).collect(toList());
        futures.forEach(future -> {
            try {
                future.get();
            } catch (InterruptedException | ExecutionException ignore) {
                log.debug("Get future error");
            }
        });
    }
    return Optional.of(new HiveWrittenPartitions(partitionUpdates.stream().map(PartitionUpdate::getName).collect(toList())));
}