use of com.facebook.presto.hive.metastore.Table in project presto by prestodb.
the class HiveMetadata method buildTableObject.
private static Table buildTableObject(String queryId, String schemaName, String tableName, String tableOwner, List<HiveColumnHandle> columnHandles, HiveStorageFormat hiveStorageFormat, List<String> partitionedBy, Optional<HiveBucketProperty> bucketProperty, Map<String, String> additionalTableParameters, Path targetPath, boolean external, String prestoVersion) {
Map<String, HiveColumnHandle> columnHandlesByName = Maps.uniqueIndex(columnHandles, HiveColumnHandle::getName);
List<Column> partitionColumns = partitionedBy.stream().map(columnHandlesByName::get).map(column -> new Column(column.getName(), column.getHiveType(), column.getComment())).collect(toList());
Set<String> partitionColumnNames = ImmutableSet.copyOf(partitionedBy);
ImmutableList.Builder<Column> columns = ImmutableList.builder();
for (HiveColumnHandle columnHandle : columnHandles) {
String name = columnHandle.getName();
HiveType type = columnHandle.getHiveType();
if (!partitionColumnNames.contains(name)) {
verify(!columnHandle.isPartitionKey(), "Column handles are not consistent with partitioned by property");
columns.add(new Column(name, type, columnHandle.getComment()));
} else {
verify(columnHandle.isPartitionKey(), "Column handles are not consistent with partitioned by property");
}
}
ImmutableMap.Builder<String, String> tableParameters = ImmutableMap.<String, String>builder().put("comment", "Created by Presto").put(PRESTO_VERSION_NAME, prestoVersion).put(PRESTO_QUERY_ID_NAME, queryId).putAll(additionalTableParameters);
if (external) {
tableParameters.put("EXTERNAL", "TRUE");
}
Table.Builder tableBuilder = Table.builder().setDatabaseName(schemaName).setTableName(tableName).setOwner(tableOwner).setTableType((external ? EXTERNAL_TABLE : MANAGED_TABLE).name()).setDataColumns(columns.build()).setPartitionColumns(partitionColumns).setParameters(tableParameters.build());
tableBuilder.getStorageBuilder().setStorageFormat(fromHiveStorageFormat(hiveStorageFormat)).setBucketProperty(bucketProperty).setLocation(targetPath.toString());
return tableBuilder.build();
}
use of com.facebook.presto.hive.metastore.Table in project presto by prestodb.
the class HiveMetadata method finishCreateTable.
@Override
public Optional<ConnectorOutputMetadata> finishCreateTable(ConnectorSession session, ConnectorOutputTableHandle tableHandle, Collection<Slice> fragments) {
HiveOutputTableHandle handle = (HiveOutputTableHandle) tableHandle;
List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes).map(partitionUpdateCodec::fromJson).collect(toList());
Path targetPath = locationService.targetPathRoot(handle.getLocationHandle());
Path writePath = locationService.writePathRoot(handle.getLocationHandle()).get();
Table table = buildTableObject(session.getQueryId(), handle.getSchemaName(), handle.getTableName(), handle.getTableOwner(), handle.getInputColumns(), handle.getTableStorageFormat(), handle.getPartitionedBy(), handle.getBucketProperty(), handle.getAdditionalTableParameters(), targetPath, false, prestoVersion);
PrincipalPrivileges principalPrivileges = buildInitialPrivilegeSet(handle.getTableOwner());
partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
if (handle.getBucketProperty().isPresent()) {
ImmutableList<PartitionUpdate> partitionUpdatesForMissingBuckets = computePartitionUpdatesForMissingBuckets(handle, table, partitionUpdates);
// replace partitionUpdates before creating the empty files so that those files will be cleaned up if we end up rollback
partitionUpdates = PartitionUpdate.mergePartitionUpdates(Iterables.concat(partitionUpdates, partitionUpdatesForMissingBuckets));
for (PartitionUpdate partitionUpdate : partitionUpdatesForMissingBuckets) {
Optional<Partition> partition = table.getPartitionColumns().isEmpty() ? Optional.empty() : Optional.of(buildPartitionObject(session.getQueryId(), table, partitionUpdate));
createEmptyFile(partitionUpdate.getWritePath(), table, partition, partitionUpdate.getFileNames());
}
}
metastore.createTable(session, table, principalPrivileges, Optional.of(writePath));
if (!handle.getPartitionedBy().isEmpty()) {
if (respectTableFormat) {
Verify.verify(handle.getPartitionStorageFormat() == handle.getTableStorageFormat());
}
partitionUpdates.forEach(partitionUpdate -> metastore.addPartition(session, handle.getSchemaName(), handle.getTableName(), buildPartitionObject(session.getQueryId(), table, partitionUpdate), partitionUpdate.getWritePath()));
}
return Optional.of(new HiveWrittenPartitions(partitionUpdates.stream().map(PartitionUpdate::getName).collect(Collectors.toList())));
}
use of com.facebook.presto.hive.metastore.Table in project presto by prestodb.
the class HiveMetadata method getTableMetadata.
private ConnectorTableMetadata getTableMetadata(SchemaTableName tableName) {
Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
if (!table.isPresent() || table.get().getTableType().equals(TableType.VIRTUAL_VIEW.name())) {
throw new TableNotFoundException(tableName);
}
Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table.get(), typeManager);
ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
for (HiveColumnHandle columnHandle : hiveColumnHandles(connectorId, table.get())) {
columns.add(metadataGetter.apply(columnHandle));
}
ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
if (table.get().getTableType().equals(EXTERNAL_TABLE.name())) {
properties.put(EXTERNAL_LOCATION_PROPERTY, table.get().getStorage().getLocation());
}
try {
HiveStorageFormat format = extractHiveStorageFormat(table.get());
properties.put(STORAGE_FORMAT_PROPERTY, format);
} catch (PrestoException ignored) {
// todo fail if format is not known
}
List<String> partitionedBy = table.get().getPartitionColumns().stream().map(Column::getName).collect(toList());
if (!partitionedBy.isEmpty()) {
properties.put(PARTITIONED_BY_PROPERTY, partitionedBy);
}
Optional<HiveBucketProperty> bucketProperty = table.get().getStorage().getBucketProperty();
if (bucketProperty.isPresent()) {
properties.put(BUCKET_COUNT_PROPERTY, bucketProperty.get().getBucketCount());
properties.put(BUCKETED_BY_PROPERTY, bucketProperty.get().getBucketedBy());
}
properties.putAll(tableParameterCodec.decode(table.get().getParameters()));
return new ConnectorTableMetadata(tableName, columns.build(), properties.build());
}
use of com.facebook.presto.hive.metastore.Table in project presto by prestodb.
the class HiveMetadata method beginInsert.
@Override
public HiveInsertTableHandle beginInsert(ConnectorSession session, ConnectorTableHandle tableHandle) {
verifyJvmTimeZone();
SchemaTableName tableName = schemaTableName(tableHandle);
Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
if (!table.isPresent()) {
throw new TableNotFoundException(tableName);
}
checkTableIsWritable(table.get());
for (Column column : table.get().getDataColumns()) {
if (!isWritableType(column.getType())) {
throw new PrestoException(NOT_SUPPORTED, format("Inserting into Hive table %s.%s with column type %s not supported", table.get().getDatabaseName(), table.get().getTableName(), column.getType()));
}
}
List<HiveColumnHandle> handles = hiveColumnHandles(connectorId, table.get()).stream().filter(columnHandle -> !columnHandle.isHidden()).collect(toList());
HiveStorageFormat tableStorageFormat = extractHiveStorageFormat(table.get());
LocationHandle locationHandle = locationService.forExistingTable(metastore, session.getUser(), session.getQueryId(), table.get());
HiveInsertTableHandle result = new HiveInsertTableHandle(connectorId, tableName.getSchemaName(), tableName.getTableName(), handles, session.getQueryId(), metastore.generatePageSinkMetadata(tableName), locationHandle, table.get().getStorage().getBucketProperty(), tableStorageFormat, respectTableFormat ? tableStorageFormat : defaultStorageFormat);
Optional<Path> writePathRoot = locationService.writePathRoot(locationHandle);
Path targetPathRoot = locationService.targetPathRoot(locationHandle);
if (writePathRoot.isPresent()) {
WriteMode mode = writePathRoot.get().equals(targetPathRoot) ? DIRECT_TO_TARGET_NEW_DIRECTORY : STAGE_AND_MOVE_TO_TARGET_DIRECTORY;
metastore.declareIntentionToWrite(session, mode, writePathRoot.get(), result.getFilePrefix(), tableName);
} else {
metastore.declareIntentionToWrite(session, DIRECT_TO_TARGET_EXISTING_DIRECTORY, targetPathRoot, result.getFilePrefix(), tableName);
}
return result;
}
use of com.facebook.presto.hive.metastore.Table in project presto by prestodb.
the class HiveMetadata method getColumnHandles.
@Override
public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) {
SchemaTableName tableName = schemaTableName(tableHandle);
Optional<Table> table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
if (!table.isPresent()) {
throw new TableNotFoundException(tableName);
}
ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
for (HiveColumnHandle columnHandle : hiveColumnHandles(connectorId, table.get())) {
columnHandles.put(columnHandle.getName(), columnHandle);
}
return columnHandles.build();
}
Aggregations