Examples with Table io.trino.plugin.hive.metastore.Table used on opensource projects

Example 1 with Table

use of io.trino.plugin.hive.metastore.Table in project trino by trinodb.

the class HiveMetadata method listTables.

private List<SchemaTableName> listTables(ConnectorSession session, SchemaTablePrefix prefix) {
    if (prefix.getSchema().map(HiveUtil::isHiveSystemSchema).orElse(false)) {
        return ImmutableList.of();
    }
    if (prefix.getTable().isEmpty()) {
        return listTables(session, prefix.getSchema());
    }
    SchemaTableName tableName = prefix.toSchemaTableName();
    Optional<Table> optionalTable;
    try {
        optionalTable = metastore.getTable(tableName.getSchemaName(), tableName.getTableName());
    } catch (HiveViewNotSupportedException e) {
        // exists, would be returned by listTables from schema
        return ImmutableList.of(tableName);
    }
    return optionalTable.filter(table -> !hideDeltaLakeTables || !isDeltaLakeTable(table)).map(table -> ImmutableList.of(tableName)).orElseGet(ImmutableList::of);
}

Example 2 with Table

use of io.trino.plugin.hive.metastore.Table in project trino by trinodb.

the class HiveMetadata method createView.

@Override
public void createView(ConnectorSession session, SchemaTableName viewName, ConnectorViewDefinition definition, boolean replace) {
    if (accessControlMetadata.isUsingSystemSecurity()) {
        definition = definition.withoutOwner();
    }
    Map<String, String> properties = ImmutableMap.<String, String>builder().put(TABLE_COMMENT, PRESTO_VIEW_COMMENT).put(PRESTO_VIEW_FLAG, "true").put(TRINO_CREATED_BY, "Trino Hive connector").put(PRESTO_VERSION_NAME, prestoVersion).put(PRESTO_QUERY_ID_NAME, session.getQueryId()).buildOrThrow();
    Column dummyColumn = new Column("dummy", HIVE_STRING, Optional.empty());
    Table.Builder tableBuilder = Table.builder().setDatabaseName(viewName.getSchemaName()).setTableName(viewName.getTableName()).setOwner(accessControlMetadata.isUsingSystemSecurity() ? Optional.empty() : Optional.ofNullable(session.getUser())).setTableType(TableType.VIRTUAL_VIEW.name()).setDataColumns(ImmutableList.of(dummyColumn)).setPartitionColumns(ImmutableList.of()).setParameters(properties).setViewOriginalText(Optional.of(encodeViewData(definition))).setViewExpandedText(Optional.of(PRESTO_VIEW_EXPANDED_TEXT_MARKER));
    tableBuilder.getStorageBuilder().setStorageFormat(VIEW_STORAGE_FORMAT).setLocation("");
    Table table = tableBuilder.build();
    PrincipalPrivileges principalPrivileges = accessControlMetadata.isUsingSystemSecurity() ? NO_PRIVILEGES : buildInitialPrivilegeSet(session.getUser());
    Optional<Table> existing = metastore.getTable(viewName.getSchemaName(), viewName.getTableName());
    if (existing.isPresent()) {
        if (!replace || !isPrestoView(existing.get())) {
            throw new ViewAlreadyExistsException(viewName);
        }
        metastore.replaceTable(viewName.getSchemaName(), viewName.getTableName(), table, principalPrivileges);
        return;
    }
    try {
        metastore.createTable(session, table, principalPrivileges, Optional.empty(), Optional.empty(), false, new PartitionStatistics(createEmptyStatistics(), ImmutableMap.of()), false);
    } catch (TableAlreadyExistsException e) {
        throw new ViewAlreadyExistsException(e.getTableName());
    }
}

Example 3 with Table

use of io.trino.plugin.hive.metastore.Table in project trino by trinodb.

the class HiveMetadata method finishOptimize.

private void finishOptimize(ConnectorSession session, ConnectorTableExecuteHandle tableExecuteHandle, Collection<Slice> fragments, List<Object> splitSourceInfo) {
    // TODO lots of that is copied from finishInsert; rafactoring opportunity
    HiveTableExecuteHandle handle = (HiveTableExecuteHandle) tableExecuteHandle;
    checkArgument(handle.getWriteDeclarationId().isPresent(), "no write declaration id present in tableExecuteHandle");
    List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes).map(partitionUpdateCodec::fromJson).collect(toImmutableList());
    HiveStorageFormat tableStorageFormat = handle.getTableStorageFormat();
    partitionUpdates = PartitionUpdate.mergePartitionUpdates(partitionUpdates);
    Table table = metastore.getTable(handle.getSchemaName(), handle.getTableName()).orElseThrow(() -> new TableNotFoundException(handle.getSchemaTableName()));
    if (!table.getStorage().getStorageFormat().getInputFormat().equals(tableStorageFormat.getInputFormat()) && isRespectTableFormat(session)) {
        throw new TrinoException(HIVE_CONCURRENT_MODIFICATION_DETECTED, "Table format changed during optimize");
    }
    // Support for bucketed tables disabled mostly so we do not need to think about grouped execution in an initial version. Possibly no change apart from testing required.
    verify(handle.getBucketProperty().isEmpty(), "bucketed table not supported");
    for (PartitionUpdate partitionUpdate : partitionUpdates) {
        // sanity check
        verify(partitionUpdate.getUpdateMode() == APPEND, "Expected partionUpdate mode to be APPEND but got %s", partitionUpdate.getUpdateMode());
        if (partitionUpdate.getName().isEmpty()) {
            // operating on an unpartitioned table
            if (!table.getStorage().getStorageFormat().getInputFormat().equals(handle.getPartitionStorageFormat().getInputFormat()) && isRespectTableFormat(session)) {
                throw new TrinoException(HIVE_CONCURRENT_MODIFICATION_DETECTED, "Table format changed during optimize");
            }
            metastore.finishInsertIntoExistingTable(session, handle.getSchemaName(), handle.getTableName(), partitionUpdate.getWritePath(), partitionUpdate.getFileNames(), PartitionStatistics.empty(), handle.isRetriesEnabled());
        } else {
            // operating on a partition
            List<String> partitionValues = toPartitionValues(partitionUpdate.getName());
            metastore.finishInsertIntoExistingPartition(session, handle.getSchemaName(), handle.getTableName(), partitionValues, partitionUpdate.getWritePath(), partitionUpdate.getFileNames(), PartitionStatistics.empty(), handle.isRetriesEnabled());
        }
    }
    // get filesystem
    FileSystem fs;
    try {
        fs = hdfsEnvironment.getFileSystem(new HdfsContext(session), new Path(table.getStorage().getLocation()));
    } catch (IOException e) {
        throw new TrinoException(HIVE_FILESYSTEM_ERROR, e);
    }
    // path to be deleted
    Set<Path> scannedPaths = splitSourceInfo.stream().map(file -> new Path((String) file)).collect(toImmutableSet());
    // track remaining files to be delted for error reporting
    Set<Path> remainingFilesToDelete = new HashSet<>(scannedPaths);
    // delete loop
    boolean someDeleted = false;
    Optional<Path> firstScannedPath = Optional.empty();
    try {
        for (Path scannedPath : scannedPaths) {
            if (firstScannedPath.isEmpty()) {
                firstScannedPath = Optional.of(scannedPath);
            }
            retry().run("delete " + scannedPath, () -> fs.delete(scannedPath, false));
            someDeleted = true;
            remainingFilesToDelete.remove(scannedPath);
        }
    } catch (Exception e) {
        if (!someDeleted && (firstScannedPath.isEmpty() || exists(fs, firstScannedPath.get()))) {
            // fs.delete above could throw exception but file was actually deleted.
            throw new TrinoException(HIVE_FILESYSTEM_ERROR, "Error while deleting original files", e);
        }
        // If we already deleted some original files we disable rollback routine so written files are not deleted.
        // The reported exception message and log entry lists files which need to be cleaned up by user manually.
        // Until table is cleaned up there will duplicate rows present.
        metastore.dropDeclaredIntentionToWrite(handle.getWriteDeclarationId().get());
        String errorMessage = "Error while deleting data files in FINISH phase of OPTIMIZE for table " + table.getTableName() + "; remaining files need to be deleted manually:  " + remainingFilesToDelete;
        log.error(e, "%s", errorMessage);
        throw new TrinoException(HIVE_FILESYSTEM_ERROR, errorMessage, e);
    }
}

Example 4 with Table

use of io.trino.plugin.hive.metastore.Table in project trino by trinodb.

the class HiveMetadata method beginDelete.

@Override
public ConnectorTableHandle beginDelete(ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) {
    HiveTableHandle handle = (HiveTableHandle) tableHandle;
    SchemaTableName tableName = handle.getSchemaTableName();
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    ensureTableSupportsDelete(table);
    if (retryMode != NO_RETRIES) {
        throw new TrinoException(NOT_SUPPORTED, "Deleting from Hive tables is not supported with query retries enabled");
    }
    if (!autoCommit) {
        throw new TrinoException(NOT_SUPPORTED, "Deleting from Hive transactional tables is not supported in explicit transactions (use autocommit mode)");
    }
    if (isSparkBucketedTable(table)) {
        throw new TrinoException(NOT_SUPPORTED, "Deleting from Spark bucketed tables is not supported");
    }
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session, table);
    AcidTransaction transaction = metastore.beginDelete(session, table);
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.declareIntentionToWrite(session, writeInfo.getWriteMode(), writeInfo.getWritePath(), handle.getSchemaTableName());
    return handle.withTransaction(transaction);
}

Example 5 with Table

use of io.trino.plugin.hive.metastore.Table in project trino by trinodb.

the class HiveMetadata method finishDelete.

@Override
public void finishDelete(ConnectorSession session, ConnectorTableHandle tableHandle, Collection<Slice> fragments) {
    HiveTableHandle handle = (HiveTableHandle) tableHandle;
    checkArgument(handle.isAcidDelete(), "handle should be a delete handle, but is %s", handle);
    requireNonNull(fragments, "fragments is null");
    SchemaTableName tableName = handle.getSchemaTableName();
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    ensureTableSupportsDelete(table);
    List<PartitionAndStatementId> partitionAndStatementIds = fragments.stream().map(Slice::getBytes).map(PartitionAndStatementId.CODEC::fromJson).collect(toImmutableList());
    HdfsContext context = new HdfsContext(session);
    for (PartitionAndStatementId ps : partitionAndStatementIds) {
        createOrcAcidVersionFile(context, new Path(ps.getDeleteDeltaDirectory()));
    }
    LocationHandle locationHandle = locationService.forExistingTable(metastore, session, table);
    WriteInfo writeInfo = locationService.getQueryWriteInfo(locationHandle);
    metastore.finishRowLevelDelete(session, table.getDatabaseName(), table.getTableName(), writeInfo.getWritePath(), partitionAndStatementIds);
}