Examples with SchemaTableName com.facebook.presto.spi.SchemaTableName used on opensource projects

Example 81 with SchemaTableName

use of com.facebook.presto.spi.SchemaTableName in project presto by prestodb.

the class SemiTransactionalHiveMetastore method commitShared.

@GuardedBy("this")
private void commitShared() {
    checkHoldsLock();
    Committer committer = new Committer();
    try {
        for (Map.Entry<SchemaTableName, Action<TableAndMore>> entry : tableActions.entrySet()) {
            SchemaTableName schemaTableName = entry.getKey();
            Action<TableAndMore> action = entry.getValue();
            HdfsContext hdfsContext = action.getContext();
            MetastoreContext metastoreContext = new MetastoreContext(hdfsContext.getIdentity(), hdfsContext.getQueryId().orElse(""), hdfsContext.getClientInfo(), hdfsContext.getSource(), hdfsContext.getSession().flatMap(MetastoreUtil::getMetastoreHeaders), hdfsContext.getSession().map(MetastoreUtil::isUserDefinedTypeEncodingEnabled).orElse(false), columnConverterProvider);
            switch(action.getType()) {
                case DROP:
                    committer.prepareDropTable(metastoreContext, schemaTableName);
                    break;
                case ALTER:
                    committer.prepareAlterTable();
                    break;
                case ADD:
                    committer.prepareAddTable(metastoreContext, hdfsContext, action.getData());
                    break;
                case INSERT_EXISTING:
                    committer.prepareInsertExistingTable(metastoreContext, hdfsContext, action.getData());
                    break;
                default:
                    throw new IllegalStateException("Unknown action type");
            }
        }
        for (Map.Entry<SchemaTableName, Map<List<String>, Action<PartitionAndMore>>> tableEntry : partitionActions.entrySet()) {
            SchemaTableName schemaTableName = tableEntry.getKey();
            for (Map.Entry<List<String>, Action<PartitionAndMore>> partitionEntry : tableEntry.getValue().entrySet()) {
                List<String> partitionValues = partitionEntry.getKey();
                Action<PartitionAndMore> action = partitionEntry.getValue();
                HdfsContext hdfsContext = action.getContext();
                MetastoreContext metastoreContext = new MetastoreContext(hdfsContext.getIdentity(), hdfsContext.getQueryId().orElse(""), hdfsContext.getClientInfo(), hdfsContext.getSource(), hdfsContext.getSession().flatMap(MetastoreUtil::getMetastoreHeaders), hdfsContext.getSession().map(MetastoreUtil::isUserDefinedTypeEncodingEnabled).orElse(false), columnConverterProvider);
                switch(action.getType()) {
                    case DROP:
                        committer.prepareDropPartition(metastoreContext, schemaTableName, partitionValues);
                        break;
                    case ALTER:
                        committer.prepareAlterPartition(metastoreContext, hdfsContext, action.getData());
                        break;
                    case ADD:
                        committer.prepareAddPartition(metastoreContext, hdfsContext, action.getData());
                        break;
                    case INSERT_EXISTING:
                        committer.prepareInsertExistingPartition(metastoreContext, hdfsContext, action.getData());
                        break;
                    default:
                        throw new IllegalStateException("Unknown action type");
                }
            }
        }
        // Wait for all renames submitted for "INSERT_EXISTING" action to finish
        ListenableFuture<?> listenableFutureAggregate = whenAllSucceed(committer.getFileRenameFutures()).call(() -> null, directExecutor());
        try {
            getFutureValue(listenableFutureAggregate, PrestoException.class);
        } catch (RuntimeException e) {
            listenableFutureAggregate.cancel(true);
            throw e;
        }
        // At this point, all file system operations, whether asynchronously issued or not, have completed successfully.
        // We are moving on to metastore operations now.
        committer.executeAddTableOperations();
        committer.executeAlterPartitionOperations();
        committer.executeAddPartitionOperations();
        committer.executeUpdateStatisticsOperations();
    } catch (Throwable t) {
        committer.cancelUnstartedAsyncRenames();
        committer.undoUpdateStatisticsOperations();
        committer.undoAddPartitionOperations();
        committer.undoAddTableOperations();
        committer.waitForAsyncRenamesSuppressThrowables();
        // fileRenameFutures must all come back before any file system cleanups are carried out.
        // Otherwise, files that should be deleted may be created after cleanup is done.
        committer.executeCleanupTasksForAbort(declaredIntentionsToWrite);
        committer.executeRenameTasksForAbort();
        // Partition directory must be put back before relevant metastore operation can be undone
        committer.undoAlterPartitionOperations();
        rollbackShared();
        throw t;
    }
    try {
        if (!committer.metastoreDeleteOperations.isEmpty()) {
            committer.executeMetastoreDeleteOperations();
        }
    // If control flow reached this point, this commit is considered successful no matter
    // what happens later. The only kind of operations that haven't been carried out yet
    // are cleanups.
    // The program control flow will go to finally next. And cleanup will run because
    // moveForwardInFinally has been set to false.
    } finally {
        // In this method, all operations are best-effort clean up operations.
        // If any operation fails, the error will be logged and ignored.
        // Additionally, other clean up operations should still be attempted.
        // Execute deletion tasks
        committer.executeDeletionTasksForFinish();
        // Clean up temporary tables
        deleteTemporaryTableDirectories(declaredIntentionsToWrite, hdfsEnvironment);
        // Clean up empty staging directories (that may recursively contain empty directories)
        committer.deleteEmptyStagingDirectories(declaredIntentionsToWrite);
        // Clean up root temp directories
        deleteTempPathRootDirectory(declaredIntentionsToWrite, hdfsEnvironment);
    }
}

Example 82 with SchemaTableName

use of com.facebook.presto.spi.SchemaTableName in project presto by prestodb.

the class SemiTransactionalHiveMetastore method getTable.

public synchronized Optional<Table> getTable(MetastoreContext metastoreContext, String databaseName, String tableName) {
    checkReadable();
    Action<TableAndMore> tableAction = tableActions.get(new SchemaTableName(databaseName, tableName));
    if (tableAction == null) {
        return delegate.getTable(metastoreContext, databaseName, tableName);
    }
    switch(tableAction.getType()) {
        case ADD:
        case ALTER:
        case INSERT_EXISTING:
            return Optional.of(tableAction.getData().getAugmentedTableForInTransactionRead());
        case DROP:
            return Optional.empty();
        default:
            throw new IllegalStateException("Unknown action type");
    }
}

Example 83 with SchemaTableName

use of com.facebook.presto.spi.SchemaTableName in project presto by prestodb.

the class SemiTransactionalHiveMetastore method setPartitionLeases.

public synchronized void setPartitionLeases(MetastoreContext metastoreContext, String databaseName, String tableName, Map<String, String> partitionNameToLocation, Duration leaseDuration) {
    checkReadable();
    Table table = getTable(metastoreContext, databaseName, tableName).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(databaseName, tableName)));
    boolean isPartitioned = table.getPartitionColumns() != null && !table.getPartitionColumns().isEmpty();
    if (table.getTableType().equals(MANAGED_TABLE) && isPartitioned && leaseDuration.toMillis() > 0) {
        delegate.setPartitionLeases(metastoreContext, databaseName, tableName, partitionNameToLocation, leaseDuration);
    }
}

Example 84 with SchemaTableName

use of com.facebook.presto.spi.SchemaTableName in project presto by prestodb.

the class AbstractTestHiveClient method doTestTransactionDeleteInsert.

private void doTestTransactionDeleteInsert(HiveStorageFormat storageFormat, SchemaTableName tableName, Domain domainToDrop, MaterializedResult insertData, MaterializedResult expectedData, TransactionDeleteInsertTestTag tag, boolean expectQuerySucceed, Optional<ConflictTrigger> conflictTrigger) throws Exception {
    Path writePath = null;
    Path targetPath = null;
    try (Transaction transaction = newTransaction()) {
        try {
            ConnectorMetadata metadata = transaction.getMetadata();
            ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
            ConnectorSession session;
            rollbackIfEquals(tag, ROLLBACK_RIGHT_AWAY);
            // Query 1: delete
            session = newSession();
            HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle).get("pk2");
            TupleDomain<ColumnHandle> tupleDomain = withColumnDomains(ImmutableMap.of(dsColumnHandle, domainToDrop));
            Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain));
            ConnectorTableLayoutHandle tableLayoutHandle = getTableLayout(session, metadata, tableHandle, constraint, transaction).getHandle();
            metadata.metadataDelete(session, tableHandle, tableLayoutHandle);
            rollbackIfEquals(tag, ROLLBACK_AFTER_DELETE);
            // Query 2: insert
            session = newSession();
            ConnectorInsertTableHandle insertTableHandle = metadata.beginInsert(session, tableHandle);
            rollbackIfEquals(tag, ROLLBACK_AFTER_BEGIN_INSERT);
            writePath = getStagingPathRoot(insertTableHandle);
            targetPath = getTargetPathRoot(insertTableHandle);
            ConnectorPageSink sink = pageSinkProvider.createPageSink(transaction.getTransactionHandle(), session, insertTableHandle, TEST_HIVE_PAGE_SINK_CONTEXT);
            sink.appendPage(insertData.toPage());
            rollbackIfEquals(tag, ROLLBACK_AFTER_APPEND_PAGE);
            Collection<Slice> fragments = getFutureValue(sink.finish());
            rollbackIfEquals(tag, ROLLBACK_AFTER_SINK_FINISH);
            metadata.finishInsert(session, insertTableHandle, fragments, ImmutableList.of());
            rollbackIfEquals(tag, ROLLBACK_AFTER_FINISH_INSERT);
            assertEquals(tag, COMMIT);
            if (conflictTrigger.isPresent()) {
                List<PartitionUpdate> partitionUpdates = fragments.stream().map(Slice::getBytes).map(HiveTestUtils.PARTITION_UPDATE_CODEC::fromJson).collect(toList());
                conflictTrigger.get().triggerConflict(session, tableName, insertTableHandle, partitionUpdates);
            }
            transaction.commit();
            if (conflictTrigger.isPresent()) {
                assertTrue(expectQuerySucceed);
                conflictTrigger.get().verifyAndCleanup(tableName);
            }
        } catch (TestingRollbackException e) {
            transaction.rollback();
        } catch (PrestoException e) {
            assertFalse(expectQuerySucceed);
            if (conflictTrigger.isPresent()) {
                conflictTrigger.get().verifyAndCleanup(tableName);
            }
        }
    }
    // check that temporary files are removed
    if (writePath != null && !writePath.equals(targetPath)) {
        HdfsContext context = new HdfsContext(newSession(), tableName.getSchemaName(), tableName.getTableName(), writePath.toString(), false);
        FileSystem fileSystem = hdfsEnvironment.getFileSystem(context, writePath);
        assertFalse(fileSystem.exists(writePath));
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), false, DEFAULT_COLUMN_CONVERTER_PROVIDER);
        // verify partitions
        List<String> partitionNames = transaction.getMetastore().getPartitionNames(metastoreContext, tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new AssertionError("Table does not exist: " + tableName));
        assertEqualsIgnoreOrder(partitionNames, expectedData.getMaterializedRows().stream().map(row -> format("pk1=%s/pk2=%s", row.getField(1), row.getField(2))).distinct().collect(toList()));
        // load the new table
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        // verify the data
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
        assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedData.getMaterializedRows());
    }
}

Example 85 with SchemaTableName

use of com.facebook.presto.spi.SchemaTableName in project presto by prestodb.

the class AbstractTestHiveClient method assertGetRecords.

protected void assertGetRecords(String tableName, HiveStorageFormat hiveStorageFormat) throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, new SchemaTableName(database, tableName));
        ConnectorTableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle);
        ConnectorTableLayoutHandle layoutHandle = getLayout(session, transaction, tableHandle, TupleDomain.all());
        List<ConnectorSplit> splits = getAllSplits(session, transaction, layoutHandle);
        assertEquals(splits.size(), 1);
        HiveSplit hiveSplit = (HiveSplit) getOnlyElement(splits);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
        ConnectorPageSource pageSource = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, hiveSplit, layoutHandle, columnHandles, NON_CACHEABLE);
        assertGetRecords(hiveStorageFormat, tableMetadata, hiveSplit, pageSource, columnHandles);
    }
}