Examples with FlinkRuntimeException org.apache.flink.util.FlinkRuntimeException used on opensource projects

Example 11 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class HiveTableSink method consume.

private DataStreamSink<?> consume(ProviderContext providerContext, DataStream<RowData> dataStream, boolean isBounded, DataStructureConverter converter) {
    checkAcidTable(catalogTable.getOptions(), identifier.toObjectPath());
    try (HiveMetastoreClientWrapper client = HiveMetastoreClientFactory.create(HiveConfUtils.create(jobConf), hiveVersion)) {
        Table table = client.getTable(identifier.getDatabaseName(), identifier.getObjectName());
        StorageDescriptor sd = table.getSd();
        Class hiveOutputFormatClz = hiveShim.getHiveOutputFormatClass(Class.forName(sd.getOutputFormat()));
        boolean isCompressed = jobConf.getBoolean(HiveConf.ConfVars.COMPRESSRESULT.varname, false);
        HiveWriterFactory writerFactory = new HiveWriterFactory(jobConf, hiveOutputFormatClz, sd.getSerdeInfo(), tableSchema, getPartitionKeyArray(), HiveReflectionUtils.getTableMetadata(hiveShim, table), hiveShim, isCompressed);
        String extension = Utilities.getFileExtension(jobConf, isCompressed, (HiveOutputFormat<?, ?>) hiveOutputFormatClz.newInstance());
        OutputFileConfig.OutputFileConfigBuilder fileNamingBuilder = OutputFileConfig.builder().withPartPrefix("part-" + UUID.randomUUID().toString()).withPartSuffix(extension == null ? "" : extension);
        final int parallelism = Optional.ofNullable(configuredParallelism).orElse(dataStream.getParallelism());
        if (isBounded) {
            OutputFileConfig fileNaming = fileNamingBuilder.build();
            return createBatchSink(dataStream, converter, sd, writerFactory, fileNaming, parallelism);
        } else {
            if (overwrite) {
                throw new IllegalStateException("Streaming mode not support overwrite.");
            }
            Properties tableProps = HiveReflectionUtils.getTableMetadata(hiveShim, table);
            return createStreamSink(providerContext, dataStream, sd, tableProps, writerFactory, fileNamingBuilder, parallelism);
        }
    } catch (TException e) {
        throw new CatalogException("Failed to query Hive metaStore", e);
    } catch (IOException e) {
        throw new FlinkRuntimeException("Failed to create staging dir", e);
    } catch (ClassNotFoundException e) {
        throw new FlinkHiveException("Failed to get output format class", e);
    } catch (IllegalAccessException | InstantiationException e) {
        throw new FlinkHiveException("Failed to instantiate output format instance", e);
    }
}

Example 12 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class PulsarCommitter method commit.

@Override
public void commit(Collection<CommitRequest<PulsarCommittable>> requests) throws IOException, InterruptedException {
    TransactionCoordinatorClient client = transactionCoordinatorClient();
    for (CommitRequest<PulsarCommittable> request : requests) {
        PulsarCommittable committable = request.getCommittable();
        TxnID txnID = committable.getTxnID();
        String topic = committable.getTopic();
        LOG.debug("Start committing the Pulsar transaction {} for topic {}", txnID, topic);
        try {
            client.commit(txnID);
        } catch (TransactionCoordinatorClientException e) {
            // This is a known bug for Pulsar Transaction.
            // We have to use instanceof instead of catching them.
            TransactionCoordinatorClientException ex = PulsarTransactionUtils.unwrap(e);
            if (ex instanceof CoordinatorNotFoundException) {
                LOG.error("We couldn't find the Transaction Coordinator from Pulsar broker {}. " + "Check your broker configuration.", committable, ex);
                request.signalFailedWithKnownReason(ex);
            } else if (ex instanceof InvalidTxnStatusException) {
                LOG.error("Unable to commit transaction ({}) because it's in an invalid state. " + "Most likely the transaction has been aborted for some reason. " + "Please check the Pulsar broker logs for more details.", committable, ex);
                request.signalAlreadyCommitted();
            } else if (ex instanceof TransactionNotFoundException) {
                if (request.getNumberOfRetries() == 0) {
                    LOG.error("Unable to commit transaction ({}) because it's not found on Pulsar broker. " + "Most likely the checkpoint interval exceed the transaction timeout.", committable, ex);
                    request.signalFailedWithKnownReason(ex);
                } else {
                    LOG.warn("We can't find the transaction {} after {} retry committing. " + "This may mean that the transaction have been committed in previous but failed with timeout. " + "So we just mark it as committed.", txnID, request.getNumberOfRetries());
                    request.signalAlreadyCommitted();
                }
            } else if (ex instanceof MetaStoreHandlerNotExistsException) {
                LOG.error("We can't find the meta store handler by the mostSigBits from TxnID {}. " + "Did you change the metadata for topic {}?", committable, TRANSACTION_COORDINATOR_ASSIGN, ex);
                request.signalFailedWithKnownReason(ex);
            } else {
                LOG.error("Encountered retriable exception while committing transaction {} for topic {}.", committable, topic, ex);
                int maxRecommitTimes = sinkConfiguration.getMaxRecommitTimes();
                if (request.getNumberOfRetries() < maxRecommitTimes) {
                    request.retryLater();
                } else {
                    String message = String.format("Failed to commit transaction %s after retrying %d times", txnID, maxRecommitTimes);
                    request.signalFailedWithKnownReason(new FlinkRuntimeException(message, ex));
                }
            }
        } catch (Exception e) {
            LOG.error("Transaction ({}) encountered unknown error and data could be potentially lost.", committable, e);
            request.signalFailedWithUnknownReason(e);
        }
    }
}

Example 13 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class TopicProducerRegister method abortTransactions.

/**
 * Abort the existed transactions. This method would be used when closing PulsarWriter.
 */
private void abortTransactions() {
    if (transactionRegister.isEmpty()) {
        return;
    }
    TransactionCoordinatorClient coordinatorClient = ((PulsarClientImpl) pulsarClient).getTcClient();
    // This null check is used for making sure transaction is enabled in client.
    checkNotNull(coordinatorClient);
    try (Closer closer = Closer.create()) {
        for (Transaction transaction : transactionRegister.values()) {
            TxnID txnID = transaction.getTxnID();
            closer.register(() -> coordinatorClient.abort(txnID));
        }
        clearTransactions();
    } catch (IOException e) {
        throw new FlinkRuntimeException(e);
    }
}

Example 14 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class TtlAggregateFunction method getResult.

@Override
public OUT getResult(TtlValue<ACC> accumulator) {
    Preconditions.checkNotNull(updater, "State updater should be set in TtlAggregatingState");
    Preconditions.checkNotNull(stateClear, "State clearing should be set in TtlAggregatingState");
    ACC userAcc;
    try {
        userAcc = getWithTtlCheckAndUpdate(() -> accumulator, updater, stateClear);
    } catch (Exception e) {
        throw new FlinkRuntimeException("Failed to retrieve original internal aggregating state", e);
    }
    return userAcc == null ? null : original.getResult(userAcc);
}

Example 15 with FlinkRuntimeException

use of org.apache.flink.util.FlinkRuntimeException in project flink by apache.

the class ApplicationDispatcherBootstrapTest method testErrorHandlerIsCalledWhenSubmissionThrowsAnException.

@Test
public void testErrorHandlerIsCalledWhenSubmissionThrowsAnException() throws Exception {
    final AtomicBoolean shutdownCalled = new AtomicBoolean(false);
    final TestingDispatcherGateway.Builder dispatcherBuilder = runningJobGatewayBuilder().setSubmitFunction(jobGraph -> {
        throw new FlinkRuntimeException("Nope!");
    }).setClusterShutdownFunction(status -> {
        shutdownCalled.set(true);
        return CompletableFuture.completedFuture(Acknowledge.get());
    });
    // we're "listening" on this to be completed to verify that the error handler is called.
    // In production, this will shut down the cluster with an exception.
    final CompletableFuture<Void> errorHandlerFuture = new CompletableFuture<>();
    final ApplicationDispatcherBootstrap bootstrap = createApplicationDispatcherBootstrap(2, dispatcherBuilder.build(), scheduledExecutor, errorHandlerFuture::completeExceptionally);
    final CompletableFuture<Acknowledge> completionFuture = bootstrap.getBootstrapCompletionFuture();
    // we call the error handler
    assertException(errorHandlerFuture, FlinkRuntimeException.class);
    // we return a future that is completed exceptionally
    assertException(completionFuture, FlinkRuntimeException.class);
    // and cluster shutdown didn't get called
    assertFalse(shutdownCalled.get());
}