Examples with HoodieException org.apache.hudi.exception.HoodieException used on opensource projects

Example 86 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class HoodieSinkTask method bootstrap.

private void bootstrap(Collection<TopicPartition> partitions) {
    LOG.info(String.format("Bootstrap task for connector %s with id %s with assignments %s part %s", connectorName, taskId, context.assignment(), partitions));
    for (TopicPartition partition : partitions) {
        try {
            // If the partition is 0, instantiate the Leader
            if (partition.partition() == ConnectTransactionCoordinator.COORDINATOR_KAFKA_PARTITION) {
                ConnectTransactionCoordinator coordinator = new ConnectTransactionCoordinator(connectConfigs, partition, controlKafkaClient);
                coordinator.start();
                transactionCoordinators.put(partition, coordinator);
            }
            ConnectTransactionParticipant worker = new ConnectTransactionParticipant(connectConfigs, partition, controlKafkaClient, context);
            transactionParticipants.put(partition, worker);
            worker.start();
        } catch (HoodieException exception) {
            LOG.error(String.format("Fatal error initializing task %s for partition %s", taskId, partition.partition()), exception);
        }
    }
}

Example 87 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class KafkaConnectUtils method getCommitMetadataForLatestInstant.

/**
 * Get the Metadata from the latest commit file.
 *
 * @param metaClient The {@link HoodieTableMetaClient} to get access to the meta data.
 * @return An Optional {@link HoodieCommitMetadata} containing the meta data from the latest commit file.
 */
public static Option<HoodieCommitMetadata> getCommitMetadataForLatestInstant(HoodieTableMetaClient metaClient) {
    HoodieTimeline timeline = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants().filter(instant -> (metaClient.getTableType() == HoodieTableType.COPY_ON_WRITE && instant.getAction().equals(HoodieActiveTimeline.COMMIT_ACTION)) || (metaClient.getTableType() == HoodieTableType.MERGE_ON_READ && instant.getAction().equals(HoodieActiveTimeline.DELTA_COMMIT_ACTION)));
    Option<HoodieInstant> latestInstant = timeline.lastInstant();
    if (latestInstant.isPresent()) {
        try {
            byte[] data = timeline.getInstantDetails(latestInstant.get()).get();
            return Option.of(HoodieCommitMetadata.fromBytes(data, HoodieCommitMetadata.class));
        } catch (Exception e) {
            throw new HoodieException("Failed to read schema from commit metadata", e);
        }
    } else {
        return Option.empty();
    }
}

Example 88 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class KafkaConnectUtils method getDefaultHadoopConf.

/**
 * Returns the default Hadoop Configuration.
 *
 * @return
 */
public static Configuration getDefaultHadoopConf(KafkaConnectConfigs connectConfigs) {
    Configuration hadoopConf = new Configuration();
    // add hadoop config files
    if (!StringUtils.isNullOrEmpty(connectConfigs.getHadoopConfDir()) || !StringUtils.isNullOrEmpty(connectConfigs.getHadoopConfHome())) {
        try {
            List<Path> configFiles = getHadoopConfigFiles(connectConfigs.getHadoopConfDir(), connectConfigs.getHadoopConfHome());
            configFiles.forEach(f -> hadoopConf.addResource(new org.apache.hadoop.fs.Path(f.toAbsolutePath().toUri())));
        } catch (Exception e) {
            throw new HoodieException("Failed to read hadoop configuration!", e);
        }
    } else {
        DEFAULT_HADOOP_CONF_FILES.forEach(f -> hadoopConf.addResource(new org.apache.hadoop.fs.Path(f.toAbsolutePath().toUri())));
    }
    connectConfigs.getProps().keySet().stream().filter(prop -> {
        // configuration items before passing to hadoop/hive configs
        return !prop.toString().startsWith(HOODIE_CONF_PREFIX);
    }).forEach(prop -> {
        hadoopConf.set(prop.toString(), connectConfigs.getProps().get(prop.toString()).toString());
    });
    return hadoopConf;
}

Example 89 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class TestAbstractConnectWriter method testAbstractWriterForAllFormats.

@ParameterizedTest
@EnumSource(value = TestInputFormats.class)
public void testAbstractWriterForAllFormats(TestInputFormats inputFormats) throws Exception {
    Schema schema = schemaProvider.getSourceSchema();
    List<?> inputRecords;
    List<HoodieRecord> expectedRecords;
    String formatConverter;
    switch(inputFormats) {
        case JSON_STRING:
            formatConverter = AbstractConnectWriter.KAFKA_STRING_CONVERTER;
            GenericDatumReader<IndexedRecord> reader = new GenericDatumReader<>(schema, schema);
            inputRecords = SchemaTestUtil.generateTestJsonRecords(0, NUM_RECORDS);
            expectedRecords = ((List<String>) inputRecords).stream().map(s -> {
                try {
                    return HoodieAvroUtils.rewriteRecord((GenericRecord) reader.read(null, DecoderFactory.get().jsonDecoder(schema, s)), schema);
                } catch (IOException exception) {
                    throw new HoodieException("Error converting JSON records to AVRO");
                }
            }).map(p -> convertToHoodieRecords(p, p.get(RECORD_KEY_INDEX).toString(), "000/00/00")).collect(Collectors.toList());
            break;
        case AVRO:
            formatConverter = AbstractConnectWriter.KAFKA_AVRO_CONVERTER;
            inputRecords = SchemaTestUtil.generateTestRecords(0, NUM_RECORDS);
            expectedRecords = inputRecords.stream().map(s -> HoodieAvroUtils.rewriteRecord((GenericRecord) s, schema)).map(p -> convertToHoodieRecords(p, p.get(RECORD_KEY_INDEX).toString(), "000/00/00")).collect(Collectors.toList());
            break;
        default:
            throw new HoodieException("Unknown test scenario " + inputFormats);
    }
    configs = KafkaConnectConfigs.newBuilder().withProperties(Collections.singletonMap(KafkaConnectConfigs.KAFKA_VALUE_CONVERTER, formatConverter)).build();
    AbstractHudiConnectWriterTestWrapper writer = new AbstractHudiConnectWriterTestWrapper(configs, keyGenerator, schemaProvider);
    for (int i = 0; i < NUM_RECORDS; i++) {
        writer.writeRecord(getNextKafkaRecord(inputRecords.get(i)));
    }
    validateRecords(writer.getWrittenRecords(), expectedRecords);
}

Example 90 with HoodieException

use of org.apache.hudi.exception.HoodieException in project hudi by apache.

the class TwoToOneDowngradeHandler method downgrade.

@Override
public Map<ConfigProperty, String> downgrade(HoodieWriteConfig config, HoodieEngineContext context, String instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) {
    HoodieTable table = upgradeDowngradeHelper.getTable(config, context);
    HoodieTableMetaClient metaClient = table.getMetaClient();
    // re-create marker files if any partial timeline server based markers are found
    HoodieTimeline inflightTimeline = metaClient.getCommitsTimeline().filterPendingExcludingCompaction();
    List<HoodieInstant> commits = inflightTimeline.getReverseOrderedInstants().collect(Collectors.toList());
    for (HoodieInstant inflightInstant : commits) {
        // Converts the markers in new format to old format of direct markers
        try {
            convertToDirectMarkers(inflightInstant.getTimestamp(), table, context, config.getMarkersDeleteParallelism());
        } catch (IOException e) {
            throw new HoodieException("Converting marker files to DIRECT style failed during downgrade", e);
        }
    }
    return Collections.EMPTY_MAP;
}