Examples with HoodieInsertException org.apache.hudi.exception.HoodieInsertException used on opensource projects

Example 6 with HoodieInsertException

use of org.apache.hudi.exception.HoodieInsertException in project hudi by apache.

the class HoodieTable method validateSchema.

/**
 * Ensure that the current writerSchema is compatible with the latest schema of this dataset.
 *
 * When inserting/updating data, we read records using the last used schema and convert them to the
 * GenericRecords with writerSchema. Hence, we need to ensure that this conversion can take place without errors.
 */
private void validateSchema() throws HoodieUpsertException, HoodieInsertException {
    if (!config.getAvroSchemaValidate() || getActiveTimeline().getCommitsTimeline().filterCompletedInstants().empty()) {
        // Check not required
        return;
    }
    Schema tableSchema;
    Schema writerSchema;
    boolean isValid;
    try {
        TableSchemaResolver schemaResolver = new TableSchemaResolver(getMetaClient());
        writerSchema = HoodieAvroUtils.createHoodieWriteSchema(config.getSchema());
        tableSchema = HoodieAvroUtils.createHoodieWriteSchema(schemaResolver.getTableAvroSchemaWithoutMetadataFields());
        isValid = TableSchemaResolver.isSchemaCompatible(tableSchema, writerSchema);
    } catch (Exception e) {
        throw new HoodieException("Failed to read schema/check compatibility for base path " + metaClient.getBasePath(), e);
    }
    if (!isValid) {
        throw new HoodieException("Failed schema compatibility check for writerSchema :" + writerSchema + ", table schema :" + tableSchema + ", base path :" + metaClient.getBasePath());
    }
}

Example 7 with HoodieInsertException

use of org.apache.hudi.exception.HoodieInsertException in project hudi by apache.

the class HoodieCreateHandle method close.

/**
 * Performs actions to durably, persist the current changes and returns a WriteStatus object.
 */
@Override
public List<WriteStatus> close() {
    LOG.info("Closing the file " + writeStatus.getFileId() + " as we are done with all the records " + recordsWritten);
    try {
        fileWriter.close();
        setupWriteStatus();
        LOG.info(String.format("CreateHandle for partitionPath %s fileID %s, took %d ms.", writeStatus.getStat().getPartitionPath(), writeStatus.getStat().getFileId(), writeStatus.getStat().getRuntimeStats().getTotalCreateTime()));
        return Collections.singletonList(writeStatus);
    } catch (IOException e) {
        throw new HoodieInsertException("Failed to close the Insert Handle for path " + path, e);
    }
}

Example 8 with HoodieInsertException

use of org.apache.hudi.exception.HoodieInsertException in project hudi by apache.

the class TestHoodieClientOnCopyOnWriteStorage method testPreCommitValidationFailureOnInsert.

@Test
public void testPreCommitValidationFailureOnInsert() throws Exception {
    int numRecords = 200;
    HoodiePreCommitValidatorConfig validatorConfig = HoodiePreCommitValidatorConfig.newBuilder().withPreCommitValidator(SqlQuerySingleResultPreCommitValidator.class.getName()).withPrecommitValidatorSingleResultSqlQueries(COUNT_SQL_QUERY_FOR_VALIDATION + "#" + 500).build();
    HoodieWriteConfig config = getConfigBuilder().withPreCommitValidatorConfig(validatorConfig).build();
    String newCommitTime = HoodieActiveTimeline.createNewInstantTime();
    try (SparkRDDWriteClient client = getHoodieWriteClient(config)) {
        Function3<JavaRDD<WriteStatus>, SparkRDDWriteClient, JavaRDD<HoodieRecord>, String> writeFn = (writeClient, recordRDD, instantTime) -> writeClient.bulkInsert(recordRDD, instantTime, Option.empty());
        JavaRDD<WriteStatus> result = insertFirstBatch(config, client, newCommitTime, "000", numRecords, writeFn, false, false, numRecords);
        fail("Expected validation to fail because we only insert 200 rows. Validation is configured to expect 500 rows");
    } catch (HoodieInsertException e) {
        if (e.getCause() instanceof HoodieValidationException) {
        // expected because wrong value passed
        } else {
            throw e;
        }
    }
    assertFalse(testTable.commitExists(newCommitTime));
}