Examples with RowBasedSchemaProvider org.apache.hudi.utilities.schema.RowBasedSchemaProvider used on opensource projects

Example 1 with RowBasedSchemaProvider

use of org.apache.hudi.utilities.schema.RowBasedSchemaProvider in project hudi by apache.

the class TestInputBatch method getSchemaProviderShouldReturnGivenSchemaProvider.

@Test
public void getSchemaProviderShouldReturnGivenSchemaProvider() {
    SchemaProvider schemaProvider = new RowBasedSchemaProvider(null);
    final InputBatch<String> inputBatch = new InputBatch<>(Option.of("foo"), null, schemaProvider);
    assertSame(schemaProvider, inputBatch.getSchemaProvider());
}

Example 2 with RowBasedSchemaProvider

use of org.apache.hudi.utilities.schema.RowBasedSchemaProvider in project hudi by apache.

the class UtilHelpers method createLatestSchemaProvider.

/**
 * Create latest schema provider for Target schema.
 *
 * @param structType spark data type of incoming batch.
 * @param jssc       instance of {@link JavaSparkContext}.
 * @param fs         instance of {@link FileSystem}.
 * @param basePath   base path of the table.
 * @return the schema provider where target schema refers to latest schema(either incoming schema or table schema).
 */
public static SchemaProvider createLatestSchemaProvider(StructType structType, JavaSparkContext jssc, FileSystem fs, String basePath) {
    SchemaProvider rowSchemaProvider = new RowBasedSchemaProvider(structType);
    Schema writeSchema = rowSchemaProvider.getTargetSchema();
    Schema latestTableSchema = writeSchema;
    try {
        if (FSUtils.isTableExists(basePath, fs)) {
            HoodieTableMetaClient tableMetaClient = HoodieTableMetaClient.builder().setConf(jssc.sc().hadoopConfiguration()).setBasePath(basePath).build();
            TableSchemaResolver tableSchemaResolver = new TableSchemaResolver(tableMetaClient);
            latestTableSchema = tableSchemaResolver.getLatestSchema(writeSchema, true, (Function1<Schema, Schema>) v1 -> AvroConversionUtils.convertStructTypeToAvroSchema(AvroConversionUtils.convertAvroSchemaToStructType(v1), RowBasedSchemaProvider.HOODIE_RECORD_STRUCT_NAME, RowBasedSchemaProvider.HOODIE_RECORD_NAMESPACE));
        }
    } catch (IOException e) {
        LOG.warn("Could not fetch table schema. Falling back to writer schema");
    }
    final Schema finalLatestTableSchema = latestTableSchema;
    return new SchemaProvider(new TypedProperties()) {

        @Override
        public Schema getSourceSchema() {
            return rowSchemaProvider.getSourceSchema();
        }

        @Override
        public Schema getTargetSchema() {
            return finalLatestTableSchema;
        }
    };
}