Examples with InternalRow org.apache.spark.sql.catalyst.InternalRow used on opensource projects

Example 96 with InternalRow

use of org.apache.spark.sql.catalyst.InternalRow in project hudi by apache.

the class TestHoodieInternalRow method testIsNullCheck.

@Test
public void testIsNullCheck() {
    for (int i = 0; i < 16; i++) {
        Object[] values = getRandomValue(true);
        InternalRow row = new GenericInternalRow(values);
        HoodieInternalRow hoodieInternalRow = new HoodieInternalRow("commitTime", "commitSeqNo", "recordKey", "partitionPath", "fileName", row);
        hoodieInternalRow.setNullAt(i);
        nullIndices.clear();
        nullIndices.add(i);
        assertValues(hoodieInternalRow, "commitTime", "commitSeqNo", "recordKey", "partitionPath", "fileName", values, nullIndices);
    }
    // run it for 5 rounds
    for (int i = 0; i < 5; i++) {
        int numNullValues = 1 + RANDOM.nextInt(4);
        List<Integer> nullsSoFar = new ArrayList<>();
        while (nullsSoFar.size() < numNullValues) {
            int randomIndex = RANDOM.nextInt(16);
            if (!nullsSoFar.contains(randomIndex)) {
                nullsSoFar.add(randomIndex);
            }
        }
        Object[] values = getRandomValue(true);
        InternalRow row = new GenericInternalRow(values);
        HoodieInternalRow hoodieInternalRow = new HoodieInternalRow("commitTime", "commitSeqNo", "recordKey", "partitionPath", "fileName", row);
        nullIndices.clear();
        for (Integer index : nullsSoFar) {
            hoodieInternalRow.setNullAt(index);
            nullIndices.add(index);
        }
        assertValues(hoodieInternalRow, "commitTime", "commitSeqNo", "recordKey", "partitionPath", "fileName", values, nullIndices);
    }
}

Example 97 with InternalRow

use of org.apache.spark.sql.catalyst.InternalRow in project hudi by apache.

the class SparkDatasetTestUtils method serializeRow.

private static InternalRow serializeRow(ExpressionEncoder encoder, Row row) throws InvocationTargetException, IllegalAccessException, NoSuchMethodException, ClassNotFoundException {
    // TODO remove reflection if Spark 2.x support is dropped
    if (package$.MODULE$.SPARK_VERSION().startsWith("2.")) {
        Method spark2method = encoder.getClass().getMethod("toRow", Object.class);
        return (InternalRow) spark2method.invoke(encoder, row);
    } else {
        Class<?> serializerClass = Class.forName("org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$Serializer");
        Object serializer = encoder.getClass().getMethod("createSerializer").invoke(encoder);
        Method aboveSpark2method = serializerClass.getMethod("apply", Object.class);
        return (InternalRow) aboveSpark2method.invoke(serializer, row);
    }
}

Example 98 with InternalRow

use of org.apache.spark.sql.catalyst.InternalRow in project hudi by apache.

the class SparkDatasetTestUtils method toInternalRows.

/**
 * Convert Dataset<Row>s to List of {@link InternalRow}s.
 *
 * @param rows Dataset<Row>s to be converted
 * @return the List of {@link InternalRow}s thus converted.
 */
public static List<InternalRow> toInternalRows(Dataset<Row> rows, ExpressionEncoder encoder) throws Exception {
    List<InternalRow> toReturn = new ArrayList<>();
    List<Row> rowList = rows.collectAsList();
    for (Row row : rowList) {
        toReturn.add(serializeRow(encoder, row).copy());
    }
    return toReturn;
}

Example 99 with InternalRow

use of org.apache.spark.sql.catalyst.InternalRow in project hudi by apache.

the class TestComplexKeyGenerator method testMultipleValueKeyGenerator.

@Test
public void testMultipleValueKeyGenerator() {
    TypedProperties properties = new TypedProperties();
    properties.setProperty(KeyGeneratorOptions.RECORDKEY_FIELD_NAME.key(), "_row_key,timestamp");
    properties.setProperty(KeyGeneratorOptions.PARTITIONPATH_FIELD_NAME.key(), "rider,driver");
    ComplexKeyGenerator compositeKeyGenerator = new ComplexKeyGenerator(properties);
    assertEquals(compositeKeyGenerator.getRecordKeyFields().size(), 2);
    assertEquals(compositeKeyGenerator.getPartitionPathFields().size(), 2);
    HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator();
    GenericRecord record = dataGenerator.generateGenericRecords(1).get(0);
    String rowKey = "_row_key" + ComplexAvroKeyGenerator.DEFAULT_RECORD_KEY_SEPARATOR + record.get("_row_key").toString() + "," + "timestamp" + ComplexAvroKeyGenerator.DEFAULT_RECORD_KEY_SEPARATOR + record.get("timestamp").toString();
    String partitionPath = record.get("rider").toString() + "/" + record.get("driver").toString();
    HoodieKey hoodieKey = compositeKeyGenerator.getKey(record);
    assertEquals(rowKey, hoodieKey.getRecordKey());
    assertEquals(partitionPath, hoodieKey.getPartitionPath());
    Row row = KeyGeneratorTestUtilities.getRow(record, HoodieTestDataGenerator.AVRO_SCHEMA, AvroConversionUtils.convertAvroSchemaToStructType(HoodieTestDataGenerator.AVRO_SCHEMA));
    Assertions.assertEquals(compositeKeyGenerator.getPartitionPath(row), partitionPath);
    InternalRow internalRow = KeyGeneratorTestUtilities.getInternalRow(row);
    Assertions.assertEquals(compositeKeyGenerator.getPartitionPath(internalRow, row.schema()), partitionPath);
}

Example 100 with InternalRow

use of org.apache.spark.sql.catalyst.InternalRow in project hudi by apache.

the class TestCustomKeyGenerator method testSimpleKeyGenerator.

public void testSimpleKeyGenerator(TypedProperties props) throws IOException {
    BuiltinKeyGenerator keyGenerator = (BuiltinKeyGenerator) HoodieSparkKeyGeneratorFactory.createKeyGenerator(props);
    GenericRecord record = getRecord();
    HoodieKey key = keyGenerator.getKey(record);
    Assertions.assertEquals(key.getRecordKey(), "key1");
    Assertions.assertEquals(key.getPartitionPath(), "timestamp=4357686");
    Row row = KeyGeneratorTestUtilities.getRow(record);
    Assertions.assertEquals(keyGenerator.getRecordKey(row), "key1");
    Assertions.assertEquals(keyGenerator.getPartitionPath(row), "timestamp=4357686");
    InternalRow internalRow = KeyGeneratorTestUtilities.getInternalRow(row);
    Assertions.assertEquals(keyGenerator.getPartitionPath(internalRow, row.schema()), "timestamp=4357686");
}