Examples with ColumnStatistics com.amazonaws.services.glue.model.ColumnStatistics used on opensource projects

Example 1 with ColumnStatistics

use of com.amazonaws.services.glue.model.ColumnStatistics in project alluxio by Alluxio.

the class GlueUtilsTest method protoColStatsWithBooleanData.

@Test
public void protoColStatsWithBooleanData() {
    // ColumnStatistics required fields: AnalyzedTime, ColumnName, ColumnType, StatisticsData
    ColumnStatistics glueColStats = new ColumnStatistics();
    glueColStats.setColumnName("colName");
    glueColStats.setColumnType("colType");
    ColumnStatisticsData data = new ColumnStatisticsData();
    // verify empty data
    data.setType("BOOLEAN");
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
    // verify non-empty data
    BooleanColumnStatisticsData booleanData = new BooleanColumnStatisticsData();
    booleanData.setNumberOfFalses(mRandom.nextLong());
    booleanData.setNumberOfTrues(mRandom.nextLong());
    booleanData.setNumberOfNulls(mRandom.nextLong());
    data.setBooleanColumnStatisticsData(booleanData);
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
}

Example 2 with ColumnStatistics

use of com.amazonaws.services.glue.model.ColumnStatistics in project alluxio by Alluxio.

the class GlueUtilsTest method protoColStatsWithBinaryData.

@Test
public void protoColStatsWithBinaryData() {
    // ColumnStatistics required fields: AnalyzedTime, ColumnName, ColumnType, StatisticsData
    ColumnStatistics glueColStats = new ColumnStatistics();
    glueColStats.setColumnName("colName");
    glueColStats.setColumnType("colType");
    ColumnStatisticsData data = new ColumnStatisticsData();
    // verify empty data
    data.setType("BINARY");
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
    // verify non-empty data
    BinaryColumnStatisticsData binaryData = new BinaryColumnStatisticsData();
    binaryData.setAverageLength(mRandom.nextDouble());
    binaryData.setMaximumLength(mRandom.nextLong());
    binaryData.setNumberOfNulls(mRandom.nextLong());
    data.setBinaryColumnStatisticsData(binaryData);
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
}

Example 3 with ColumnStatistics

use of com.amazonaws.services.glue.model.ColumnStatistics in project alluxio by Alluxio.

the class GlueUtilsTest method protoColStatsWithDateData.

@Test
public void protoColStatsWithDateData() {
    // ColumnStatistics required fields: AnalyzedTime, ColumnName, ColumnType, StatisticsData
    ColumnStatistics glueColStats = new ColumnStatistics();
    glueColStats.setColumnName("colName");
    glueColStats.setColumnType("colType");
    ColumnStatisticsData data = new ColumnStatisticsData();
    // verify empty data
    data.setType("DATE");
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
    // verify non-empty data
    DateColumnStatisticsData dateData = new DateColumnStatisticsData();
    dateData.setMaximumValue(new Date(mRandom.nextLong()));
    dateData.setMinimumValue(new Date(mRandom.nextLong()));
    dateData.setNumberOfNulls(mRandom.nextLong());
    dateData.setNumberOfDistinctValues(mRandom.nextLong());
    data.setDateColumnStatisticsData(dateData);
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
    // verify null column values
    dateData.setMaximumValue(null);
    dateData.setMinimumValue(null);
    data.setDateColumnStatisticsData(dateData);
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
}

Example 4 with ColumnStatistics

use of com.amazonaws.services.glue.model.ColumnStatistics in project alluxio by Alluxio.

the class GlueUtilsTest method protoColStatsWithDecimalData.

@Test
public void protoColStatsWithDecimalData() {
    // ColumnStatistics required fields: AnalyzedTime, ColumnName, ColumnType, StatisticsData
    ColumnStatistics glueColStats = new ColumnStatistics();
    glueColStats.setColumnName("colName");
    glueColStats.setColumnType("colType");
    ColumnStatisticsData data = new ColumnStatisticsData();
    // verify empty data
    data.setType("DECIMAL");
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
    // verify non-empty data
    DecimalColumnStatisticsData decimalData = new DecimalColumnStatisticsData();
    DecimalNumber maxDecimalNumber = new DecimalNumber();
    maxDecimalNumber.setScale(mRandom.nextInt());
    maxDecimalNumber.setUnscaledValue(ByteBuffer.wrap(CommonUtils.randomBytes(5)));
    DecimalNumber minDecimalNumber = new DecimalNumber();
    minDecimalNumber.setScale(mRandom.nextInt());
    minDecimalNumber.setUnscaledValue(ByteBuffer.wrap(CommonUtils.randomBytes(5)));
    decimalData.setMaximumValue(maxDecimalNumber);
    decimalData.setMinimumValue(minDecimalNumber);
    decimalData.setNumberOfNulls(mRandom.nextLong());
    decimalData.setNumberOfDistinctValues(mRandom.nextLong());
    data.setDecimalColumnStatisticsData(decimalData);
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
    // verify null column values
    decimalData.setMaximumValue(null);
    decimalData.setMinimumValue(null);
    data.setDecimalColumnStatisticsData(decimalData);
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
}

Example 5 with ColumnStatistics

use of com.amazonaws.services.glue.model.ColumnStatistics in project alluxio by Alluxio.

the class GlueUtilsTest method protoColStatsWithStringData.

@Test
public void protoColStatsWithStringData() {
    // ColumnStatistics required fields: AnalyzedTime, ColumnName, ColumnType, StatisticsData
    ColumnStatistics glueColStats = new ColumnStatistics();
    glueColStats.setColumnName("colName");
    glueColStats.setColumnType("colType");
    ColumnStatisticsData data = new ColumnStatisticsData();
    // verify empty data
    data.setType("STRING");
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
    // verify non-empty data
    StringColumnStatisticsData stringData = new StringColumnStatisticsData();
    stringData.setMaximumLength(mRandom.nextLong());
    stringData.setAverageLength(mRandom.nextDouble());
    stringData.setNumberOfNulls(mRandom.nextLong());
    stringData.setNumberOfDistinctValues(mRandom.nextLong());
    data.setStringColumnStatisticsData(stringData);
    glueColStats.setStatisticsData(data);
    verifyColumnStats(glueColStats);
}