Examples with Schema org.apache.pinot.spi.data.Schema used on opensource projects

Example 1 with Schema

use of org.apache.pinot.spi.data.Schema in project presto by prestodb.

the class TestPinotColumnMetadata method testTimeFieldInPinotSchemaToPinotColumns.

@Test
public void testTimeFieldInPinotSchemaToPinotColumns() {
    PinotConfig pinotConfig = new PinotConfig();
    pinotConfig.setInferDateTypeInSchema(true);
    pinotConfig.setInferTimestampTypeInSchema(true);
    // Test Date
    Schema testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch")).build();
    List<PinotColumn> pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    PinotColumn column = pinotColumns.get(0);
    assertEquals(column.getName(), "daysSinceEpoch");
    assertEquals(column.getType(), DateType.DATE);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "daysSinceEpoch");
    assertEquals(column.getType(), DateType.DATE);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    // Test Timestamp
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "millisSinceEpoch");
    assertEquals(column.getType(), TimestampType.TIMESTAMP);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "millisSinceEpoch");
    assertEquals(column.getType(), TimestampType.TIMESTAMP);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    // Test fallback to BIGINT
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "secondsSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "secondsSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "secondsSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
}

Example 2 with Schema

use of org.apache.pinot.spi.data.Schema in project presto by prestodb.

the class TestPinotColumnMetadata method testConversionWithoutConfigSwitchOn.

@Test
public void testConversionWithoutConfigSwitchOn() {
    PinotConfig pinotConfig = new PinotConfig();
    pinotConfig.setInferDateTypeInSchema(false);
    pinotConfig.setInferTimestampTypeInSchema(false);
    // Test Date
    Schema testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch")).build();
    List<PinotColumn> pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    PinotColumn column = pinotColumns.get(0);
    assertEquals(column.getName(), "daysSinceEpoch");
    assertEquals(column.getType(), INTEGER);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "daysSinceEpoch");
    assertEquals(column.getType(), INTEGER);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    // Test Timestamp
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "millisSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "millisSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    // Test fallback to BIGINT
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "secondsSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.MILLISECONDS, "millisSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "secondsSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
    testSchema = new Schema.SchemaBuilder().addTime(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.LONG, TimeUnit.SECONDS, "secondsSinceEpoch")).build();
    pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    column = pinotColumns.get(0);
    assertEquals(column.getName(), "secondsSinceEpoch");
    assertEquals(column.getType(), BIGINT);
    assertEquals(column.getComment(), FieldSpec.FieldType.TIME.name());
}

Example 3 with Schema

use of org.apache.pinot.spi.data.Schema in project presto by prestodb.

the class TestPinotColumnMetadata method testParsePinotSchemaToPinotColumns.

@Test
public void testParsePinotSchemaToPinotColumns() {
    PinotConfig pinotConfig = new PinotConfig();
    pinotConfig.setInferDateTypeInSchema(true);
    pinotConfig.setInferTimestampTypeInSchema(true);
    Schema testPinotSchema = new Schema.SchemaBuilder().addSingleValueDimension("singleValueIntDimension", FieldSpec.DataType.INT).addSingleValueDimension("singleValueLongDimension", FieldSpec.DataType.LONG).addSingleValueDimension("singleValueFloatDimension", FieldSpec.DataType.FLOAT).addSingleValueDimension("singleValueDoubleDimension", FieldSpec.DataType.DOUBLE).addSingleValueDimension("singleValueBytesDimension", FieldSpec.DataType.BYTES).addSingleValueDimension("singleValueBooleanDimension", FieldSpec.DataType.BOOLEAN).addSingleValueDimension("singleValueStringDimension", FieldSpec.DataType.STRING).addMultiValueDimension("multiValueIntDimension", FieldSpec.DataType.INT).addMultiValueDimension("multiValueLongDimension", FieldSpec.DataType.LONG).addMultiValueDimension("multiValueFloatDimension", FieldSpec.DataType.FLOAT).addMultiValueDimension("multiValueDoubleDimension", FieldSpec.DataType.DOUBLE).addMultiValueDimension("multiValueBytesDimension", FieldSpec.DataType.BYTES).addMultiValueDimension("multiValueBooleanDimension", FieldSpec.DataType.BOOLEAN).addMultiValueDimension("multiValueStringDimension", FieldSpec.DataType.STRING).addMetric("intMetric", FieldSpec.DataType.INT).addMetric("longMetric", FieldSpec.DataType.LONG).addMetric("floatMetric", FieldSpec.DataType.FLOAT).addMetric("doubleMetric", FieldSpec.DataType.DOUBLE).addMetric("bytesMetric", FieldSpec.DataType.BYTES).addTime(new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch"), new TimeGranularitySpec(FieldSpec.DataType.INT, TimeUnit.DAYS, "daysSinceEpoch")).addDateTime("epochDayDateTime", FieldSpec.DataType.INT, "1:DAYS:EPOCH", "1:DAYS").addDateTime("epochMillisDateTime", FieldSpec.DataType.LONG, "1:MILLISECONDS:EPOCH", "1:SECONDS").addDateTime("epochTenDayDateTime", FieldSpec.DataType.INT, "10:DAYS:EPOCH", "1:DAYS").addDateTime("epochSecondsDateTime", FieldSpec.DataType.LONG, "1:SECONDS:EPOCH", "1:SECONDS").build();
    Map<String, Type> expectedTypeMap = new ImmutableMap.Builder<String, Type>().put("singleValueIntDimension", INTEGER).put("singleValueLongDimension", BIGINT).put("singleValueFloatDimension", DOUBLE).put("singleValueDoubleDimension", DOUBLE).put("singleValueBytesDimension", VARBINARY).put("singleValueBooleanDimension", BOOLEAN).put("singleValueStringDimension", VARCHAR).put("multiValueIntDimension", new ArrayType(INTEGER)).put("multiValueLongDimension", new ArrayType(BIGINT)).put("multiValueFloatDimension", new ArrayType(DOUBLE)).put("multiValueDoubleDimension", new ArrayType(DOUBLE)).put("multiValueBytesDimension", new ArrayType(VARBINARY)).put("multiValueBooleanDimension", new ArrayType(BOOLEAN)).put("multiValueStringDimension", new ArrayType(VARCHAR)).put("intMetric", INTEGER).put("longMetric", BIGINT).put("floatMetric", DOUBLE).put("doubleMetric", DOUBLE).put("bytesMetric", VARBINARY).put("daysSinceEpoch", DateType.DATE).put("epochDayDateTime", DateType.DATE).put("epochMillisDateTime", TimestampType.TIMESTAMP).put("epochTenDayDateTime", INTEGER).put("epochSecondsDateTime", BIGINT).build();
    Map<String, String> expectedComment = new ImmutableMap.Builder<String, String>().put("sd1", FieldSpec.FieldType.DIMENSION.name()).put("singleValueIntDimension", FieldSpec.FieldType.DIMENSION.name()).put("singleValueLongDimension", FieldSpec.FieldType.DIMENSION.name()).put("singleValueFloatDimension", FieldSpec.FieldType.DIMENSION.name()).put("singleValueDoubleDimension", FieldSpec.FieldType.DIMENSION.name()).put("singleValueBytesDimension", FieldSpec.FieldType.DIMENSION.name()).put("singleValueBooleanDimension", FieldSpec.FieldType.DIMENSION.name()).put("singleValueStringDimension", FieldSpec.FieldType.DIMENSION.name()).put("multiValueIntDimension", FieldSpec.FieldType.DIMENSION.name()).put("multiValueLongDimension", FieldSpec.FieldType.DIMENSION.name()).put("multiValueFloatDimension", FieldSpec.FieldType.DIMENSION.name()).put("multiValueDoubleDimension", FieldSpec.FieldType.DIMENSION.name()).put("multiValueBytesDimension", FieldSpec.FieldType.DIMENSION.name()).put("multiValueBooleanDimension", FieldSpec.FieldType.DIMENSION.name()).put("multiValueStringDimension", FieldSpec.FieldType.DIMENSION.name()).put("intMetric", FieldSpec.FieldType.METRIC.name()).put("longMetric", FieldSpec.FieldType.METRIC.name()).put("floatMetric", FieldSpec.FieldType.METRIC.name()).put("doubleMetric", FieldSpec.FieldType.METRIC.name()).put("bytesMetric", FieldSpec.FieldType.METRIC.name()).put("daysSinceEpoch", FieldSpec.FieldType.TIME.name()).put("epochDayDateTime", FieldSpec.FieldType.DATE_TIME.name()).put("epochMillisDateTime", FieldSpec.FieldType.DATE_TIME.name()).put("epochTenDayDateTime", FieldSpec.FieldType.DATE_TIME.name()).put("epochSecondsDateTime", FieldSpec.FieldType.DATE_TIME.name()).build();
    List<PinotColumn> pinotColumns = PinotColumnUtils.getPinotColumnsForPinotSchema(testPinotSchema, pinotConfig.isInferDateTypeInSchema(), pinotConfig.isInferTimestampTypeInSchema());
    for (PinotColumn column : pinotColumns) {
        assertEquals(column.getType(), expectedTypeMap.get(column.getName()), "Failed to compare column type for field - " + column.getName());
        assertEquals(column.getComment(), expectedComment.get(column.getName()), "Failed to compare column comment for field - " + column.getName());
        assertEquals(column.isNullable(), false);
    }
}