Examples with DataSchema com.linkedin.pinot.common.utils.DataSchema used on opensource projects

Example 6 with DataSchema

use of com.linkedin.pinot.common.utils.DataSchema in project pinot by linkedin.

the class InnerSegmentSelectionSingleValueQueriesTest method testSelectionOnly.

@Test
public void testSelectionOnly() {
    String query = "SELECT" + SELECTION + " FROM testTable";
    // Test query without filter.
    MSelectionOnlyOperator selectionOnlyOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 30L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 3);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(2), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    List<Serializable[]> selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 3);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 1578964907);
    Assert.assertEquals((String) firstRow[2], "P");
    // Test query with filter.
    selectionOnlyOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 48241L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 30L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 3);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(2), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 3);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 351823652);
    Assert.assertEquals((String) firstRow[2], "t");
}

Example 7 with DataSchema

use of com.linkedin.pinot.common.utils.DataSchema in project pinot by linkedin.

the class InnerSegmentSelectionSingleValueQueriesTest method testSelectionOrderBy.

@Test
public void testSelectionOrderBy() {
    String query = "SELECT" + SELECTION + " FROM testTable" + ORDER_BY;
    // Test query without filter.
    MSelectionOrderByOperator selectionOrderByOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 30000L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 120000L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.INT);
    Queue<Serializable[]> selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals(((Integer) lastRow[0]).intValue(), 6043515);
    Assert.assertEquals(((Integer) lastRow[1]).intValue(), 10542595);
    // Test query with filter.
    selectionOrderByOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 6129L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 84134L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 24516L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.INT);
    selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals(((Integer) lastRow[0]).intValue(), 6043515);
    Assert.assertEquals(((Integer) lastRow[1]).intValue(), 462769197);
}

Example 8 with DataSchema

use of com.linkedin.pinot.common.utils.DataSchema in project pinot by linkedin.

the class InnerSegmentSelectionSingleValueQueriesTest method testSelectStar.

@Test
public void testSelectStar() {
    String query = "SELECT * FROM testTable";
    // Test query without filter.
    MSelectionOnlyOperator selectionOnlyOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 110L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 11);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    List<Serializable[]> selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 11);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 1578964907);
    Assert.assertEquals((String) firstRow[1], "P");
    // Test query with filter.
    selectionOnlyOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOnlyOperator.nextBlock();
    executionStatistics = selectionOnlyOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 10L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 48241L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 110L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 11);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column11");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.STRING);
    selectionResult = (List<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    firstRow = selectionResult.get(0);
    Assert.assertEquals(firstRow.length, 11);
    Assert.assertEquals(((Integer) firstRow[0]).intValue(), 351823652);
    Assert.assertEquals((String) firstRow[1], "t");
}

Example 9 with DataSchema

use of com.linkedin.pinot.common.utils.DataSchema in project pinot by linkedin.

the class DataTableSerDeTest method testAllDataTypes.

@Test
public void testAllDataTypes() throws IOException {
    DataType[] columnTypes = DataType.values();
    int numColumns = columnTypes.length;
    String[] columnNames = new String[numColumns];
    for (int i = 0; i < numColumns; i++) {
        columnNames[i] = columnTypes[i].name();
    }
    DataSchema dataSchema = new DataSchema(columnNames, columnTypes);
    DataTableBuilder dataTableBuilder = new DataTableBuilder(dataSchema);
    boolean[] booleans = new boolean[NUM_ROWS];
    byte[] bytes = new byte[NUM_ROWS];
    char[] chars = new char[NUM_ROWS];
    short[] shorts = new short[NUM_ROWS];
    int[] ints = new int[NUM_ROWS];
    long[] longs = new long[NUM_ROWS];
    float[] floats = new float[NUM_ROWS];
    double[] doubles = new double[NUM_ROWS];
    String[] strings = new String[NUM_ROWS];
    Object[] objects = new Object[NUM_ROWS];
    byte[][] byteArrays = new byte[NUM_ROWS][];
    char[][] charArrays = new char[NUM_ROWS][];
    short[][] shortArrays = new short[NUM_ROWS][];
    int[][] intArrays = new int[NUM_ROWS][];
    long[][] longArrays = new long[NUM_ROWS][];
    float[][] floatArrays = new float[NUM_ROWS][];
    double[][] doubleArrays = new double[NUM_ROWS][];
    String[][] stringArrays = new String[NUM_ROWS][];
    for (int rowId = 0; rowId < NUM_ROWS; rowId++) {
        dataTableBuilder.startRow();
        for (int colId = 0; colId < numColumns; colId++) {
            switch(columnTypes[colId]) {
                case BOOLEAN:
                    booleans[rowId] = RANDOM.nextBoolean();
                    dataTableBuilder.setColumn(colId, booleans[rowId]);
                    break;
                case BYTE:
                    bytes[rowId] = (byte) RANDOM.nextInt();
                    dataTableBuilder.setColumn(colId, bytes[rowId]);
                    break;
                case CHAR:
                    chars[rowId] = (char) RANDOM.nextInt();
                    dataTableBuilder.setColumn(colId, chars[rowId]);
                    break;
                case SHORT:
                    shorts[rowId] = (short) RANDOM.nextInt();
                    dataTableBuilder.setColumn(colId, shorts[rowId]);
                    break;
                case INT:
                    ints[rowId] = RANDOM.nextInt();
                    dataTableBuilder.setColumn(colId, ints[rowId]);
                    break;
                case LONG:
                    longs[rowId] = RANDOM.nextLong();
                    dataTableBuilder.setColumn(colId, longs[rowId]);
                    break;
                case FLOAT:
                    floats[rowId] = RANDOM.nextFloat();
                    dataTableBuilder.setColumn(colId, floats[rowId]);
                    break;
                case DOUBLE:
                    doubles[rowId] = RANDOM.nextDouble();
                    dataTableBuilder.setColumn(colId, doubles[rowId]);
                    break;
                case STRING:
                    strings[rowId] = RandomStringUtils.random(RANDOM.nextInt(20));
                    dataTableBuilder.setColumn(colId, strings[rowId]);
                    break;
                // Just test Double here, all object types will be covered in ObjectCustomSerDeTest.
                case OBJECT:
                    objects[rowId] = RANDOM.nextDouble();
                    dataTableBuilder.setColumn(colId, objects[rowId]);
                    break;
                case BYTE_ARRAY:
                    int length = RANDOM.nextInt(20);
                    byte[] byteArray = new byte[length];
                    for (int i = 0; i < length; i++) {
                        byteArray[i] = (byte) RANDOM.nextInt();
                    }
                    byteArrays[rowId] = byteArray;
                    dataTableBuilder.setColumn(colId, byteArray);
                    break;
                case CHAR_ARRAY:
                    length = RANDOM.nextInt(20);
                    char[] charArray = new char[length];
                    for (int i = 0; i < length; i++) {
                        charArray[i] = (char) RANDOM.nextInt();
                    }
                    charArrays[rowId] = charArray;
                    dataTableBuilder.setColumn(colId, charArray);
                    break;
                case SHORT_ARRAY:
                    length = RANDOM.nextInt(20);
                    short[] shortArray = new short[length];
                    for (int i = 0; i < length; i++) {
                        shortArray[i] = (short) RANDOM.nextInt();
                    }
                    shortArrays[rowId] = shortArray;
                    dataTableBuilder.setColumn(colId, shortArray);
                    break;
                case INT_ARRAY:
                    length = RANDOM.nextInt(20);
                    int[] intArray = new int[length];
                    for (int i = 0; i < length; i++) {
                        intArray[i] = RANDOM.nextInt();
                    }
                    intArrays[rowId] = intArray;
                    dataTableBuilder.setColumn(colId, intArray);
                    break;
                case LONG_ARRAY:
                    length = RANDOM.nextInt(20);
                    long[] longArray = new long[length];
                    for (int i = 0; i < length; i++) {
                        longArray[i] = RANDOM.nextLong();
                    }
                    longArrays[rowId] = longArray;
                    dataTableBuilder.setColumn(colId, longArray);
                    break;
                case FLOAT_ARRAY:
                    length = RANDOM.nextInt(20);
                    float[] floatArray = new float[length];
                    for (int i = 0; i < length; i++) {
                        floatArray[i] = RANDOM.nextFloat();
                    }
                    floatArrays[rowId] = floatArray;
                    dataTableBuilder.setColumn(colId, floatArray);
                    break;
                case DOUBLE_ARRAY:
                    length = RANDOM.nextInt(20);
                    double[] doubleArray = new double[length];
                    for (int i = 0; i < length; i++) {
                        doubleArray[i] = RANDOM.nextDouble();
                    }
                    doubleArrays[rowId] = doubleArray;
                    dataTableBuilder.setColumn(colId, doubleArray);
                    break;
                case STRING_ARRAY:
                    length = RANDOM.nextInt(20);
                    String[] stringArray = new String[length];
                    for (int i = 0; i < length; i++) {
                        stringArray[i] = RandomStringUtils.random(RANDOM.nextInt(20));
                    }
                    stringArrays[rowId] = stringArray;
                    dataTableBuilder.setColumn(colId, stringArray);
                    break;
            }
        }
        dataTableBuilder.finishRow();
    }
    DataTable dataTable = dataTableBuilder.build();
    DataTable newDataTable = DataTableFactory.getDataTable(dataTable.toBytes());
    Assert.assertEquals(newDataTable.getDataSchema(), dataSchema, ERROR_MESSAGE);
    Assert.assertEquals(newDataTable.getNumberOfRows(), NUM_ROWS, ERROR_MESSAGE);
    for (int rowId = 0; rowId < NUM_ROWS; rowId++) {
        for (int colId = 0; colId < numColumns; colId++) {
            switch(columnTypes[colId]) {
                case BOOLEAN:
                    Assert.assertEquals(newDataTable.getBoolean(rowId, colId), booleans[rowId], ERROR_MESSAGE);
                    break;
                case BYTE:
                    Assert.assertEquals(newDataTable.getByte(rowId, colId), bytes[rowId], ERROR_MESSAGE);
                    break;
                case CHAR:
                    Assert.assertEquals(newDataTable.getChar(rowId, colId), chars[rowId], ERROR_MESSAGE);
                    break;
                case SHORT:
                    Assert.assertEquals(newDataTable.getShort(rowId, colId), shorts[rowId], ERROR_MESSAGE);
                    break;
                case INT:
                    Assert.assertEquals(newDataTable.getInt(rowId, colId), ints[rowId], ERROR_MESSAGE);
                    break;
                case LONG:
                    Assert.assertEquals(newDataTable.getLong(rowId, colId), longs[rowId], ERROR_MESSAGE);
                    break;
                case FLOAT:
                    Assert.assertEquals(newDataTable.getFloat(rowId, colId), floats[rowId], ERROR_MESSAGE);
                    break;
                case DOUBLE:
                    Assert.assertEquals(newDataTable.getDouble(rowId, colId), doubles[rowId], ERROR_MESSAGE);
                    break;
                case STRING:
                    Assert.assertEquals(newDataTable.getString(rowId, colId), strings[rowId], ERROR_MESSAGE);
                    break;
                case OBJECT:
                    Assert.assertEquals(newDataTable.getObject(rowId, colId), objects[rowId], ERROR_MESSAGE);
                    break;
                case BYTE_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getByteArray(rowId, colId), byteArrays[rowId]), ERROR_MESSAGE);
                    break;
                case CHAR_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getCharArray(rowId, colId), charArrays[rowId]), ERROR_MESSAGE);
                    break;
                case SHORT_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getShortArray(rowId, colId), shortArrays[rowId]), ERROR_MESSAGE);
                    break;
                case INT_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getIntArray(rowId, colId), intArrays[rowId]), ERROR_MESSAGE);
                    break;
                case LONG_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getLongArray(rowId, colId), longArrays[rowId]), ERROR_MESSAGE);
                    break;
                case FLOAT_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getFloatArray(rowId, colId), floatArrays[rowId]), ERROR_MESSAGE);
                    break;
                case DOUBLE_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getDoubleArray(rowId, colId), doubleArrays[rowId]), ERROR_MESSAGE);
                    break;
                case STRING_ARRAY:
                    Assert.assertTrue(Arrays.equals(newDataTable.getStringArray(rowId, colId), stringArrays[rowId]), ERROR_MESSAGE);
                    break;
            }
        }
    }
}

Example 10 with DataSchema

use of com.linkedin.pinot.common.utils.DataSchema in project pinot by linkedin.

the class SelectionOperatorServiceTest method testCompatibleRowsDataTableTransformation.

@Test
public void testCompatibleRowsDataTableTransformation() throws Exception {
    Collection<Serializable[]> rows = new ArrayList<>(2);
    rows.add(_row1.clone());
    rows.add(_compatibleRow1.clone());
    DataSchema dataSchema = _dataSchema.clone();
    Assert.assertTrue(dataSchema.isTypeCompatibleWith(_compatibleDataSchema));
    dataSchema.upgradeToCover(_compatibleDataSchema);
    Assert.assertEquals(dataSchema, _upgradedDataSchema);
    DataTable dataTable = SelectionOperatorUtils.getDataTableFromRows(rows, dataSchema);
    Serializable[] expectedRow1 = { 0L, 1.0, 2.0, 3.0, "4", new long[] { 5L }, new double[] { 6.0 }, new double[] { 7.0 }, new double[] { 8.0 }, new String[] { "9" } };
    Serializable[] expectedCompatibleRow1 = { 1L, 2.0, 3.0, 4.0, "5", new long[] { 6L }, new double[] { 7.0 }, new double[] { 8.0 }, new double[] { 9.0 }, new String[] { "10" } };
    Assert.assertEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 0), expectedRow1);
    Assert.assertEquals(SelectionOperatorUtils.extractRowFromDataTable(dataTable, 1), expectedCompatibleRow1);
}