Examples with FieldSpec com.linkedin.pinot.common.data.FieldSpec used on opensource projects

Example 26 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class BitmapInvertedIndexCreatorTest method testSingleValue.

@Test
public void testSingleValue() throws IOException {
    boolean singleValue = true;
    String colName = "single_value_col";
    FieldSpec spec = new DimensionFieldSpec(colName, DataType.INT, singleValue);
    int numDocs = 20;
    int[] data = new int[numDocs];
    int cardinality = 10;
    File indexDirHeap = new File("/tmp/indexDirHeap");
    FileUtils.forceMkdir(indexDirHeap);
    indexDirHeap.mkdirs();
    File indexDirOffHeap = new File("/tmp/indexDirOffHeap");
    FileUtils.forceMkdir(indexDirOffHeap);
    indexDirOffHeap.mkdirs();
    File bitmapIndexFileOffHeap = new File(indexDirOffHeap, colName + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
    File bitmapIndexFileHeap = new File(indexDirHeap, colName + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
    // GENERATE RANDOM DATA SET
    Random r = new Random();
    Map<Integer, Set<Integer>> postingListMap = new HashMap<>();
    for (int i = 0; i < cardinality; i++) {
        postingListMap.put(i, new LinkedHashSet<Integer>());
    }
    for (int i = 0; i < numDocs; i++) {
        data[i] = r.nextInt(cardinality);
        LOGGER.debug("docId:" + i + "  dictId:" + data[i]);
        postingListMap.get(data[i]).add(i);
    }
    for (int i = 0; i < cardinality; i++) {
        LOGGER.debug("Posting list for " + i + " : " + postingListMap.get(i));
    }
    // GENERATE BITMAP USING OffHeapCreator and validate
    OffHeapBitmapInvertedIndexCreator offHeapCreator = new OffHeapBitmapInvertedIndexCreator(indexDirOffHeap, cardinality, numDocs, numDocs, spec);
    for (int i = 0; i < numDocs; i++) {
        offHeapCreator.add(i, data[i]);
    }
    offHeapCreator.seal();
    validate(colName, bitmapIndexFileOffHeap, cardinality, postingListMap);
    // GENERATE BITMAP USING HeapCreator and validate
    HeapBitmapInvertedIndexCreator heapCreator = new HeapBitmapInvertedIndexCreator(indexDirHeap, cardinality, numDocs, 0, spec);
    for (int i = 0; i < numDocs; i++) {
        heapCreator.add(i, data[i]);
    }
    heapCreator.seal();
    validate(colName, bitmapIndexFileHeap, cardinality, postingListMap);
    // assert that the file sizes and contents are the same
    Assert.assertEquals(bitmapIndexFileHeap.length(), bitmapIndexFileHeap.length());
    Assert.assertTrue(FileUtils.contentEquals(bitmapIndexFileHeap, bitmapIndexFileHeap));
    FileUtils.deleteQuietly(indexDirHeap);
    FileUtils.deleteQuietly(indexDirOffHeap);
}

Example 27 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class SegmentTestUtils method extractSchemaFromAvroWithoutTime.

public static Schema extractSchemaFromAvroWithoutTime(File avroFile) throws FileNotFoundException, IOException {
    DataFileStream<GenericRecord> dataStream = new DataFileStream<GenericRecord>(new FileInputStream(avroFile), new GenericDatumReader<GenericRecord>());
    Schema schema = new Schema();
    for (final Field field : dataStream.getSchema().getFields()) {
        try {
            getColumnType(field);
        } catch (Exception e) {
            LOGGER.warn("Caught exception while converting Avro field {} of type {}, field will not be in schema.", field.name(), field.schema().getType());
            continue;
        }
        final String columnName = field.name();
        final String pinotType = field.getProp("pinotType");
        final FieldSpec fieldSpec;
        if (pinotType != null && "METRIC".equals(pinotType)) {
            fieldSpec = new MetricFieldSpec();
        } else {
            fieldSpec = new DimensionFieldSpec();
        }
        fieldSpec.setName(columnName);
        fieldSpec.setDataType(getColumnType(dataStream.getSchema().getField(columnName)));
        fieldSpec.setSingleValueField(isSingleValueField(dataStream.getSchema().getField(columnName)));
        schema.addField(fieldSpec);
    }
    dataStream.close();
    return schema;
}

Example 28 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class BaseDefaultColumnHandler method computeDefaultColumnActionMap.

/**
   * Compute the action needed for each column.
   * This method compares the column metadata across schema and segment.
   *
   * @return Action Map for each column.
   */
private Map<String, DefaultColumnAction> computeDefaultColumnActionMap() {
    Map<String, DefaultColumnAction> defaultColumnActionMap = new HashMap<>();
    // Compute ADD and UPDATE actions.
    Collection<String> columnsInSchema = _schema.getColumnNames();
    for (String column : columnsInSchema) {
        FieldSpec fieldSpecInSchema = _schema.getFieldSpecFor(column);
        Preconditions.checkNotNull(fieldSpecInSchema);
        FieldSpec.FieldType fieldTypeInSchema = fieldSpecInSchema.getFieldType();
        ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
        if (columnMetadata != null) {
            // Only check for auto-generated column.
            if (!columnMetadata.isAutoGenerated()) {
                continue;
            }
            // Check the field type matches.
            FieldSpec.FieldType fieldTypeInMetadata = columnMetadata.getFieldType();
            if (fieldTypeInMetadata != fieldTypeInSchema) {
                String failureMessage = "Field type: " + fieldTypeInMetadata + " for auto-generated column: " + column + " does not match field type: " + fieldTypeInSchema + " in schema, throw exception to drop and re-download the segment.";
                throw new RuntimeException(failureMessage);
            }
            // Check the data type and default value matches.
            FieldSpec.DataType dataTypeInMetadata = columnMetadata.getDataType();
            FieldSpec.DataType dataTypeInSchema = fieldSpecInSchema.getDataType();
            boolean isSingleValueInMetadata = columnMetadata.isSingleValue();
            boolean isSingleValueInSchema = fieldSpecInSchema.isSingleValueField();
            String defaultValueInMetadata = columnMetadata.getDefaultNullValueString();
            String defaultValueInSchema = fieldSpecInSchema.getDefaultNullValue().toString();
            if (dataTypeInMetadata != dataTypeInSchema || isSingleValueInMetadata != isSingleValueInSchema || !defaultValueInSchema.equals(defaultValueInMetadata)) {
                if (fieldTypeInMetadata == FieldSpec.FieldType.DIMENSION) {
                    defaultColumnActionMap.put(column, DefaultColumnAction.UPDATE_DIMENSION);
                } else {
                    Preconditions.checkState(fieldTypeInMetadata == FieldSpec.FieldType.METRIC);
                    defaultColumnActionMap.put(column, DefaultColumnAction.UPDATE_METRIC);
                }
            }
        } else {
            switch(fieldTypeInSchema) {
                case DIMENSION:
                    defaultColumnActionMap.put(column, DefaultColumnAction.ADD_DIMENSION);
                    break;
                case METRIC:
                    defaultColumnActionMap.put(column, DefaultColumnAction.ADD_METRIC);
                    break;
                default:
                    LOGGER.warn("Skip adding default column for column: {} with field type: {}", column, fieldTypeInSchema);
                    break;
            }
        }
    }
    // Compute REMOVE actions.
    Set<String> columnsInMetadata = _segmentMetadata.getAllColumns();
    for (String column : columnsInMetadata) {
        if (!columnsInSchema.contains(column)) {
            ColumnMetadata columnMetadata = _segmentMetadata.getColumnMetadataFor(column);
            // Only remove auto-generated columns.
            if (columnMetadata.isAutoGenerated()) {
                FieldSpec.FieldType fieldTypeInMetadata = columnMetadata.getFieldType();
                if (fieldTypeInMetadata == FieldSpec.FieldType.DIMENSION) {
                    defaultColumnActionMap.put(column, DefaultColumnAction.REMOVE_DIMENSION);
                } else {
                    Preconditions.checkState(fieldTypeInMetadata == FieldSpec.FieldType.METRIC);
                    defaultColumnActionMap.put(column, DefaultColumnAction.REMOVE_METRIC);
                }
            }
        }
    }
    return defaultColumnActionMap;
}

Example 29 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class BaseDefaultColumnHandler method createColumnV1Indices.

/**
   * Helper method to create the V1 indices (dictionary and forward index) for a column.
   *
   * @param column column name.
   */
protected void createColumnV1Indices(String column) throws Exception {
    FieldSpec fieldSpec = _schema.getFieldSpecFor(column);
    Preconditions.checkNotNull(fieldSpec);
    // Generate column index creation information.
    int totalDocs = _segmentMetadata.getTotalDocs();
    int totalRawDocs = _segmentMetadata.getTotalRawDocs();
    int totalAggDocs = totalDocs - totalRawDocs;
    FieldSpec.DataType dataType = fieldSpec.getDataType();
    Object defaultValue = fieldSpec.getDefaultNullValue();
    boolean isSingleValue = fieldSpec.isSingleValueField();
    int maxNumberOfMultiValueElements = isSingleValue ? 0 : 1;
    int dictionaryElementSize = 0;
    Object sortedArray;
    switch(dataType) {
        case STRING:
            Preconditions.checkState(defaultValue instanceof String);
            String stringDefaultValue = (String) defaultValue;
            // Length of the UTF-8 encoded byte array.
            dictionaryElementSize = stringDefaultValue.getBytes("UTF8").length;
            sortedArray = new String[] { stringDefaultValue };
            break;
        case INT:
            Preconditions.checkState(defaultValue instanceof Integer);
            sortedArray = new int[] { (Integer) defaultValue };
            break;
        case LONG:
            Preconditions.checkState(defaultValue instanceof Long);
            sortedArray = new long[] { (Long) defaultValue };
            break;
        case FLOAT:
            Preconditions.checkState(defaultValue instanceof Float);
            sortedArray = new float[] { (Float) defaultValue };
            break;
        case DOUBLE:
            Preconditions.checkState(defaultValue instanceof Double);
            sortedArray = new double[] { (Double) defaultValue };
            break;
        default:
            throw new UnsupportedOperationException("Unsupported data type: " + dataType + " for column: " + column);
    }
    ColumnIndexCreationInfo columnIndexCreationInfo = new ColumnIndexCreationInfo(true, /*createDictionary*/
    defaultValue, /*min*/
    defaultValue, /*max*/
    sortedArray, ForwardIndexType.FIXED_BIT_COMPRESSED, InvertedIndexType.SORTED_INDEX, isSingleValue, /*isSortedColumn*/
    false, /*hasNulls*/
    totalDocs, /*totalNumberOfEntries*/
    maxNumberOfMultiValueElements, -1, /* Unused max length*/
    true, /*isAutoGenerated*/
    defaultValue);
    // Create dictionary.
    // We will have only one value in the dictionary.
    SegmentDictionaryCreator segmentDictionaryCreator = new SegmentDictionaryCreator(false, /*hasNulls*/
    sortedArray, fieldSpec, _indexDir, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
    segmentDictionaryCreator.build(new boolean[] { true });
    segmentDictionaryCreator.close();
    // Create forward index.
    if (isSingleValue) {
        // Single-value column.
        SingleValueSortedForwardIndexCreator svFwdIndexCreator = new SingleValueSortedForwardIndexCreator(_indexDir, 1, /*cardinality*/
        fieldSpec);
        for (int docId = 0; docId < totalDocs; docId++) {
            svFwdIndexCreator.add(0, /*dictionaryId*/
            docId);
        }
        svFwdIndexCreator.close();
    } else {
        // Multi-value column.
        MultiValueUnsortedForwardIndexCreator mvFwdIndexCreator = new MultiValueUnsortedForwardIndexCreator(fieldSpec, _indexDir, 1, /*cardinality*/
        totalDocs, /*numDocs*/
        totalDocs, /*totalNumberOfValues*/
        false);
        int[] dictionaryIds = { 0 };
        for (int docId = 0; docId < totalDocs; docId++) {
            mvFwdIndexCreator.index(docId, dictionaryIds);
        }
        mvFwdIndexCreator.close();
    }
    // Add the column metadata information to the metadata properties.
    SegmentColumnarIndexCreator.addColumnMetadataInfo(_segmentProperties, column, columnIndexCreationInfo, totalDocs, totalRawDocs, totalAggDocs, fieldSpec, true, /*hasDictionary*/
    dictionaryElementSize, true, /*hasInvertedIndex*/
    null);
}

Example 30 with FieldSpec

use of com.linkedin.pinot.common.data.FieldSpec in project pinot by linkedin.

the class V3DefaultColumnHandler method updateDefaultColumn.

@Override
protected void updateDefaultColumn(String column, DefaultColumnAction action) throws Exception {
    LOGGER.info("Starting default column action: {} on column: {}", action, column);
    // Throw exception to drop and re-download the segment.
    if (action.isRemoveAction()) {
        throw new V3RemoveIndexException("Default value indices for column: " + column + " cannot be removed for V3 format segment.");
    }
    // Delete existing dictionary and forward index for the column. For V3, this is for error handling.
    removeColumnV1Indices(column);
    // For ADD and UPDATE action, need to create new dictionary and forward index, and update column metadata.
    if (!action.isRemoveAction()) {
        createColumnV1Indices(column);
        // Write index to V3 format.
        FieldSpec fieldSpec = _schema.getFieldSpecFor(column);
        Preconditions.checkNotNull(fieldSpec);
        boolean isSingleValue = fieldSpec.isSingleValueField();
        File dictionaryFile = new File(_indexDir, column + V1Constants.Dict.FILE_EXTENTION);
        File forwardIndexFile;
        if (isSingleValue) {
            forwardIndexFile = new File(_indexDir, column + V1Constants.Indexes.SORTED_FWD_IDX_FILE_EXTENTION);
        } else {
            forwardIndexFile = new File(_indexDir, column + V1Constants.Indexes.UN_SORTED_MV_FWD_IDX_FILE_EXTENTION);
        }
        LoaderUtils.writeIndexToV3Format(_segmentWriter, column, dictionaryFile, ColumnIndexType.DICTIONARY);
        LoaderUtils.writeIndexToV3Format(_segmentWriter, column, forwardIndexFile, ColumnIndexType.FORWARD_INDEX);
    }
}