Examples with DimensionFieldSpec com.linkedin.pinot.common.data.DimensionFieldSpec used on opensource projects

Example 11 with DimensionFieldSpec

use of com.linkedin.pinot.common.data.DimensionFieldSpec in project pinot by linkedin.

the class DictionariesTest method testLongColumnPreIndexStatsCollector.

@Test
public void testLongColumnPreIndexStatsCollector() throws Exception {
    FieldSpec spec = new DimensionFieldSpec("column1", DataType.LONG, true);
    AbstractColumnStatisticsCollector statsCollector = new LongColumnPreIndexStatsCollector(spec);
    statsCollector.collect(new Integer(1));
    Assert.assertTrue(statsCollector.isSorted());
    statsCollector.collect(new Float(2));
    Assert.assertTrue(statsCollector.isSorted());
    statsCollector.collect(new Long(3));
    Assert.assertTrue(statsCollector.isSorted());
    statsCollector.collect(new Double(4));
    Assert.assertTrue(statsCollector.isSorted());
    statsCollector.collect(new Integer(4));
    Assert.assertTrue(statsCollector.isSorted());
    statsCollector.collect(new Float(2));
    Assert.assertFalse(statsCollector.isSorted());
    statsCollector.collect(new Double(40));
    Assert.assertFalse(statsCollector.isSorted());
    statsCollector.collect(new Double(20));
    Assert.assertFalse(statsCollector.isSorted());
    statsCollector.seal();
    Assert.assertEquals(statsCollector.getCardinality(), 6);
    Assert.assertEquals(((Number) statsCollector.getMinValue()).intValue(), 1);
    Assert.assertEquals(((Number) statsCollector.getMaxValue()).intValue(), 40);
    Assert.assertFalse(statsCollector.isSorted());
}

Example 12 with DimensionFieldSpec

use of com.linkedin.pinot.common.data.DimensionFieldSpec in project pinot by linkedin.

the class DictionariesTest method testStringsValuesWithPadding.

/**
   * Tests DictionaryCreator for case when one value is a substring of another.
   * For example, in case of sorted values {"abc", "abc def"} after padding,
   * the sorted order would change to {"abc def%%%%", "abc%%%%%%%"}
   *
   * This test asserts that DictionaryCreator.indexOfSV("abc") returns 1 (ie index of "abc%%%%%%%"
   * in actual padded dictionary), and not 0.
   *
   * @throws Exception
   */
@Test
public void testStringsValuesWithPadding() throws Exception {
    File indexDir = new File("/tmp/dict.test");
    indexDir.deleteOnExit();
    FieldSpec fieldSpec = new DimensionFieldSpec("test", DataType.STRING, true);
    String[] inputStrings = new String[2];
    String[] paddedStrings = new String[2];
    char paddingChar = '%';
    inputStrings[0] = "abc def";
    inputStrings[1] = "abc";
    // Sorted order: {"abc", "abc def"}
    Arrays.sort(inputStrings);
    boolean[] isSorted = new boolean[1];
    isSorted[0] = true;
    SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(false, inputStrings, fieldSpec, indexDir, paddingChar);
    dictionaryCreator.build(isSorted);
    Assert.assertFalse(isSorted[0]);
    // Get the padded string as stored in the dictionary.
    int targetPaddedLength = dictionaryCreator.getStringColumnMaxLength();
    for (int i = 0; i < inputStrings.length; i++) {
        paddedStrings[i] = SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, paddingChar);
    }
    // Sorted Order: {"abc def%%%%", "abc%%%%%%%"}
    Arrays.sort(paddedStrings);
    // Assert that indexOfSV for un-padded string returns the index of the corresponding padded string.
    for (int i = 0; i < inputStrings.length; i++) {
        int paddedIndex = dictionaryCreator.indexOfSV(inputStrings[i]);
        Assert.assertTrue(paddedStrings[paddedIndex].equals(SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, paddingChar)));
    }
    dictionaryCreator.close();
    FileUtils.deleteQuietly(indexDir);
}

Example 13 with DimensionFieldSpec

use of com.linkedin.pinot.common.data.DimensionFieldSpec in project pinot by linkedin.

the class DictionariesTest method testSingleNullString.

/**
   * Tests SegmentDictionaryCreator for case when there is only one string
   * and it is "null"
   *
   * This test asserts that the padded length of the null string is 4
   *
   * @throws Exception
   */
@Test
public void testSingleNullString() throws Exception {
    File indexDir = new File("/tmp/dict.test");
    FieldSpec fieldSpec = new DimensionFieldSpec("test", DataType.STRING, true);
    String[] inputStrings = new String[1];
    String[] paddedStrings = new String[1];
    inputStrings[0] = "null";
    // Sorted order: {"null"}
    Arrays.sort(inputStrings);
    try {
        SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(false, inputStrings, fieldSpec, indexDir, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
        boolean[] isSorted = new boolean[1];
        isSorted[0] = true;
        dictionaryCreator.build(isSorted);
        // Get the padded string as stored in the dictionary.
        int targetPaddedLength = dictionaryCreator.getStringColumnMaxLength();
        Assert.assertTrue(targetPaddedLength == 4);
        for (int i = 0; i < inputStrings.length; i++) {
            paddedStrings[i] = SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
        }
        // Sorted Order: {"null"}
        Arrays.sort(paddedStrings);
        // Assert that indexOfSV for un-padded string returns the index of the corresponding padded string.
        for (int i = 0; i < inputStrings.length; i++) {
            int paddedIndex = dictionaryCreator.indexOfSV(inputStrings[i]);
            Assert.assertTrue(paddedStrings[paddedIndex].equals(SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, V1Constants.Str.DEFAULT_STRING_PAD_CHAR)));
        }
        // Verify that the string "null" did not get changed
        Assert.assertTrue(paddedStrings[0].equals("null"));
        dictionaryCreator.close();
    } catch (Exception e) {
        throw e;
    } finally {
        FileUtils.deleteQuietly(indexDir);
    }
}

Example 14 with DimensionFieldSpec

use of com.linkedin.pinot.common.data.DimensionFieldSpec in project pinot by linkedin.

the class BitmapInvertedIndexCreatorTest method testSingleValue.

@Test
public void testSingleValue() throws IOException {
    boolean singleValue = true;
    String colName = "single_value_col";
    FieldSpec spec = new DimensionFieldSpec(colName, DataType.INT, singleValue);
    int numDocs = 20;
    int[] data = new int[numDocs];
    int cardinality = 10;
    File indexDirHeap = new File("/tmp/indexDirHeap");
    FileUtils.forceMkdir(indexDirHeap);
    indexDirHeap.mkdirs();
    File indexDirOffHeap = new File("/tmp/indexDirOffHeap");
    FileUtils.forceMkdir(indexDirOffHeap);
    indexDirOffHeap.mkdirs();
    File bitmapIndexFileOffHeap = new File(indexDirOffHeap, colName + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
    File bitmapIndexFileHeap = new File(indexDirHeap, colName + V1Constants.Indexes.BITMAP_INVERTED_INDEX_FILE_EXTENSION);
    // GENERATE RANDOM DATA SET
    Random r = new Random();
    Map<Integer, Set<Integer>> postingListMap = new HashMap<>();
    for (int i = 0; i < cardinality; i++) {
        postingListMap.put(i, new LinkedHashSet<Integer>());
    }
    for (int i = 0; i < numDocs; i++) {
        data[i] = r.nextInt(cardinality);
        LOGGER.debug("docId:" + i + "  dictId:" + data[i]);
        postingListMap.get(data[i]).add(i);
    }
    for (int i = 0; i < cardinality; i++) {
        LOGGER.debug("Posting list for " + i + " : " + postingListMap.get(i));
    }
    // GENERATE BITMAP USING OffHeapCreator and validate
    OffHeapBitmapInvertedIndexCreator offHeapCreator = new OffHeapBitmapInvertedIndexCreator(indexDirOffHeap, cardinality, numDocs, numDocs, spec);
    for (int i = 0; i < numDocs; i++) {
        offHeapCreator.add(i, data[i]);
    }
    offHeapCreator.seal();
    validate(colName, bitmapIndexFileOffHeap, cardinality, postingListMap);
    // GENERATE BITMAP USING HeapCreator and validate
    HeapBitmapInvertedIndexCreator heapCreator = new HeapBitmapInvertedIndexCreator(indexDirHeap, cardinality, numDocs, 0, spec);
    for (int i = 0; i < numDocs; i++) {
        heapCreator.add(i, data[i]);
    }
    heapCreator.seal();
    validate(colName, bitmapIndexFileHeap, cardinality, postingListMap);
    // assert that the file sizes and contents are the same
    Assert.assertEquals(bitmapIndexFileHeap.length(), bitmapIndexFileHeap.length());
    Assert.assertTrue(FileUtils.contentEquals(bitmapIndexFileHeap, bitmapIndexFileHeap));
    FileUtils.deleteQuietly(indexDirHeap);
    FileUtils.deleteQuietly(indexDirOffHeap);
}

Example 15 with DimensionFieldSpec

use of com.linkedin.pinot.common.data.DimensionFieldSpec in project pinot by linkedin.

the class SegmentTestUtils method extractSchemaFromAvroWithoutTime.

public static Schema extractSchemaFromAvroWithoutTime(File avroFile) throws FileNotFoundException, IOException {
    DataFileStream<GenericRecord> dataStream = new DataFileStream<GenericRecord>(new FileInputStream(avroFile), new GenericDatumReader<GenericRecord>());
    Schema schema = new Schema();
    for (final Field field : dataStream.getSchema().getFields()) {
        try {
            getColumnType(field);
        } catch (Exception e) {
            LOGGER.warn("Caught exception while converting Avro field {} of type {}, field will not be in schema.", field.name(), field.schema().getType());
            continue;
        }
        final String columnName = field.name();
        final String pinotType = field.getProp("pinotType");
        final FieldSpec fieldSpec;
        if (pinotType != null && "METRIC".equals(pinotType)) {
            fieldSpec = new MetricFieldSpec();
        } else {
            fieldSpec = new DimensionFieldSpec();
        }
        fieldSpec.setName(columnName);
        fieldSpec.setDataType(getColumnType(dataStream.getSchema().getField(columnName)));
        fieldSpec.setSingleValueField(isSingleValueField(dataStream.getSchema().getField(columnName)));
        schema.addField(fieldSpec);
    }
    dataStream.close();
    return schema;
}