Examples with SegmentDictionaryCreator com.linkedin.pinot.core.segment.creator.impl.SegmentDictionaryCreator used on opensource projects

Example 1 with SegmentDictionaryCreator

use of com.linkedin.pinot.core.segment.creator.impl.SegmentDictionaryCreator in project pinot by linkedin.

the class DictionariesTest method testUTF8Characters.

/**
   * Test for ensuring that Strings with special characters can be handled
   * correctly.
   *
   * @throws Exception
   */
@Test
public void testUTF8Characters() throws Exception {
    File indexDir = new File("/tmp/dict.test");
    indexDir.deleteOnExit();
    FieldSpec fieldSpec = new DimensionFieldSpec("test", DataType.STRING, true);
    String[] inputStrings = new String[3];
    char paddingChar = '%';
    // "Café";
    inputStrings[0] = new String(new byte[] { 67, 97, 102, -61, -87 });
    // "François";
    inputStrings[1] = new String(new byte[] { 70, 114, 97, 110, -61, -89, 111, 105, 115 });
    // "Côte d'Ivoire";
    inputStrings[2] = new String(new byte[] { 67, -61, -76, 116, 101, 32, 100, 39, 73, 118, 111, 105, 114, 101 });
    Arrays.sort(inputStrings);
    SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(false, inputStrings, fieldSpec, indexDir, paddingChar);
    dictionaryCreator.build(new boolean[] { false });
    for (String inputString : inputStrings) {
        Assert.assertTrue(dictionaryCreator.indexOfSV(inputString) >= 0, "Value not found in dictionary " + inputString);
    }
    dictionaryCreator.close();
    FileUtils.deleteQuietly(indexDir);
}

Example 2 with SegmentDictionaryCreator

use of com.linkedin.pinot.core.segment.creator.impl.SegmentDictionaryCreator in project pinot by linkedin.

the class DictionariesTest method testStringsValuesWithPadding.

/**
   * Tests DictionaryCreator for case when one value is a substring of another.
   * For example, in case of sorted values {"abc", "abc def"} after padding,
   * the sorted order would change to {"abc def%%%%", "abc%%%%%%%"}
   *
   * This test asserts that DictionaryCreator.indexOfSV("abc") returns 1 (ie index of "abc%%%%%%%"
   * in actual padded dictionary), and not 0.
   *
   * @throws Exception
   */
@Test
public void testStringsValuesWithPadding() throws Exception {
    File indexDir = new File("/tmp/dict.test");
    indexDir.deleteOnExit();
    FieldSpec fieldSpec = new DimensionFieldSpec("test", DataType.STRING, true);
    String[] inputStrings = new String[2];
    String[] paddedStrings = new String[2];
    char paddingChar = '%';
    inputStrings[0] = "abc def";
    inputStrings[1] = "abc";
    // Sorted order: {"abc", "abc def"}
    Arrays.sort(inputStrings);
    boolean[] isSorted = new boolean[1];
    isSorted[0] = true;
    SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(false, inputStrings, fieldSpec, indexDir, paddingChar);
    dictionaryCreator.build(isSorted);
    Assert.assertFalse(isSorted[0]);
    // Get the padded string as stored in the dictionary.
    int targetPaddedLength = dictionaryCreator.getStringColumnMaxLength();
    for (int i = 0; i < inputStrings.length; i++) {
        paddedStrings[i] = SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, paddingChar);
    }
    // Sorted Order: {"abc def%%%%", "abc%%%%%%%"}
    Arrays.sort(paddedStrings);
    // Assert that indexOfSV for un-padded string returns the index of the corresponding padded string.
    for (int i = 0; i < inputStrings.length; i++) {
        int paddedIndex = dictionaryCreator.indexOfSV(inputStrings[i]);
        Assert.assertTrue(paddedStrings[paddedIndex].equals(SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, paddingChar)));
    }
    dictionaryCreator.close();
    FileUtils.deleteQuietly(indexDir);
}

Example 3 with SegmentDictionaryCreator

use of com.linkedin.pinot.core.segment.creator.impl.SegmentDictionaryCreator in project pinot by linkedin.

the class DictionariesTest method testSingleNullString.

/**
   * Tests SegmentDictionaryCreator for case when there is only one string
   * and it is "null"
   *
   * This test asserts that the padded length of the null string is 4
   *
   * @throws Exception
   */
@Test
public void testSingleNullString() throws Exception {
    File indexDir = new File("/tmp/dict.test");
    FieldSpec fieldSpec = new DimensionFieldSpec("test", DataType.STRING, true);
    String[] inputStrings = new String[1];
    String[] paddedStrings = new String[1];
    inputStrings[0] = "null";
    // Sorted order: {"null"}
    Arrays.sort(inputStrings);
    try {
        SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(false, inputStrings, fieldSpec, indexDir, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
        boolean[] isSorted = new boolean[1];
        isSorted[0] = true;
        dictionaryCreator.build(isSorted);
        // Get the padded string as stored in the dictionary.
        int targetPaddedLength = dictionaryCreator.getStringColumnMaxLength();
        Assert.assertTrue(targetPaddedLength == 4);
        for (int i = 0; i < inputStrings.length; i++) {
            paddedStrings[i] = SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
        }
        // Sorted Order: {"null"}
        Arrays.sort(paddedStrings);
        // Assert that indexOfSV for un-padded string returns the index of the corresponding padded string.
        for (int i = 0; i < inputStrings.length; i++) {
            int paddedIndex = dictionaryCreator.indexOfSV(inputStrings[i]);
            Assert.assertTrue(paddedStrings[paddedIndex].equals(SegmentDictionaryCreator.getPaddedString(inputStrings[i], targetPaddedLength, V1Constants.Str.DEFAULT_STRING_PAD_CHAR)));
        }
        // Verify that the string "null" did not get changed
        Assert.assertTrue(paddedStrings[0].equals("null"));
        dictionaryCreator.close();
    } catch (Exception e) {
        throw e;
    } finally {
        FileUtils.deleteQuietly(indexDir);
    }
}

Example 4 with SegmentDictionaryCreator

use of com.linkedin.pinot.core.segment.creator.impl.SegmentDictionaryCreator in project pinot by linkedin.

the class BaseDefaultColumnHandler method createColumnV1Indices.

/**
   * Helper method to create the V1 indices (dictionary and forward index) for a column.
   *
   * @param column column name.
   */
protected void createColumnV1Indices(String column) throws Exception {
    FieldSpec fieldSpec = _schema.getFieldSpecFor(column);
    Preconditions.checkNotNull(fieldSpec);
    // Generate column index creation information.
    int totalDocs = _segmentMetadata.getTotalDocs();
    int totalRawDocs = _segmentMetadata.getTotalRawDocs();
    int totalAggDocs = totalDocs - totalRawDocs;
    FieldSpec.DataType dataType = fieldSpec.getDataType();
    Object defaultValue = fieldSpec.getDefaultNullValue();
    boolean isSingleValue = fieldSpec.isSingleValueField();
    int maxNumberOfMultiValueElements = isSingleValue ? 0 : 1;
    int dictionaryElementSize = 0;
    Object sortedArray;
    switch(dataType) {
        case STRING:
            Preconditions.checkState(defaultValue instanceof String);
            String stringDefaultValue = (String) defaultValue;
            // Length of the UTF-8 encoded byte array.
            dictionaryElementSize = stringDefaultValue.getBytes("UTF8").length;
            sortedArray = new String[] { stringDefaultValue };
            break;
        case INT:
            Preconditions.checkState(defaultValue instanceof Integer);
            sortedArray = new int[] { (Integer) defaultValue };
            break;
        case LONG:
            Preconditions.checkState(defaultValue instanceof Long);
            sortedArray = new long[] { (Long) defaultValue };
            break;
        case FLOAT:
            Preconditions.checkState(defaultValue instanceof Float);
            sortedArray = new float[] { (Float) defaultValue };
            break;
        case DOUBLE:
            Preconditions.checkState(defaultValue instanceof Double);
            sortedArray = new double[] { (Double) defaultValue };
            break;
        default:
            throw new UnsupportedOperationException("Unsupported data type: " + dataType + " for column: " + column);
    }
    ColumnIndexCreationInfo columnIndexCreationInfo = new ColumnIndexCreationInfo(true, /*createDictionary*/
    defaultValue, /*min*/
    defaultValue, /*max*/
    sortedArray, ForwardIndexType.FIXED_BIT_COMPRESSED, InvertedIndexType.SORTED_INDEX, isSingleValue, /*isSortedColumn*/
    false, /*hasNulls*/
    totalDocs, /*totalNumberOfEntries*/
    maxNumberOfMultiValueElements, -1, /* Unused max length*/
    true, /*isAutoGenerated*/
    defaultValue);
    // Create dictionary.
    // We will have only one value in the dictionary.
    SegmentDictionaryCreator segmentDictionaryCreator = new SegmentDictionaryCreator(false, /*hasNulls*/
    sortedArray, fieldSpec, _indexDir, V1Constants.Str.DEFAULT_STRING_PAD_CHAR);
    segmentDictionaryCreator.build(new boolean[] { true });
    segmentDictionaryCreator.close();
    // Create forward index.
    if (isSingleValue) {
        // Single-value column.
        SingleValueSortedForwardIndexCreator svFwdIndexCreator = new SingleValueSortedForwardIndexCreator(_indexDir, 1, /*cardinality*/
        fieldSpec);
        for (int docId = 0; docId < totalDocs; docId++) {
            svFwdIndexCreator.add(0, /*dictionaryId*/
            docId);
        }
        svFwdIndexCreator.close();
    } else {
        // Multi-value column.
        MultiValueUnsortedForwardIndexCreator mvFwdIndexCreator = new MultiValueUnsortedForwardIndexCreator(fieldSpec, _indexDir, 1, /*cardinality*/
        totalDocs, /*numDocs*/
        totalDocs, /*totalNumberOfValues*/
        false);
        int[] dictionaryIds = { 0 };
        for (int docId = 0; docId < totalDocs; docId++) {
            mvFwdIndexCreator.index(docId, dictionaryIds);
        }
        mvFwdIndexCreator.close();
    }
    // Add the column metadata information to the metadata properties.
    SegmentColumnarIndexCreator.addColumnMetadataInfo(_segmentProperties, column, columnIndexCreationInfo, totalDocs, totalRawDocs, totalAggDocs, fieldSpec, true, /*hasDictionary*/
    dictionaryElementSize, true, /*hasInvertedIndex*/
    null);
}

Example 5 with SegmentDictionaryCreator

use of com.linkedin.pinot.core.segment.creator.impl.SegmentDictionaryCreator in project pinot by linkedin.

the class DictionariesTest method testPaddedConflict.

/**
   * Tests SegmentDictionaryCreator for case when there is one empty string
   * and a string with a single padding character
   *
   * This test asserts that the padded length of the empty string is 1
   * in actual padded dictionary), and not 0.
   *
   * @throws Exception
   */
@Test
public void testPaddedConflict() throws Exception {
    File indexDir = new File("/tmp/dict.test");
    indexDir.deleteOnExit();
    FieldSpec fieldSpec = new DimensionFieldSpec("test", DataType.STRING, true);
    String[] inputStrings = new String[2];
    String[] paddedStrings = new String[2];
    char paddingChar = '%';
    try {
        inputStrings[0] = "";
        inputStrings[1] = "%";
        // Sorted order: {"", "%"}
        Arrays.sort(inputStrings);
        SegmentDictionaryCreator dictionaryCreator = new SegmentDictionaryCreator(false, inputStrings, fieldSpec, indexDir, paddingChar);
        boolean[] isSorted = new boolean[1];
        isSorted[0] = true;
        dictionaryCreator.build(isSorted);
    } catch (Exception e) {
        Assert.assertEquals(e.getMessage(), "Number of entries in dictionary != number of unique values in the data in column test");
    } finally {
        FileUtils.deleteQuietly(indexDir);
    }
}