Examples with ITypeTraits org.apache.hyracks.api.dataflow.value.ITypeTraits used on opensource projects

Example 56 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class SecondaryIndexOperationsHelper method setPrimaryRecDescAndComparators.

protected void setPrimaryRecDescAndComparators() throws AlgebricksException {
    List<List<String>> partitioningKeys = dataset.getPrimaryKeys();
    ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)];
    ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)];
    primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys];
    primaryBloomFilterKeyFields = new int[numPrimaryKeys];
    ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
    List<Integer> indicators = null;
    if (dataset.hasMetaPart()) {
        indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
    }
    for (int i = 0; i < numPrimaryKeys; i++) {
        IAType keyType = (indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i));
        primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType);
        primaryComparatorFactories[i] = BinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(keyType, true);
        primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType);
        primaryBloomFilterKeyFields[i] = i;
    }
    primaryRecFields[numPrimaryKeys] = payloadSerde;
    primaryTypeTraits[numPrimaryKeys] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType);
    if (dataset.hasMetaPart()) {
        primaryRecFields[numPrimaryKeys + 1] = payloadSerde;
        primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(metaType);
    }
    primaryRecDesc = new RecordDescriptor(primaryRecFields, primaryTypeTraits);
}

Example 57 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class SecondaryInvertedIndexOperationsHelper method setSecondaryRecDescAndComparators.

@Override
@SuppressWarnings("rawtypes")
protected void setSecondaryRecDescAndComparators() throws AlgebricksException {
    int numSecondaryKeys = index.getKeyFieldNames().size();
    IndexType indexType = index.getIndexType();
    boolean isEnforcingKeyTypes = index.isEnforcingKeyFileds();
    // Sanity checks.
    if (numPrimaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_FOR_DATASET_WITH_COMPOSITE_PRIMARY_INDEX, indexType, RecordUtil.toFullyQualifiedName(dataset.getDataverseName(), dataset.getDatasetName()));
    }
    if (numSecondaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD, numSecondaryKeys, indexType, 1);
    }
    if (indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX) {
        isPartitioned = true;
    } else {
        isPartitioned = false;
    }
    // Prepare record descriptor used in the assign op, and the optional
    // select op.
    secondaryFieldAccessEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeys + numFilterFields];
    ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys + numSecondaryKeys + numFilterFields];
    ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields];
    secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
    ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys];
    ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider();
    ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider();
    if (numSecondaryKeys > 0) {
        secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(isEnforcingKeyTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0), numPrimaryKeys);
        Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), itemType);
        secondaryKeyType = keyTypePair.first;
        anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
        ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(secondaryKeyType);
        secondaryRecFields[0] = keySerde;
        secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
    }
    if (numFilterFields > 0) {
        secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys);
        Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
        IAType type = keyTypePair.first;
        ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
        secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
    }
    secondaryRecDesc = new RecordDescriptor(secondaryRecFields);
    // Comparators and type traits for tokens.
    int numTokenFields = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1;
    tokenComparatorFactories = new IBinaryComparatorFactory[numTokenFields];
    tokenTypeTraits = new ITypeTraits[numTokenFields];
    tokenComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
    tokenTypeTraits[0] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
    if (isPartitioned) {
        // The partitioning field is hardcoded to be a short *without* an Asterix type tag.
        tokenComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
        tokenTypeTraits[1] = ShortPointable.TYPE_TRAITS;
    }
    // Set tokenizer factory.
    // TODO: We might want to expose the hashing option at the AQL level,
    // and add the choice to the index metadata.
    tokenizerFactory = NonTaggedFormatUtil.getBinaryTokenizerFactory(secondaryKeyType.getTypeTag(), indexType, index.getGramLength());
    // Type traits for inverted-list elements. Inverted lists contain
    // primary keys.
    invListsTypeTraits = new ITypeTraits[numPrimaryKeys];
    if (numPrimaryKeys > 0) {
        invListsTypeTraits[0] = primaryRecDesc.getTypeTraits()[0];
        enforcedRecFields[0] = primaryRecDesc.getFields()[0];
        enforcedTypeTraits[0] = primaryRecDesc.getTypeTraits()[0];
    }
    enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType);
    enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
    // For tokenization, sorting and loading.
    // One token (+ optional partitioning field) + primary keys.
    numTokenKeyPairFields = (!isPartitioned) ? 1 + numPrimaryKeys : 2 + numPrimaryKeys;
    ISerializerDeserializer[] tokenKeyPairFields = new ISerializerDeserializer[numTokenKeyPairFields + numFilterFields];
    ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields];
    tokenKeyPairComparatorFactories = new IBinaryComparatorFactory[numTokenKeyPairFields];
    tokenKeyPairFields[0] = serdeProvider.getSerializerDeserializer(secondaryKeyType);
    tokenKeyPairTypeTraits[0] = tokenTypeTraits[0];
    tokenKeyPairComparatorFactories[0] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType);
    int pkOff = 1;
    if (isPartitioned) {
        tokenKeyPairFields[1] = ShortSerializerDeserializer.INSTANCE;
        tokenKeyPairTypeTraits[1] = tokenTypeTraits[1];
        tokenKeyPairComparatorFactories[1] = PointableBinaryComparatorFactory.of(ShortPointable.FACTORY);
        pkOff = 2;
    }
    if (numPrimaryKeys > 0) {
        tokenKeyPairFields[pkOff] = primaryRecDesc.getFields()[0];
        tokenKeyPairTypeTraits[pkOff] = primaryRecDesc.getTypeTraits()[0];
        tokenKeyPairComparatorFactories[pkOff] = primaryComparatorFactories[0];
    }
    if (numFilterFields > 0) {
        tokenKeyPairFields[numPrimaryKeys + pkOff] = secondaryRecFields[numPrimaryKeys + numSecondaryKeys];
    }
    tokenKeyPairRecDesc = new RecordDescriptor(tokenKeyPairFields, tokenKeyPairTypeTraits);
    if (filterFieldName != null) {
        invertedIndexFields = new int[numTokenKeyPairFields];
        for (int i = 0; i < invertedIndexFields.length; i++) {
            invertedIndexFields[i] = i;
        }
        secondaryFilterFieldsForNonBulkLoadOps = new int[numFilterFields];
        secondaryFilterFieldsForNonBulkLoadOps[0] = numSecondaryKeys + numPrimaryKeys;
        invertedIndexFieldsForNonBulkLoadOps = new int[numSecondaryKeys + numPrimaryKeys];
        for (int i = 0; i < invertedIndexFieldsForNonBulkLoadOps.length; i++) {
            invertedIndexFieldsForNonBulkLoadOps[i] = i;
        }
    }
}

Example 58 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class DatasetUtil method createPrimaryIndexUpsertOp.

/**
     * Creates a primary index upsert operator for a given dataset.
     *
     * @param spec,
     *            the job specification.
     * @param metadataProvider,
     *            the metadata provider.
     * @param dataset,
     *            the dataset to upsert.
     * @param inputRecordDesc,the
     *            record descriptor for an input tuple.
     * @param fieldPermutation,
     *            the field permutation according to the input.
     * @param missingWriterFactory,
     *            the factory for customizing missing value serialization.
     * @return a primary index scan operator and its location constraints.
     * @throws AlgebricksException
     */
public static Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> createPrimaryIndexUpsertOp(JobSpecification spec, MetadataProvider metadataProvider, Dataset dataset, RecordDescriptor inputRecordDesc, int[] fieldPermutation, IMissingWriterFactory missingWriterFactory) throws AlgebricksException {
    int numKeys = dataset.getPrimaryKeys().size();
    int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1;
    ARecordType itemType = (ARecordType) metadataProvider.findType(dataset);
    ARecordType metaItemType = (ARecordType) metadataProvider.findMetaType(dataset);
    try {
        Index primaryIndex = metadataProvider.getIndex(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName());
        Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset);
        // prepare callback
        JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
        int[] primaryKeyFields = new int[numKeys];
        for (int i = 0; i < numKeys; i++) {
            primaryKeyFields[i] = i;
        }
        boolean hasSecondaries = metadataProvider.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName()).size() > 1;
        IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
        IModificationOperationCallbackFactory modificationCallbackFactory = dataset.getModificationCallbackFactory(storageComponentProvider, primaryIndex, jobId, IndexOperation.UPSERT, primaryKeyFields);
        ISearchOperationCallbackFactory searchCallbackFactory = dataset.getSearchCallbackFactory(storageComponentProvider, primaryIndex, jobId, IndexOperation.UPSERT, primaryKeyFields);
        IIndexDataflowHelperFactory idfh = new IndexDataflowHelperFactory(storageComponentProvider.getStorageManager(), splitsAndConstraint.first);
        LSMPrimaryUpsertOperatorDescriptor op;
        ITypeTraits[] outputTypeTraits = new ITypeTraits[inputRecordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
        ISerializerDeserializer<?>[] outputSerDes = new ISerializerDeserializer[inputRecordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields];
        // add the previous record first
        int f = 0;
        outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType);
        f++;
        // add the previous meta second
        if (dataset.hasMetaPart()) {
            outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(metaItemType);
            outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(metaItemType);
            f++;
        }
        // add the previous filter third
        int fieldIdx = -1;
        if (numFilterFields > 0) {
            String filterField = DatasetUtil.getFilterField(dataset).get(0);
            String[] fieldNames = itemType.getFieldNames();
            int i = 0;
            for (; i < fieldNames.length; i++) {
                if (fieldNames[i].equals(filterField)) {
                    break;
                }
            }
            fieldIdx = i;
            outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(itemType.getFieldTypes()[fieldIdx]);
            outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]);
            f++;
        }
        for (int j = 0; j < inputRecordDesc.getFieldCount(); j++) {
            outputTypeTraits[j + f] = inputRecordDesc.getTypeTraits()[j];
            outputSerDes[j + f] = inputRecordDesc.getFields()[j];
        }
        RecordDescriptor outputRecordDesc = new RecordDescriptor(outputSerDes, outputTypeTraits);
        op = new LSMPrimaryUpsertOperatorDescriptor(spec, outputRecordDesc, fieldPermutation, idfh, missingWriterFactory, modificationCallbackFactory, searchCallbackFactory, dataset.getFrameOpCallbackFactory(), numKeys, itemType, fieldIdx, hasSecondaries);
        return new Pair<>(op, splitsAndConstraint.second);
    } catch (MetadataException me) {
        throw new AlgebricksException(me);
    }
}

Example 59 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class RTreeResourceFactoryProvider method getResourceFactory.

@Override
public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException {
    if (index.getKeyFieldNames().size() != 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD, index.getKeyFieldNames().size(), index.getIndexType(), 1);
    }
    IAType spatialType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), recordType).first;
    if (spatialType == null) {
        throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, StringUtils.join(index.getKeyFieldNames().get(0), '.'));
    }
    List<List<String>> primaryKeyFields = dataset.getPrimaryKeys();
    int numPrimaryKeys = primaryKeyFields.size();
    ITypeTraits[] primaryTypeTraits = null;
    IBinaryComparatorFactory[] primaryComparatorFactories = null;
    IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider();
    if (dataset.getDatasetType() == DatasetType.INTERNAL) {
        primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)];
        primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys];
        List<Integer> indicators = null;
        if (dataset.hasMetaPart()) {
            indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator();
        }
        for (int i = 0; i < numPrimaryKeys; i++) {
            IAType keyType = (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(primaryKeyFields.get(i)) : metaType.getSubFieldType(primaryKeyFields.get(i));
            primaryComparatorFactories[i] = storageComponentProvider.getComparatorFactoryProvider().getBinaryComparatorFactory(keyType, true);
            primaryTypeTraits[i] = storageComponentProvider.getTypeTraitProvider().getTypeTrait(keyType);
        }
        primaryTypeTraits[numPrimaryKeys] = storageComponentProvider.getTypeTraitProvider().getTypeTrait(recordType);
        if (dataset.hasMetaPart()) {
            primaryTypeTraits[numPrimaryKeys + 1] = storageComponentProvider.getTypeTraitProvider().getTypeTrait(recordType);
        }
    }
    boolean isPointMBR = spatialType.getTypeTag() == ATypeTag.POINT || spatialType.getTypeTag() == ATypeTag.POINT3D;
    int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
    int numNestedSecondaryKeyFields = numDimensions * 2;
    IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
    IPrimitiveValueProviderFactory[] valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
    ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numNestedSecondaryKeyFields + numPrimaryKeys];
    IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(spatialType.getTypeTag());
    ATypeTag keyType = nestedKeyType.getTypeTag();
    for (int i = 0; i < numNestedSecondaryKeyFields; i++) {
        secondaryComparatorFactories[i] = storageComponentProvider.getComparatorFactoryProvider().getBinaryComparatorFactory(nestedKeyType, true);
        secondaryTypeTraits[i] = storageComponentProvider.getTypeTraitProvider().getTypeTrait(nestedKeyType);
        valueProviderFactories[i] = storageComponentProvider.getPrimitiveValueProviderFactory();
    }
    for (int i = 0; i < numPrimaryKeys; i++) {
        secondaryTypeTraits[numNestedSecondaryKeyFields + i] = (dataset.getDatasetType() == DatasetType.INTERNAL) ? primaryTypeTraits[i] : IndexingConstants.getTypeTraits(i);
    }
    int[] rtreeFields = null;
    if (filterTypeTraits != null && filterTypeTraits.length > 0) {
        rtreeFields = new int[numNestedSecondaryKeyFields + numPrimaryKeys];
        for (int i = 0; i < rtreeFields.length; i++) {
            rtreeFields[i] = i;
        }
    }
    IStorageManager storageManager = storageComponentProvider.getStorageManager();
    ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index);
    ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index);
    IMetadataPageManagerFactory metadataPageManagerFactory = storageComponentProvider.getMetadataPageManagerFactory();
    ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider();
    boolean durable = !dataset.isTemp();
    ILinearizeComparatorFactory linearizeCmpFactory = MetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length);
    ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType);
    IBinaryComparatorFactory[] rtreeCmpFactories = getCmpFactories(mdProvider, index, recordType, metaType);
    int[] secondaryFilterFields = (filterTypeTraits != null && filterTypeTraits.length > 0) ? new int[] { numNestedSecondaryKeyFields + numPrimaryKeys } : null;
    IBinaryComparatorFactory[] btreeCompFactories = dataset.getDatasetType() == DatasetType.EXTERNAL ? IndexingConstants.getBuddyBtreeComparatorFactories() : getComparatorFactoriesForDeletedKeyBTree(secondaryTypeTraits, primaryComparatorFactories, secondaryComparatorFactories);
    if (dataset.getDatasetType() == DatasetType.INTERNAL) {
        AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId());
        return new LSMRTreeWithAntiMatterLocalResourceFactory(storageManager, typeTraits, rtreeCmpFactories, filterTypeTraits, filterCmpFactories, secondaryFilterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, valueProviderFactories, rTreePolicyType, linearizeCmpFactory, rtreeFields, isPointMBR, btreeCompFactories);
    } else {
        return new ExternalRTreeLocalResourceFactory(storageManager, typeTraits, rtreeCmpFactories, filterTypeTraits, filterCmpFactories, secondaryFilterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, btreeCompFactories, valueProviderFactories, rTreePolicyType, linearizeCmpFactory, rtreeFields, new int[] { numNestedSecondaryKeyFields }, isPointMBR, mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate());
    }
}

Example 60 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class SecondaryBTreeOperationsHelper method setSecondaryRecDescAndComparators.

@Override
@SuppressWarnings("rawtypes")
protected void setSecondaryRecDescAndComparators() throws AlgebricksException {
    int numSecondaryKeys = index.getKeyFieldNames().size();
    secondaryFieldAccessEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeys + numFilterFields];
    secondaryComparatorFactories = new IBinaryComparatorFactory[numSecondaryKeys + numPrimaryKeys];
    secondaryBloomFilterKeyFields = new int[numSecondaryKeys];
    ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys + numSecondaryKeys + numFilterFields];
    ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
    ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields];
    secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys];
    ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
    ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider();
    IBinaryComparatorFactoryProvider comparatorFactoryProvider = metadataProvider.getFormat().getBinaryComparatorFactoryProvider();
    // Record column is 0 for external datasets, numPrimaryKeys for internal ones
    int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0;
    boolean isEnforcingKeyTypes = index.isEnforcingKeyFileds();
    for (int i = 0; i < numSecondaryKeys; i++) {
        ARecordType sourceType;
        int sourceColumn;
        List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators();
        if (keySourceIndicators == null || keySourceIndicators.get(i) == 0) {
            sourceType = itemType;
            sourceColumn = recordColumn;
        } else {
            sourceType = metaType;
            sourceColumn = recordColumn + 1;
        }
        secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(isEnforcingKeyTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i), sourceColumn);
        Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), sourceType);
        IAType keyType = keyTypePair.first;
        anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second;
        ISerializerDeserializer keySerde = serdeProvider.getSerializerDeserializer(keyType);
        secondaryRecFields[i] = keySerde;
        secondaryComparatorFactories[i] = comparatorFactoryProvider.getBinaryComparatorFactory(keyType, true);
        secondaryTypeTraits[i] = typeTraitProvider.getTypeTrait(keyType);
        secondaryBloomFilterKeyFields[i] = i;
    }
    if (dataset.getDatasetType() == DatasetType.INTERNAL) {
        // Add serializers and comparators for primary index fields.
        for (int i = 0; i < numPrimaryKeys; i++) {
            secondaryRecFields[numSecondaryKeys + i] = primaryRecDesc.getFields()[i];
            enforcedRecFields[i] = primaryRecDesc.getFields()[i];
            secondaryTypeTraits[numSecondaryKeys + i] = primaryRecDesc.getTypeTraits()[i];
            enforcedTypeTraits[i] = primaryRecDesc.getTypeTraits()[i];
            secondaryComparatorFactories[numSecondaryKeys + i] = primaryComparatorFactories[i];
        }
    } else {
        // Add serializers and comparators for RID fields.
        for (int i = 0; i < numPrimaryKeys; i++) {
            secondaryRecFields[numSecondaryKeys + i] = IndexingConstants.getSerializerDeserializer(i);
            enforcedRecFields[i] = IndexingConstants.getSerializerDeserializer(i);
            secondaryTypeTraits[numSecondaryKeys + i] = IndexingConstants.getTypeTraits(i);
            enforcedTypeTraits[i] = IndexingConstants.getTypeTraits(i);
            secondaryComparatorFactories[numSecondaryKeys + i] = IndexingConstants.getComparatorFactory(i);
        }
    }
    enforcedRecFields[numPrimaryKeys] = serdeProvider.getSerializerDeserializer(itemType);
    enforcedTypeTraits[numPrimaryKeys] = typeTraitProvider.getTypeTrait(itemType);
    if (dataset.hasMetaPart()) {
        enforcedRecFields[numPrimaryKeys + 1] = serdeProvider.getSerializerDeserializer(metaType);
        enforcedTypeTraits[numPrimaryKeys + 1] = typeTraitProvider.getTypeTrait(metaType);
    }
    if (numFilterFields > 0) {
        secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys);
        Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
        IAType type = keyTypePair.first;
        ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
        secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
        enforcedRecFields[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)] = serde;
        enforcedTypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)] = typeTraitProvider.getTypeTrait(type);
    }
    secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
    enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
}