Search in sources :

Example 21 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class Dataset method getPrimaryTypeTraits.

public ITypeTraits[] getPrimaryTypeTraits(MetadataProvider metadataProvider, ARecordType recordType, ARecordType metaType) throws AlgebricksException {
    IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
    ITypeTraitProvider ttProvider = storageComponentProvider.getTypeTraitProvider();
    List<List<String>> partitioningKeys = getPrimaryKeys();
    int numPrimaryKeys = partitioningKeys.size();
    ITypeTraits[] typeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)];
    List<Integer> indicators = null;
    if (hasMetaPart()) {
        indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator();
    }
    for (int i = 0; i < numPrimaryKeys; i++) {
        IAType keyType = datasetType == DatasetType.EXTERNAL ? IndexingConstants.getFieldType(i) : (indicators == null || indicators.get(i) == 0) ? recordType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i));
        typeTraits[i] = ttProvider.getTypeTrait(keyType);
    }
    typeTraits[numPrimaryKeys] = ttProvider.getTypeTrait(recordType);
    if (hasMetaPart()) {
        typeTraits[numPrimaryKeys + 1] = ttProvider.getTypeTrait(metaType);
    }
    return typeTraits;
}
Also used : IStorageComponentProvider(org.apache.asterix.common.context.IStorageComponentProvider) ITypeTraitProvider(org.apache.hyracks.algebricks.data.ITypeTraitProvider) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) List(java.util.List) IAType(org.apache.asterix.om.types.IAType)

Example 22 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class BTreeResourceFactoryProvider method getResourceFactory.

@Override
public IResourceFactory getResourceFactory(MetadataProvider mdProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType, ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories) throws AlgebricksException {
    int[] filterFields = IndexUtil.getFilterFields(dataset, index, filterTypeTraits);
    int[] btreeFields = IndexUtil.getBtreeFieldsIfFiltered(dataset, index);
    IStorageComponentProvider storageComponentProvider = mdProvider.getStorageComponentProvider();
    ITypeTraits[] typeTraits = getTypeTraits(mdProvider, dataset, index, recordType, metaType);
    IBinaryComparatorFactory[] cmpFactories = getCmpFactories(mdProvider, dataset, index, recordType, metaType);
    int[] bloomFilterFields = getBloomFilterFields(dataset, index);
    boolean durable = !dataset.isTemp();
    double bloomFilterFalsePositiveRate = mdProvider.getStorageProperties().getBloomFilterFalsePositiveRate();
    ILSMOperationTrackerFactory opTrackerFactory = dataset.getIndexOperationTrackerFactory(index);
    ILSMIOOperationCallbackFactory ioOpCallbackFactory = dataset.getIoOperationCallbackFactory(index);
    IStorageManager storageManager = storageComponentProvider.getStorageManager();
    IMetadataPageManagerFactory metadataPageManagerFactory = storageComponentProvider.getMetadataPageManagerFactory();
    ILSMIOOperationSchedulerProvider ioSchedulerProvider = storageComponentProvider.getIoOperationSchedulerProvider();
    switch(dataset.getDatasetType()) {
        case EXTERNAL:
            return index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName())) ? new ExternalBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, bloomFilterFalsePositiveRate, false, btreeFields) : new ExternalBTreeWithBuddyLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, bloomFilterFalsePositiveRate, false, btreeFields);
        case INTERNAL:
            AsterixVirtualBufferCacheProvider vbcProvider = new AsterixVirtualBufferCacheProvider(dataset.getDatasetId());
            return new LSMBTreeLocalResourceFactory(storageManager, typeTraits, cmpFactories, filterTypeTraits, filterCmpFactories, filterFields, opTrackerFactory, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider, mergePolicyFactory, mergePolicyProperties, durable, bloomFilterFields, bloomFilterFalsePositiveRate, index.isPrimaryIndex(), btreeFields);
        default:
            throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_DATASET_TYPE, dataset.getDatasetType().toString());
    }
}
Also used : CompilationException(org.apache.asterix.common.exceptions.CompilationException) ILSMIOOperationCallbackFactory(org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory) IStorageComponentProvider(org.apache.asterix.common.context.IStorageComponentProvider) ExternalBTreeLocalResourceFactory(org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeLocalResourceFactory) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) IMetadataPageManagerFactory(org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) AsterixVirtualBufferCacheProvider(org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider) ILSMOperationTrackerFactory(org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory) ExternalBTreeWithBuddyLocalResourceFactory(org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyLocalResourceFactory) IStorageManager(org.apache.hyracks.storage.common.IStorageManager) ILSMIOOperationSchedulerProvider(org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationSchedulerProvider) LSMBTreeLocalResourceFactory(org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResourceFactory)

Example 23 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class MetadataBootstrap method enlistMetadataDataset.

/**
     * Enlist a metadata index so it is available for metadata operations
     * should be performed upon bootstrapping
     *
     * @param index
     * @throws HyracksDataException
     */
public static void enlistMetadataDataset(INCServiceContext ncServiceCtx, IMetadataIndex index) throws HyracksDataException {
    ClusterPartition metadataPartition = appContext.getMetadataProperties().getMetadataPartition();
    int metadataDeviceId = metadataPartition.getIODeviceNum();
    String metadataPartitionPath = StoragePathUtil.prepareStoragePartitionPath(ClusterProperties.INSTANCE.getStorageDirectoryName(), metadataPartition.getPartitionId());
    String resourceName = metadataPartitionPath + File.separator + index.getFileNameRelativePath();
    FileReference file = ioManager.getFileReference(metadataDeviceId, resourceName);
    index.setFile(file);
    ITypeTraits[] typeTraits = index.getTypeTraits();
    IBinaryComparatorFactory[] cmpFactories = index.getKeyBinaryComparatorFactory();
    int[] bloomFilterKeyFields = index.getBloomFilterKeyFields();
    // opTrackerProvider and ioOpCallbackFactory should both be acquired through IStorageManager
    // We are unable to do this since IStorageManager needs a dataset to determine the appropriate
    // objects
    ILSMOperationTrackerFactory opTrackerFactory = index.isPrimaryIndex() ? new PrimaryIndexOperationTrackerFactory(index.getDatasetId().getId()) : new SecondaryIndexOperationTrackerFactory(index.getDatasetId().getId());
    ILSMIOOperationCallbackFactory ioOpCallbackFactory = LSMBTreeIOOperationCallbackFactory.INSTANCE;
    IStorageComponentProvider storageComponentProvider = appContext.getStorageComponentProvider();
    if (isNewUniverse()) {
        LSMBTreeLocalResourceFactory lsmBtreeFactory = new LSMBTreeLocalResourceFactory(storageComponentProvider.getStorageManager(), typeTraits, cmpFactories, null, null, null, opTrackerFactory, ioOpCallbackFactory, storageComponentProvider.getMetadataPageManagerFactory(), new AsterixVirtualBufferCacheProvider(index.getDatasetId().getId()), storageComponentProvider.getIoOperationSchedulerProvider(), appContext.getMetadataMergePolicyFactory(), GlobalConfig.DEFAULT_COMPACTION_POLICY_PROPERTIES, true, bloomFilterKeyFields, appContext.getBloomFilterFalsePositiveRate(), true, null);
        DatasetLocalResourceFactory dsLocalResourceFactory = new DatasetLocalResourceFactory(index.getDatasetId().getId(), lsmBtreeFactory);
        // TODO(amoudi) Creating the index should be done through the same code path as other indexes
        // This is to be done by having a metadata dataset associated with each index
        IIndexBuilder indexBuilder = new IndexBuilder(ncServiceCtx, storageComponentProvider.getStorageManager(), index::getResourceId, file, dsLocalResourceFactory, true);
        indexBuilder.build();
    } else {
        final LocalResource resource = localResourceRepository.get(file.getRelativePath());
        if (resource == null) {
            throw new HyracksDataException("Could not find required metadata indexes. Please delete " + appContext.getMetadataProperties().getTransactionLogDirs().get(appContext.getTransactionSubsystem().getId()) + " to intialize as a new instance. (WARNING: all data will be lost.)");
        }
        // Why do we care about metadata dataset's resource ids? why not assign them ids similar to other resources?
        if (index.getResourceId() != resource.getId()) {
            throw new HyracksDataException("Resource Id doesn't match expected metadata index resource id");
        }
        IndexDataflowHelper indexHelper = new IndexDataflowHelper(ncServiceCtx, storageComponentProvider.getStorageManager(), file);
        // Opening the index through the helper will ensure it gets instantiated
        indexHelper.open();
        indexHelper.close();
    }
}
Also used : DatasetLocalResourceFactory(org.apache.asterix.transaction.management.resource.DatasetLocalResourceFactory) ILSMIOOperationCallbackFactory(org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory) IStorageComponentProvider(org.apache.asterix.common.context.IStorageComponentProvider) ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) IBinaryComparatorFactory(org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory) AsterixVirtualBufferCacheProvider(org.apache.asterix.common.context.AsterixVirtualBufferCacheProvider) ILSMOperationTrackerFactory(org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) IIndexBuilder(org.apache.hyracks.storage.am.common.api.IIndexBuilder) IndexBuilder(org.apache.hyracks.storage.am.common.build.IndexBuilder) LocalResource(org.apache.hyracks.storage.common.LocalResource) SecondaryIndexOperationTrackerFactory(org.apache.asterix.transaction.management.opcallbacks.SecondaryIndexOperationTrackerFactory) IIndexBuilder(org.apache.hyracks.storage.am.common.api.IIndexBuilder) LSMBTreeLocalResourceFactory(org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResourceFactory) FileReference(org.apache.hyracks.api.io.FileReference) IndexDataflowHelper(org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper) PrimaryIndexOperationTrackerFactory(org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexOperationTrackerFactory) ClusterPartition(org.apache.asterix.common.cluster.ClusterPartition)

Example 24 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class Dataset method getPrimaryRecordDescriptor.

/**
     * Gets the record descriptor for primary records of this dataset.
     *
     * @param metadataProvider,
     *            the metadata provider.
     * @return the record descriptor for primary records of this dataset.
     * @throws AlgebricksException
     */
public RecordDescriptor getPrimaryRecordDescriptor(MetadataProvider metadataProvider) throws AlgebricksException {
    List<List<String>> partitioningKeys = getPrimaryKeys();
    int numPrimaryKeys = partitioningKeys.size();
    ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)];
    ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)];
    ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider();
    List<Integer> indicators = null;
    if (hasMetaPart()) {
        indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator();
    }
    ARecordType itemType = (ARecordType) metadataProvider.findType(this);
    ARecordType metaType = (ARecordType) metadataProvider.findMetaType(this);
    // Set the serde/traits for primary keys
    for (int i = 0; i < numPrimaryKeys; i++) {
        IAType keyType = (indicators == null || indicators.get(i) == 0) ? itemType.getSubFieldType(partitioningKeys.get(i)) : metaType.getSubFieldType(partitioningKeys.get(i));
        primaryRecFields[i] = serdeProvider.getSerializerDeserializer(keyType);
        primaryTypeTraits[i] = TypeTraitProvider.INSTANCE.getTypeTrait(keyType);
    }
    // Set the serde for the record field
    primaryRecFields[numPrimaryKeys] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
    primaryTypeTraits[numPrimaryKeys] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType);
    if (hasMetaPart()) {
        // Set the serde and traits for the meta record field
        primaryRecFields[numPrimaryKeys + 1] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(metaType);
        primaryTypeTraits[numPrimaryKeys + 1] = TypeTraitProvider.INSTANCE.getTypeTrait(itemType);
    }
    return new RecordDescriptor(primaryRecFields, primaryTypeTraits);
}
Also used : ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) RecordDescriptor(org.apache.hyracks.api.dataflow.value.RecordDescriptor) ISerializerDeserializer(org.apache.hyracks.api.dataflow.value.ISerializerDeserializer) ISerializerDeserializerProvider(org.apache.hyracks.algebricks.data.ISerializerDeserializerProvider) List(java.util.List) ARecordType(org.apache.asterix.om.types.ARecordType) IAType(org.apache.asterix.om.types.IAType)

Example 25 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class JobGenHelper method variablesToTypeTraits.

public static ITypeTraits[] variablesToTypeTraits(List<LogicalVariable> varLogical, int start, int size, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
    ITypeTraits[] typeTraits = new ITypeTraits[size];
    ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider();
    for (int i = 0; i < size; i++) {
        Object type = env.getVarType(varLogical.get(start + i));
        typeTraits[i] = typeTraitProvider.getTypeTrait(type);
    }
    return typeTraits;
}
Also used : ITypeTraits(org.apache.hyracks.api.dataflow.value.ITypeTraits) ITypeTraitProvider(org.apache.hyracks.algebricks.data.ITypeTraitProvider)

Aggregations

ITypeTraits (org.apache.hyracks.api.dataflow.value.ITypeTraits)67 IBinaryComparatorFactory (org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory)45 ISerializerDeserializer (org.apache.hyracks.api.dataflow.value.ISerializerDeserializer)34 Test (org.junit.Test)22 ArrayTupleBuilder (org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder)21 ArrayTupleReference (org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference)18 ITreeIndex (org.apache.hyracks.storage.am.common.api.ITreeIndex)17 HyracksDataException (org.apache.hyracks.api.exceptions.HyracksDataException)16 RecordDescriptor (org.apache.hyracks.api.dataflow.value.RecordDescriptor)15 IIndexAccessor (org.apache.hyracks.storage.common.IIndexAccessor)15 IAType (org.apache.asterix.om.types.IAType)14 UTF8StringSerializerDeserializer (org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer)11 IPrimitiveValueProviderFactory (org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory)10 List (java.util.List)9 CompilationException (org.apache.asterix.common.exceptions.CompilationException)9 ARecordType (org.apache.asterix.om.types.ARecordType)8 ITypeTraitProvider (org.apache.hyracks.algebricks.data.ITypeTraitProvider)8 IStorageManager (org.apache.hyracks.storage.common.IStorageManager)8 IFileSplitProvider (org.apache.hyracks.dataflow.std.file.IFileSplitProvider)7 IIndexDataflowHelperFactory (org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory)7