Examples with ITypeTraits org.apache.hyracks.api.dataflow.value.ITypeTraits used on opensource projects

Example 6 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class OrderedIndexExamplesTest method fixedLengthKeyValueExample.

/**
     * Fixed-Length Key,Value Example. Create a tree index with one fixed-length
     * key field and one fixed-length value field. Fill index with random values
     * using insertions (not bulk load). Perform scans and range search.
     */
@Test
public void fixedLengthKeyValueExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Fixed-Length Key,Value Example.");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = IntegerPointable.TYPE_TRAITS;
    typeTraits[1] = IntegerPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    int numInserts = 10000;
    for (int i = 0; i < numInserts; i++) {
        int f0 = rnd.nextInt() % numInserts;
        int f1 = 5;
        TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 1000 == 0) {
                LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
    }
    orderedScan(indexAccessor, fieldSerdes);
    diskOrderScan(indexAccessor, fieldSerdes);
    // Build low key.
    ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
    ArrayTupleReference lowKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
    // Build high key.
    ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
    ArrayTupleReference highKey = new ArrayTupleReference();
    TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
    rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 7 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class OrderedIndexExamplesTest method varLenKeyValueExample.

/**
     * Variable-Length Example. Create a BTree with one variable-length key
     * field and one variable-length value field. Fill BTree with random values
     * using insertions (not bulk load) Perform ordered scans and range search.
     */
@Test
public void varLenKeyValueExample() throws Exception {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Variable-Length Key,Value Example");
    }
    // Declare fields.
    int fieldCount = 2;
    ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
    typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
    typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
    // Declare field serdes.
    ISerializerDeserializer[] fieldSerdes = { new UTF8StringSerializerDeserializer(), new UTF8StringSerializerDeserializer() };
    // Declare keys.
    int keyFieldCount = 1;
    IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
    cmpFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
    // This is only used for the LSM-BTree.
    int[] bloomFilterKeyFields = new int[keyFieldCount];
    bloomFilterKeyFields[0] = 0;
    ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null, null);
    treeIndex.create();
    treeIndex.activate();
    long start = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Inserting into tree...");
    }
    ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
    ArrayTupleReference tuple = new ArrayTupleReference();
    IIndexAccessor indexAccessor = treeIndex.createAccessor(TestOperationCallback.INSTANCE, TestOperationCallback.INSTANCE);
    // Max string length to be generated.
    int maxLength = 10;
    int numInserts = 10000;
    for (int i = 0; i < 10000; i++) {
        String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
        TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
        if (LOGGER.isLoggable(Level.INFO)) {
            if (i % 1000 == 0) {
                LOGGER.info("Inserting[" + i + "] " + f0 + " " + f1);
            }
        }
        try {
            indexAccessor.insert(tuple);
        } catch (HyracksDataException e) {
            if (e.getErrorCode() != ErrorCode.DUPLICATE_KEY) {
                throw e;
            }
        }
    }
    long end = System.currentTimeMillis();
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
    }
    orderedScan(indexAccessor, fieldSerdes);
    diskOrderScan(indexAccessor, fieldSerdes);
    // Build low key.
    ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference lowKey = new ArrayTupleReference();
    TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
    // Build high key.
    ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
    ArrayTupleReference highKey = new ArrayTupleReference();
    TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
    rangeSearch(cmpFactories, indexAccessor, fieldSerdes, lowKey, highKey, null, null);
    treeIndex.validate();
    treeIndex.deactivate();
    treeIndex.destroy();
}

Example 8 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class DatasetUtil method computeFilterTypeTraits.

public static ITypeTraits[] computeFilterTypeTraits(Dataset dataset, ARecordType itemType) throws AlgebricksException {
    if (dataset.getDatasetType() == DatasetType.EXTERNAL) {
        return null;
    }
    List<String> filterField = getFilterField(dataset);
    if (filterField == null) {
        return null;
    }
    ITypeTraits[] typeTraits = new ITypeTraits[1];
    IAType type = itemType.getSubFieldType(filterField);
    typeTraits[0] = TypeTraitProvider.INSTANCE.getTypeTrait(type);
    return typeTraits;
}

Example 9 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class InvertedIndexResourceFactoryProvider method getTokenTypeTraits.

private static ITypeTraits[] getTokenTypeTraits(Dataset dataset, Index index, ARecordType recordType, ARecordType metaType) throws AlgebricksException {
    int numPrimaryKeys = dataset.getPrimaryKeys().size();
    int numSecondaryKeys = index.getKeyFieldNames().size();
    IndexType indexType = index.getIndexType();
    // Sanity checks.
    if (numPrimaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_FOR_DATASET_WITH_COMPOSITE_PRIMARY_INDEX, indexType, RecordUtil.toFullyQualifiedName(dataset.getDataverseName(), dataset.getDatasetName()));
    }
    if (numSecondaryKeys > 1) {
        throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_INDEX_NUM_OF_FIELD, numSecondaryKeys, indexType, 1);
    }
    boolean isPartitioned = indexType == IndexType.LENGTH_PARTITIONED_WORD_INVIX || indexType == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
    ARecordType sourceType;
    List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators();
    if (keySourceIndicators == null || keySourceIndicators.get(0) == 0) {
        sourceType = recordType;
    } else {
        sourceType = metaType;
    }
    Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), sourceType);
    IAType secondaryKeyType = keyTypePair.first;
    int numTokenFields = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1;
    ITypeTraits[] tokenTypeTraits = new ITypeTraits[numTokenFields];
    tokenTypeTraits[0] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType);
    if (isPartitioned) {
        // The partitioning field is hardcoded to be a short *without* an Asterix type tag.
        tokenTypeTraits[1] = ShortPointable.TYPE_TRAITS;
    }
    return tokenTypeTraits;
}

Example 10 with ITypeTraits

use of org.apache.hyracks.api.dataflow.value.ITypeTraits in project asterixdb by apache.

the class BTreeTestContext method create.

public static BTreeTestContext create(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType, IPageManager pageManager) throws Exception {
    ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
    IBinaryComparatorFactory[] cmpFactories = SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
    BTree btree = BTreeUtils.createBTree(bufferCache, fileMapProvider, typeTraits, cmpFactories, leafType, file, pageManager);
    BTreeTestContext testCtx = new BTreeTestContext(fieldSerdes, btree);
    return testCtx;
}