Examples with IntVector org.apache.drill.exec.vector.IntVector used on opensource projects

Example 21 with IntVector

use of org.apache.drill.exec.vector.IntVector in project drill by apache.

the class HashTableTemplate method allocMetadataVector.

private IntVector allocMetadataVector(int size, int initialValue) {
    IntVector vector = (IntVector) TypeHelper.getNewVector(dummyIntField, allocator);
    vector.allocateNew(size);
    for (int i = 0; i < size; i++) {
        vector.getMutator().set(i, initialValue);
    }
    vector.getMutator().setValueCount(size);
    return vector;
}

Example 22 with IntVector

use of org.apache.drill.exec.vector.IntVector in project drill by apache.

the class HashTableTemplate method resizeAndRehashIfNeeded.

// Resize the hash table if needed by creating a new one with double the number of buckets.
// For each entry in the old hash table, re-hash it to the new table and update the metadata
// in the new table.. the metadata consists of the startIndices, links and hashValues.
// Note that the keys stored in the BatchHolders are not moved around.
private void resizeAndRehashIfNeeded() {
    if (numEntries < threshold) {
        return;
    }
    if (EXTRA_DEBUG) {
        logger.debug("Hash table numEntries = {}, threshold = {}; resizing the table...", numEntries, threshold);
    }
    // future attempts to resize will return immediately.
    if (tableSize == MAXIMUM_CAPACITY) {
        threshold = Integer.MAX_VALUE;
        return;
    }
    int newTableSize = 2 * tableSize;
    newTableSize = roundUpToPowerOf2(newTableSize);
    // the new hash-values (to replace the existing ones - inside rehash() ), then OOM
    if (4 * /* sizeof(int) */
    (newTableSize + 2 * HashTable.BATCH_SIZE) >= allocator.getLimit() - allocator.getAllocatedMemory()) {
        throw new OutOfMemoryException("Resize Hash Table");
    }
    tableSize = newTableSize;
    if (tableSize > MAXIMUM_CAPACITY) {
        tableSize = MAXIMUM_CAPACITY;
    }
    long t0 = System.currentTimeMillis();
    // set the new threshold based on the new table size and load factor
    threshold = (int) Math.ceil(tableSize * htConfig.getLoadFactor());
    IntVector newStartIndices = allocMetadataVector(tableSize, EMPTY_SLOT);
    for (int i = 0; i < batchHolders.size(); i++) {
        BatchHolder bh = batchHolders.get(i);
        int batchStartIdx = i * BATCH_SIZE;
        bh.rehash(tableSize, newStartIndices, batchStartIdx);
    }
    startIndices.clear();
    startIndices = newStartIndices;
    if (EXTRA_DEBUG) {
        logger.debug("After resizing and rehashing, dumping the hash table...");
        logger.debug("Number of buckets = {}.", startIndices.getAccessor().getValueCount());
        for (int i = 0; i < startIndices.getAccessor().getValueCount(); i++) {
            logger.debug("Bucket: {}, startIdx[ {} ] = {}.", i, i, startIndices.getAccessor().get(i));
            int startIdx = startIndices.getAccessor().get(i);
            BatchHolder bh = batchHolders.get((startIdx >>> 16) & BATCH_MASK);
            bh.dump(startIdx);
        }
    }
    resizingTime += System.currentTimeMillis() - t0;
    numResizing++;
}

Example 23 with IntVector

use of org.apache.drill.exec.vector.IntVector in project drill by apache.

the class ExpressionTest method testSpecial.

@Test
public void testSpecial() throws Exception {
    final RecordBatch batch = mock(RecordBatch.class);
    final VectorWrapper wrapper = mock(VectorWrapper.class);
    final TypeProtos.MajorType type = Types.optional(MinorType.INT);
    final TypedFieldId tfid = new TypedFieldId.Builder().finalType(type).hyper(false).addId(0).build();
    when(wrapper.getValueVector()).thenReturn(new IntVector(MaterializedField.create("result", type), RootAllocatorFactory.newRoot(c)));
    when(batch.getValueVectorId(new SchemaPath("alpha", ExpressionPosition.UNKNOWN))).thenReturn(tfid);
    when(batch.getValueAccessorById(IntVector.class, tfid.getFieldIds())).thenReturn(wrapper);
    getExpressionCode("1 + 1", batch);
}

Example 24 with IntVector

use of org.apache.drill.exec.vector.IntVector in project drill by apache.

the class TestMathFunctions method testBasicMathFunctions.

@Test
public void testBasicMathFunctions() throws Throwable {
    final DrillbitContext bitContext = mockDrillbitContext();
    final UserClientConnection connection = Mockito.mock(UserClientConnection.class);
    final PhysicalPlanReader reader = PhysicalPlanReaderTestFactory.defaultPhysicalPlanReader(c);
    final PhysicalPlan plan = reader.readPhysicalPlan(Files.asCharSource(DrillFileUtils.getResourceAsFile("/functions/simple_math_functions.json"), Charsets.UTF_8).read());
    final FunctionImplementationRegistry registry = new FunctionImplementationRegistry(c);
    final FragmentContextImpl context = new FragmentContextImpl(bitContext, BitControl.PlanFragment.getDefaultInstance(), connection, registry);
    final SimpleRootExec exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next()));
    while (exec.next()) {
        final IntVector intMulVector = exec.getValueVectorById(new SchemaPath("INTMUL", ExpressionPosition.UNKNOWN), IntVector.class);
        final Float8Vector floatMulVector = exec.getValueVectorById(new SchemaPath("FLOATMUL", ExpressionPosition.UNKNOWN), Float8Vector.class);
        final IntVector intAddVector = exec.getValueVectorById(new SchemaPath("INTADD", ExpressionPosition.UNKNOWN), IntVector.class);
        final Float8Vector floatAddVector = exec.getValueVectorById(new SchemaPath("FLOATADD", ExpressionPosition.UNKNOWN), Float8Vector.class);
        assertEquals(exec.getRecordCount(), 1);
        assertEquals(intMulVector.getAccessor().get(0), 2);
        assertEquals(floatMulVector.getAccessor().get(0), (1.1 * 2.2), 0);
        assertEquals(intAddVector.getAccessor().get(0), 3);
        assertEquals(floatAddVector.getAccessor().get(0), (1.1 + 2.2), 0);
    }
    if (context.getExecutorState().getFailureCause() != null) {
        throw context.getExecutorState().getFailureCause();
    }
    assertTrue(!context.getExecutorState().isFailed());
}

Example 25 with IntVector

use of org.apache.drill.exec.vector.IntVector in project drill by apache.

the class TestFixedWidthWriter method testRollover.

/**
 * The rollover method is used during vector overflow.
 */
@Test
public void testRollover() {
    try (IntVector vector = allocVector(1000)) {
        TestIndex index = new TestIndex();
        IntColumnWriter writer = makeWriter(vector, index);
        writer.startWrite();
        for (int i = 0; i < 10; i++) {
            index.index = i;
            writer.startRow();
            writer.setInt(i);
            writer.saveRow();
        }
        // Overflow occurs after writing the 11th row
        index.index = 10;
        writer.startRow();
        writer.setInt(10);
        // Overflow occurs
        writer.preRollover();
        for (int i = 0; i < 15; i++) {
            vector.getMutator().set(i, 0xdeadbeef);
        }
        vector.getMutator().set(0, 10);
        writer.postRollover();
        index.index = 0;
        writer.saveRow();
        for (int i = 1; i < 5; i++) {
            index.index = i;
            writer.startRow();
            writer.setInt(10 + i);
            writer.saveRow();
        }
        writer.endWrite();
        for (int i = 0; i < 5; i++) {
            assertEquals(10 + i, vector.getAccessor().get(i));
        }
    }
}