Examples with Ordering org.apache.drill.common.logical.data.Order.Ordering used on opensource projects

Example 36 with Ordering

use of org.apache.drill.common.logical.data.Order.Ordering in project drill by axbaretto.

the class OrderedPartitionRecordBatch method setupNewSchema.

/**
 * Sets up projection that will transfer all of the columns in batch, and also populate the partition column based on
 * which partition a record falls into in the partition table
 *
 * @param batch
 * @throws SchemaChangeException
 */
protected void setupNewSchema(VectorAccessible batch) throws SchemaChangeException {
    container.clear();
    final ErrorCollector collector = new ErrorCollectorImpl();
    final List<TransferPair> transfers = Lists.newArrayList();
    final ClassGenerator<OrderedPartitionProjector> cg = CodeGenerator.getRoot(OrderedPartitionProjector.TEMPLATE_DEFINITION, context.getOptions());
    for (VectorWrapper<?> vw : batch) {
        TransferPair tp = vw.getValueVector().getTransferPair(oContext.getAllocator());
        transfers.add(tp);
        container.add(tp.getTo());
    }
    cg.setMappingSet(mainMapping);
    int count = 0;
    for (Ordering od : popConfig.getOrderings()) {
        final LogicalExpression expr = ExpressionTreeMaterializer.materialize(od.getExpr(), batch, collector, context.getFunctionRegistry());
        if (collector.hasErrors()) {
            throw new SchemaChangeException("Failure while materializing expression. " + collector.toErrorString());
        }
        cg.setMappingSet(incomingMapping);
        ClassGenerator.HoldingContainer left = cg.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
        cg.setMappingSet(partitionMapping);
        ClassGenerator.HoldingContainer right = cg.addExpr(new ValueVectorReadExpression(new TypedFieldId(expr.getMajorType(), count++)), ClassGenerator.BlkCreateMode.FALSE);
        cg.setMappingSet(mainMapping);
        // next we wrap the two comparison sides and add the expression block for the comparison.
        LogicalExpression fh = FunctionGenerationHelper.getOrderingComparator(od.nullsSortHigh(), left, right, context.getFunctionRegistry());
        ClassGenerator.HoldingContainer out = cg.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
        JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
        if (od.getDirection() == Direction.ASCENDING) {
            jc._then()._return(out.getValue());
        } else {
            jc._then()._return(out.getValue().minus());
        }
    }
    cg.getEvalBlock()._return(JExpr.lit(0));
    container.add(this.partitionKeyVector);
    container.buildSchema(batch.getSchema().getSelectionVectorMode());
    try {
        this.projector = context.getImplementationClass(cg);
        projector.setup(context, batch, this, transfers, partitionVectors, partitions, popConfig.getRef());
    } catch (ClassTransformationException | IOException e) {
        throw new SchemaChangeException("Failure while attempting to load generated class", e);
    }
}

Example 37 with Ordering

use of org.apache.drill.common.logical.data.Order.Ordering in project drill by axbaretto.

the class OrderedPartitionRecordBatch method buildTable.

private void buildTable() throws SchemaChangeException, ClassTransformationException, IOException {
    // Get all samples from distributed map
    @SuppressWarnings("resource") SortRecordBatchBuilder containerBuilder = new SortRecordBatchBuilder(context.getAllocator());
    final VectorContainer allSamplesContainer = new VectorContainer();
    final VectorContainer candidatePartitionTable = new VectorContainer();
    CachedVectorContainer wrap = null;
    try {
        for (CachedVectorContainer w : mmap.get(mapKey)) {
            containerBuilder.add(w.get());
        }
        containerBuilder.build(allSamplesContainer);
        List<Ordering> orderDefs = Lists.newArrayList();
        int i = 0;
        for (Ordering od : popConfig.getOrderings()) {
            SchemaPath sp = SchemaPath.getSimplePath("f" + i++);
            orderDefs.add(new Ordering(od.getDirection(), new FieldReference(sp)));
        }
        // sort the data incoming samples.
        @SuppressWarnings("resource") SelectionVector4 newSv4 = containerBuilder.getSv4();
        Sorter sorter = SortBatch.createNewSorter(context, orderDefs, allSamplesContainer);
        sorter.setup(context, newSv4, allSamplesContainer);
        sorter.sort(newSv4, allSamplesContainer);
        // Copy every Nth record from the samples into a candidate partition table, where N = totalSampledRecords/partitions
        // Attempt to push this to the distributed map. Only the first candidate to get pushed will be used.
        SampleCopier copier = null;
        List<ValueVector> localAllocationVectors = Lists.newArrayList();
        copier = getCopier(newSv4, allSamplesContainer, candidatePartitionTable, orderDefs, localAllocationVectors);
        int allocationSize = 50;
        while (true) {
            for (ValueVector vv : localAllocationVectors) {
                AllocationHelper.allocate(vv, samplingFactor * partitions, allocationSize);
            }
            int skipRecords = containerBuilder.getSv4().getTotalCount() / partitions;
            if (copier.copyRecords(skipRecords, skipRecords, partitions - 1)) {
                assert copier.getOutputRecords() == partitions - 1 : String.format("output records: %d partitions: %d", copier.getOutputRecords(), partitions);
                for (VectorWrapper<?> vw : candidatePartitionTable) {
                    vw.getValueVector().getMutator().setValueCount(copier.getOutputRecords());
                }
                break;
            } else {
                candidatePartitionTable.zeroVectors();
                allocationSize *= 2;
            }
        }
        candidatePartitionTable.setRecordCount(copier.getOutputRecords());
        @SuppressWarnings("resource") WritableBatch batch = WritableBatch.getBatchNoHVWrap(candidatePartitionTable.getRecordCount(), candidatePartitionTable, false);
        wrap = new CachedVectorContainer(batch, context.getAllocator());
        tableMap.putIfAbsent(mapKey + "final", wrap, 1, TimeUnit.MINUTES);
    } finally {
        candidatePartitionTable.clear();
        allSamplesContainer.clear();
        containerBuilder.clear();
        containerBuilder.close();
        if (wrap != null) {
            wrap.clear();
        }
    }
}

Example 38 with Ordering

use of org.apache.drill.common.logical.data.Order.Ordering in project drill by axbaretto.

the class OrderTest method test_Ordering_roundTripDescAndNullsLast.

@Test
public void test_Ordering_roundTripDescAndNullsLast() {
    Ordering src = new Ordering(Direction.DESCENDING, null, NullDirection.LAST);
    Ordering reconstituted = new Ordering(src.getDirection(), (LogicalExpression) null, src.getNullDirection());
    assertThat(reconstituted.getDirection(), equalTo(RelFieldCollation.Direction.DESCENDING));
    assertThat(reconstituted.getNullDirection(), equalTo(NullDirection.LAST));
}

Example 39 with Ordering

use of org.apache.drill.common.logical.data.Order.Ordering in project drill by axbaretto.

the class OrderTest method test_Ordering_roundTripDescAndNullsUnspecified.

@Test
public void test_Ordering_roundTripDescAndNullsUnspecified() {
    Ordering src = new Ordering(Direction.DESCENDING, null, NullDirection.UNSPECIFIED);
    Ordering reconstituted = new Ordering(src.getDirection(), (LogicalExpression) null, src.getNullDirection());
    assertThat(reconstituted.getDirection(), equalTo(RelFieldCollation.Direction.DESCENDING));
    assertThat(reconstituted.getNullDirection(), equalTo(NullDirection.UNSPECIFIED));
}

Example 40 with Ordering

use of org.apache.drill.common.logical.data.Order.Ordering in project drill by apache.

the class OrderTest method testOrdering_nullStrings.

// Defaults-value/null-strings test:
@Test
public void testOrdering_nullStrings() {
    Ordering ordering = new Ordering((String) null, (LogicalExpression) null, null);
    assertThat(ordering.getDirection(), equalTo(RelFieldCollation.Direction.ASCENDING));
    assertThat(ordering.getNullDirection(), equalTo(RelFieldCollation.NullDirection.UNSPECIFIED));
    assertThat(ordering.getOrder(), equalTo("ASC"));
}