Examples with IClusterCapacity org.apache.hyracks.api.job.resource.IClusterCapacity used on opensource projects

Example 6 with IClusterCapacity

use of org.apache.hyracks.api.job.resource.IClusterCapacity in project asterixdb by apache.

the class RequiredCapacityVisitorTest method testUnPartitionedJoin.

@Test
public void testUnPartitionedJoin() throws AlgebricksException {
    IClusterCapacity clusterCapacity = new ClusterCapacity();
    RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity);
    // Constructs a join query plan.
    InnerJoinOperator join = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    // Left child plan of the join.
    ExchangeOperator leftChildExchange = new ExchangeOperator();
    leftChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    leftChildExchange.setPhysicalOperator(new OneToOneExchangePOperator());
    InnerJoinOperator leftChild = makeJoinOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    join.getInputs().add(new MutableObject<>(leftChildExchange));
    leftChildExchange.getInputs().add(new MutableObject<>(leftChild));
    EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
    ets.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    leftChild.getInputs().add(new MutableObject<>(ets));
    leftChild.getInputs().add(new MutableObject<>(ets));
    // Right child plan of the join.
    ExchangeOperator rightChildExchange = new ExchangeOperator();
    rightChildExchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    rightChildExchange.setPhysicalOperator(new OneToOneExchangePOperator());
    GroupByOperator rightChild = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    join.getInputs().add(new MutableObject<>(rightChildExchange));
    rightChildExchange.getInputs().add(new MutableObject<>(rightChild));
    rightChild.getInputs().add(new MutableObject<>(ets));
    // Verifies the calculated cluster capacity requirement for the test quer plan.
    join.accept(visitor, null);
    Assert.assertTrue(clusterCapacity.getAggregatedCores() == 1);
    Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 3 * MEMORY_BUDGET + 5L * FRAME_SIZE);
}

Example 7 with IClusterCapacity

use of org.apache.hyracks.api.job.resource.IClusterCapacity in project asterixdb by apache.

the class SleepOperatorDescriptor method setJobCapacity.

private void setJobCapacity(JobSpecification spec) {
    IClusterCapacity reqCapacity = new ClusterCapacity();
    reqCapacity.setAggregatedMemoryByteSize(Long.MAX_VALUE);
    spec.setRequiredClusterCapacity(reqCapacity);
}

Example 8 with IClusterCapacity

use of org.apache.hyracks.api.job.resource.IClusterCapacity in project asterixdb by apache.

the class ResourceUtils method getRequiredCompacity.

/**
     * Calculates the required cluster capacity from a given query plan, the computation locations,
     * the operator memory budgets, and frame size.
     *
     * @param plan,
     *            a given query plan.
     * @param computationLocations,
     *            the partitions for computation.
     * @param sortFrameLimit,
     *            the frame limit for one sorter partition.
     * @param groupFrameLimit,
     *            the frame limit for one group-by partition.
     * @param joinFrameLimit
     *            the frame limit for one joiner partition.
     * @param frameSize
     *            the frame size used in query execution.
     * @return the required cluster capacity for executing the query.
     * @throws AlgebricksException
     *             if the query plan is malformed.
     */
public static IClusterCapacity getRequiredCompacity(ILogicalPlan plan, AlgebricksAbsolutePartitionConstraint computationLocations, int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int frameSize) throws AlgebricksException {
    // Creates a cluster capacity visitor.
    IClusterCapacity clusterCapacity = new ClusterCapacity();
    RequiredCapacityVisitor visitor = new RequiredCapacityVisitor(computationLocations.getLocations().length, sortFrameLimit, groupFrameLimit, joinFrameLimit, frameSize, clusterCapacity);
    // There could be only one root operator for a top-level query plan.
    ILogicalOperator rootOp = plan.getRoots().get(0).getValue();
    rootOp.accept(visitor, null);
    return clusterCapacity;
}

Example 9 with IClusterCapacity

use of org.apache.hyracks.api.job.resource.IClusterCapacity in project asterixdb by apache.

the class RequiredCapacityVisitorTest method testUnPartitionedGroupBy.

@Test
public void testUnPartitionedGroupBy() throws AlgebricksException {
    IClusterCapacity clusterCapacity = new ClusterCapacity();
    RequiredCapacityVisitor visitor = makeComputationCapacityVisitor(PARALLELISM, clusterCapacity);
    // Constructs a parallel group-by query plan.
    GroupByOperator globalGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    ExchangeOperator exchange = new ExchangeOperator();
    exchange.setPhysicalOperator(new OneToOneExchangePOperator());
    exchange.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    GroupByOperator localGby = makeGroupByOperator(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
    globalGby.getInputs().add(new MutableObject<>(exchange));
    exchange.getInputs().add(new MutableObject<>(localGby));
    // Verifies the calculated cluster capacity requirement for the test quer plan.
    globalGby.accept(visitor, null);
    Assert.assertTrue(clusterCapacity.getAggregatedCores() == 1);
    Assert.assertTrue(clusterCapacity.getAggregatedMemoryByteSize() == 2 * MEMORY_BUDGET + FRAME_SIZE);
}

Example 10 with IClusterCapacity

use of org.apache.hyracks.api.job.resource.IClusterCapacity in project asterixdb by apache.

the class JobCapacityController method allocate.

@Override
public JobSubmissionStatus allocate(JobSpecification job) throws HyracksException {
    IClusterCapacity requiredCapacity = job.getRequiredClusterCapacity();
    long reqAggregatedMemoryByteSize = requiredCapacity.getAggregatedMemoryByteSize();
    int reqAggregatedNumCores = requiredCapacity.getAggregatedCores();
    IReadOnlyClusterCapacity maximumCapacity = resourceManager.getMaximumCapacity();
    if (!(reqAggregatedMemoryByteSize <= maximumCapacity.getAggregatedMemoryByteSize() && reqAggregatedNumCores <= maximumCapacity.getAggregatedCores())) {
        throw HyracksException.create(ErrorCode.JOB_REQUIREMENTS_EXCEED_CAPACITY, requiredCapacity.toString(), maximumCapacity.toString());
    }
    IClusterCapacity currentCapacity = resourceManager.getCurrentCapacity();
    long currentAggregatedMemoryByteSize = currentCapacity.getAggregatedMemoryByteSize();
    int currentAggregatedAvailableCores = currentCapacity.getAggregatedCores();
    if (!(reqAggregatedMemoryByteSize <= currentAggregatedMemoryByteSize && reqAggregatedNumCores <= currentAggregatedAvailableCores)) {
        return JobSubmissionStatus.QUEUE;
    }
    currentCapacity.setAggregatedMemoryByteSize(currentAggregatedMemoryByteSize - reqAggregatedMemoryByteSize);
    currentCapacity.setAggregatedCores(currentAggregatedAvailableCores - reqAggregatedNumCores);
    return JobSubmissionStatus.EXECUTE;
}