Examples with AlgebricksAbsolutePartitionConstraint org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint used on opensource projects

Example 11 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class APIFrameworkTest method testChooseLocations.

@Test
public void testChooseLocations() throws Exception {
    // Mocks cluster info collector.
    IClusterInfoCollector clusterInfoCollector = mock(IClusterInfoCollector.class);
    // Constructs mocked cluster nodes.
    Map<String, NodeControllerInfo> map = new HashMap<>();
    NodeControllerInfo nc1Info = mock(NodeControllerInfo.class);
    when(nc1Info.getNumAvailableCores()).thenReturn(1);
    NodeControllerInfo nc2Info = mock(NodeControllerInfo.class);
    when(nc2Info.getNumAvailableCores()).thenReturn(1);
    String nc1 = "nc1";
    String nc2 = "nc2";
    map.put(nc1, nc1Info);
    map.put(nc2, nc2Info);
    when(clusterInfoCollector.getNodeControllerInfos()).thenReturn(map);
    // Creates an APIFramework.
    APIFramework apiFramework = new APIFramework(mock(ILangCompilationProvider.class));
    // Tests large storage locations.
    AlgebricksAbsolutePartitionConstraint storageLocations = new AlgebricksAbsolutePartitionConstraint(new String[] { "node1", "node1", "node2" });
    AlgebricksAbsolutePartitionConstraint computationLocations = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework, "chooseLocations(" + IClusterInfoCollector.class.getName() + ",int," + AlgebricksAbsolutePartitionConstraint.class.getName() + ")", clusterInfoCollector, CompilerProperties.COMPILER_PARALLELISM_AS_STORAGE, storageLocations);
    Assert.assertTrue(computationLocations.getLocations().length == 2);
    // Tests suitable storage locations.
    storageLocations = new AlgebricksAbsolutePartitionConstraint(new String[] { "node1", "node2" });
    computationLocations = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework, "chooseLocations(" + IClusterInfoCollector.class.getName() + ",int," + AlgebricksAbsolutePartitionConstraint.class.getName() + ")", clusterInfoCollector, CompilerProperties.COMPILER_PARALLELISM_AS_STORAGE, storageLocations);
    Assert.assertTrue(computationLocations.getLocations().length == 2);
    // Tests small storage locations.
    storageLocations = new AlgebricksAbsolutePartitionConstraint(new String[] { "node1" });
    computationLocations = (AlgebricksAbsolutePartitionConstraint) PA.invokeMethod(apiFramework, "chooseLocations(" + IClusterInfoCollector.class.getName() + ",int," + AlgebricksAbsolutePartitionConstraint.class.getName() + ")", clusterInfoCollector, CompilerProperties.COMPILER_PARALLELISM_AS_STORAGE, storageLocations);
    Assert.assertTrue(computationLocations.getLocations().length == 1);
    // Verifies the number of calls on clusterInfoCollector.getNodeControllerInfos() in
    // APIFramework.chooseLocations(...).
    verify(clusterInfoCollector, times(3)).getNodeControllerInfos();
}

Example 12 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class IExternalDataSourceFactory method getPartitionConstraints.

/**
     * returns the passed partition constraints if not null, otherwise returns round robin absolute partition
     * constraints that matches the count.
     *
     * @param constraints
     * @param count
     * @return
     * @throws AlgebricksException
     */
public static AlgebricksAbsolutePartitionConstraint getPartitionConstraints(IApplicationContext appCtx, AlgebricksAbsolutePartitionConstraint constraints, int count) throws AlgebricksException {
    if (constraints == null) {
        ArrayList<String> locs = new ArrayList<>();
        Set<String> stores = appCtx.getMetadataProperties().getStores().keySet();
        if (stores.isEmpty()) {
            throw new AlgebricksException("Configurations don't have any stores");
        }
        int i = 0;
        outer: while (i < count) {
            Iterator<String> storeIt = stores.iterator();
            while (storeIt.hasNext()) {
                String node = storeIt.next();
                int numIODevices = ClusterStateManager.INSTANCE.getIODevices(node).length;
                for (int k = 0; k < numIODevices; k++) {
                    locs.add(node);
                    i++;
                    if (i == count) {
                        break outer;
                    }
                }
            }
            if (i == 0) {
                throw new AlgebricksException("All stores have 0 IO devices");
            }
        }
        return new AlgebricksAbsolutePartitionConstraint(locs.toArray(new String[locs.size()]));
    }
    return constraints;
}

Example 13 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class RSSRecordReaderFactory method getPartitionConstraint.

@Override
public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() throws AlgebricksException {
    int count = urls.size();
    clusterLocations = IExternalDataSourceFactory.getPartitionConstraints((IApplicationContext) serviceContext.getApplicationContext(), clusterLocations, count);
    return clusterLocations;
}

Example 14 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class TwitterFirehoseStreamFactory method getPartitionConstraint.

@Override
public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
    String ingestionCardinalityParam = configuration.get(KEY_INGESTION_CARDINALITY);
    String ingestionLocationParam = configuration.get(KEY_INGESTION_LOCATIONS);
    String[] locations = null;
    if (ingestionLocationParam != null) {
        locations = ingestionLocationParam.split(",");
    }
    int count = locations != null ? locations.length : 1;
    if (ingestionCardinalityParam != null) {
        count = Integer.parseInt(ingestionCardinalityParam);
    }
    List<String> chosenLocations = new ArrayList<>();
    String[] availableLocations = locations != null ? locations : ClusterStateManager.INSTANCE.getParticipantNodes().toArray(new String[] {});
    for (int i = 0, k = 0; i < count; i++, k = (k + 1) % availableLocations.length) {
        chosenLocations.add(availableLocations[k]);
    }
    return new AlgebricksAbsolutePartitionConstraint(chosenLocations.toArray(new String[] {}));
}

Example 15 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class PigletMetadataProvider method getWriteFileRuntime.

@Override
public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink, int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc) throws AlgebricksException {
    PigletFileDataSink ds = (PigletFileDataSink) sink;
    FileSplit[] fileSplits = ds.getFileSplits();
    String[] locations = new String[fileSplits.length];
    for (int i = 0; i < fileSplits.length; ++i) {
        locations[i] = fileSplits[i].getNodeName();
    }
    IPushRuntimeFactory prf;
    try {
        prf = new SinkWriterRuntimeFactory(printColumns, printerFactories, fileSplits[0].getFile(null), PrinterBasedWriterFactory.INSTANCE, inputDesc);
        AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
        return new Pair<>(prf, constraint);
    } catch (HyracksDataException e) {
        throw new AlgebricksException(e);
    }
}