Examples with AlgebricksAbsolutePartitionConstraint org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint used on opensource projects

Example 6 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class ClusterStateManager method resetClusterPartitionConstraint.

private synchronized void resetClusterPartitionConstraint() {
    ArrayList<String> clusterActiveLocations = new ArrayList<>();
    for (ClusterPartition p : clusterPartitions.values()) {
        if (p.isActive()) {
            clusterActiveLocations.add(p.getActiveNodeId());
        }
    }
    clusterPartitionConstraint = new AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new String[] {}));
}

Example 7 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class PigletMetadataProvider method getScannerRuntime.

@Override
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource, List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed, List<LogicalVariable> minFilterVars, List<LogicalVariable> maxFilterVars, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig) throws AlgebricksException {
    PigletFileDataSource ds = (PigletFileDataSource) dataSource;
    FileSplit[] fileSplits = ds.getFileSplits();
    String[] locations = new String[fileSplits.length];
    for (int i = 0; i < fileSplits.length; ++i) {
        locations[i] = fileSplits[i].getNodeName();
    }
    IFileSplitProvider fsp = new ConstantFileSplitProvider(fileSplits);
    Object[] colTypes = ds.getSchemaTypes();
    IValueParserFactory[] vpfs = new IValueParserFactory[colTypes.length];
    ISerializerDeserializer[] serDesers = new ISerializerDeserializer[colTypes.length];
    for (int i = 0; i < colTypes.length; ++i) {
        Type colType = (Type) colTypes[i];
        IValueParserFactory vpf;
        ISerializerDeserializer serDeser;
        switch(colType.getTag()) {
            case INTEGER:
                vpf = IntegerParserFactory.INSTANCE;
                serDeser = IntegerSerializerDeserializer.INSTANCE;
                break;
            case CHAR_ARRAY:
                vpf = UTF8StringParserFactory.INSTANCE;
                serDeser = new UTF8StringSerializerDeserializer();
                break;
            case FLOAT:
                vpf = FloatParserFactory.INSTANCE;
                serDeser = FloatSerializerDeserializer.INSTANCE;
                break;
            default:
                throw new UnsupportedOperationException();
        }
        vpfs[i] = vpf;
        serDesers[i] = serDeser;
    }
    ITupleParserFactory tpf = new DelimitedDataTupleParserFactory(vpfs, ',');
    RecordDescriptor rDesc = new RecordDescriptor(serDesers);
    IOperatorDescriptor scanner = new FileScanOperatorDescriptor(jobSpec, fsp, tpf, rDesc);
    AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
    return new Pair<>(scanner, constraint);
}

Example 8 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class HDFSUtils method getPartitionConstraints.

public static AlgebricksAbsolutePartitionConstraint getPartitionConstraints(IApplicationContext appCtx, AlgebricksAbsolutePartitionConstraint clusterLocations) {
    if (clusterLocations == null) {
        ArrayList<String> locs = new ArrayList<>();
        Map<String, String[]> stores = appCtx.getMetadataProperties().getStores();
        for (String node : stores.keySet()) {
            int numIODevices = ClusterStateManager.INSTANCE.getIODevices(node).length;
            for (int k = 0; k < numIODevices; k++) {
                locs.add(node);
            }
        }
        String[] cluster = new String[locs.size()];
        cluster = locs.toArray(cluster);
        return new AlgebricksAbsolutePartitionConstraint(cluster);
    }
    return clusterLocations;
}

Example 9 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class APIFramework method chooseLocations.

// Chooses the location constraints, i.e., whether to use storage parallelism or use a user-sepcified number
// of cores.
private static AlgebricksAbsolutePartitionConstraint chooseLocations(IClusterInfoCollector clusterInfoCollector, int parallelismHint, AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException {
    try {
        Map<String, NodeControllerInfo> ncMap = clusterInfoCollector.getNodeControllerInfos();
        // Gets total number of cores in the cluster.
        int totalNumCores = getTotalNumCores(ncMap);
        // Otherwise, we will use all available cores.
        if (parallelismHint == CompilerProperties.COMPILER_PARALLELISM_AS_STORAGE && storageLocations.getLocations().length <= totalNumCores) {
            return storageLocations;
        }
        return getComputationLocations(ncMap, parallelismHint);
    } catch (HyracksException e) {
        throw new AlgebricksException(e);
    }
}

Example 10 with AlgebricksAbsolutePartitionConstraint

use of org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint in project asterixdb by apache.

the class QueryTranslator method handleStartFeedStatement.

private void handleStartFeedStatement(MetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc) throws Exception {
    StartFeedStatement sfs = (StartFeedStatement) stmt;
    String dataverseName = getActiveDataverse(sfs.getDataverseName());
    String feedName = sfs.getFeedName().getValue();
    // Transcation handler
    MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
    metadataProvider.setMetadataTxnContext(mdTxnCtx);
    // Runtime handler
    EntityId entityId = new EntityId(Feed.EXTENSION_NAME, dataverseName, feedName);
    // Feed & Feed Connections
    Feed feed = FeedMetadataUtil.validateIfFeedExists(dataverseName, feedName, metadataProvider.getMetadataTxnContext());
    List<FeedConnection> feedConnections = MetadataManager.INSTANCE.getFeedConections(metadataProvider.getMetadataTxnContext(), dataverseName, feedName);
    ILangCompilationProvider compilationProvider = new AqlCompilationProvider();
    IStorageComponentProvider storageComponentProvider = new StorageComponentProvider();
    DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory();
    ActiveLifecycleListener activeListener = (ActiveLifecycleListener) appCtx.getActiveLifecycleListener();
    ActiveJobNotificationHandler activeEventHandler = activeListener.getNotificationHandler();
    FeedEventsListener listener = (FeedEventsListener) activeEventHandler.getActiveEntityListener(entityId);
    if (listener != null) {
        throw new AlgebricksException("Feed " + feedName + " is started already.");
    }
    // Start
    MetadataLockManager.INSTANCE.startFeedBegin(metadataProvider.getLocks(), dataverseName, dataverseName + "." + feedName, feedConnections);
    try {
        // Prepare policy
        List<IDataset> datasets = new ArrayList<>();
        for (FeedConnection connection : feedConnections) {
            Dataset ds = metadataProvider.findDataset(connection.getDataverseName(), connection.getDatasetName());
            datasets.add(ds);
        }
        org.apache.commons.lang3.tuple.Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> jobInfo = FeedOperations.buildStartFeedJob(sessionOutput, metadataProvider, feed, feedConnections, compilationProvider, storageComponentProvider, qtFactory, hcc);
        JobSpecification feedJob = jobInfo.getLeft();
        listener = new FeedEventsListener(appCtx, entityId, datasets, jobInfo.getRight().getLocations());
        activeEventHandler.registerListener(listener);
        IActiveEventSubscriber eventSubscriber = listener.subscribe(ActivityState.STARTED);
        feedJob.setProperty(ActiveJobNotificationHandler.ACTIVE_ENTITY_PROPERTY_NAME, entityId);
        JobUtils.runJob(hcc, feedJob, Boolean.valueOf(metadataProvider.getConfig().get(StartFeedStatement.WAIT_FOR_COMPLETION)));
        eventSubscriber.sync();
        LOGGER.log(Level.INFO, "Submitted");
    } catch (Exception e) {
        abort(e, e, mdTxnCtx);
        if (listener != null) {
            activeEventHandler.unregisterListener(listener);
        }
        throw e;
    } finally {
        metadataProvider.getLocks().unlock();
    }
}