Search in sources :

Example 6 with EntityId

use of org.apache.asterix.active.EntityId in project asterixdb by apache.

the class QueryTranslator method handleDropFeedStatement.

protected void handleDropFeedStatement(MetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc) throws Exception {
    FeedDropStatement stmtFeedDrop = (FeedDropStatement) stmt;
    String dataverseName = getActiveDataverse(stmtFeedDrop.getDataverseName());
    String feedName = stmtFeedDrop.getFeedName().getValue();
    MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
    metadataProvider.setMetadataTxnContext(mdTxnCtx);
    MetadataLockManager.INSTANCE.dropFeedBegin(metadataProvider.getLocks(), dataverseName, dataverseName + "." + feedName);
    try {
        Feed feed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, dataverseName, feedName);
        if (feed == null) {
            if (!stmtFeedDrop.getIfExists()) {
                throw new AlgebricksException("There is no feed with this name " + feedName + ".");
            }
            MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
            return;
        }
        EntityId feedId = new EntityId(Feed.EXTENSION_NAME, dataverseName, feedName);
        ActiveLifecycleListener activeListener = (ActiveLifecycleListener) appCtx.getActiveLifecycleListener();
        ActiveJobNotificationHandler activeEventHandler = activeListener.getNotificationHandler();
        FeedEventsListener listener = (FeedEventsListener) activeEventHandler.getActiveEntityListener(feedId);
        if (listener != null) {
            throw new AlgebricksException("Feed " + feedId + " is currently active and connected to the following dataset(s) \n" + listener.toString());
        } else {
            JobSpecification spec = FeedOperations.buildRemoveFeedStorageJob(metadataProvider, MetadataManager.INSTANCE.getFeed(mdTxnCtx, feedId.getDataverse(), feedId.getEntityName()));
            JobUtils.runJob(hcc, spec, true);
            MetadataManager.INSTANCE.dropFeed(mdTxnCtx, dataverseName, feedName);
        }
        if (LOGGER.isLoggable(Level.INFO)) {
            LOGGER.info("Removed feed " + feedId);
        }
        MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
    } catch (Exception e) {
        abort(e, e, mdTxnCtx);
        throw e;
    } finally {
        metadataProvider.getLocks().unlock();
    }
}
Also used : EntityId(org.apache.asterix.active.EntityId) FeedDropStatement(org.apache.asterix.lang.common.statement.FeedDropStatement) ActiveLifecycleListener(org.apache.asterix.active.ActiveLifecycleListener) FeedEventsListener(org.apache.asterix.external.feed.management.FeedEventsListener) AlgebricksException(org.apache.hyracks.algebricks.common.exceptions.AlgebricksException) MetadataTransactionContext(org.apache.asterix.metadata.MetadataTransactionContext) JobSpecification(org.apache.hyracks.api.job.JobSpecification) ActiveJobNotificationHandler(org.apache.asterix.active.ActiveJobNotificationHandler) ACIDException(org.apache.asterix.common.exceptions.ACIDException) MetadataException(org.apache.asterix.metadata.MetadataException) AlgebricksException(org.apache.hyracks.algebricks.common.exceptions.AlgebricksException) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) CompilationException(org.apache.asterix.common.exceptions.CompilationException) IOException(java.io.IOException) RemoteException(java.rmi.RemoteException) AsterixException(org.apache.asterix.common.exceptions.AsterixException) Feed(org.apache.asterix.metadata.entities.Feed)

Example 7 with EntityId

use of org.apache.asterix.active.EntityId in project asterixdb by apache.

the class QueryTranslator method handleStartFeedStatement.

private void handleStartFeedStatement(MetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc) throws Exception {
    StartFeedStatement sfs = (StartFeedStatement) stmt;
    String dataverseName = getActiveDataverse(sfs.getDataverseName());
    String feedName = sfs.getFeedName().getValue();
    // Transcation handler
    MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
    metadataProvider.setMetadataTxnContext(mdTxnCtx);
    // Runtime handler
    EntityId entityId = new EntityId(Feed.EXTENSION_NAME, dataverseName, feedName);
    // Feed & Feed Connections
    Feed feed = FeedMetadataUtil.validateIfFeedExists(dataverseName, feedName, metadataProvider.getMetadataTxnContext());
    List<FeedConnection> feedConnections = MetadataManager.INSTANCE.getFeedConections(metadataProvider.getMetadataTxnContext(), dataverseName, feedName);
    ILangCompilationProvider compilationProvider = new AqlCompilationProvider();
    IStorageComponentProvider storageComponentProvider = new StorageComponentProvider();
    DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory();
    ActiveLifecycleListener activeListener = (ActiveLifecycleListener) appCtx.getActiveLifecycleListener();
    ActiveJobNotificationHandler activeEventHandler = activeListener.getNotificationHandler();
    FeedEventsListener listener = (FeedEventsListener) activeEventHandler.getActiveEntityListener(entityId);
    if (listener != null) {
        throw new AlgebricksException("Feed " + feedName + " is started already.");
    }
    // Start
    MetadataLockManager.INSTANCE.startFeedBegin(metadataProvider.getLocks(), dataverseName, dataverseName + "." + feedName, feedConnections);
    try {
        // Prepare policy
        List<IDataset> datasets = new ArrayList<>();
        for (FeedConnection connection : feedConnections) {
            Dataset ds = metadataProvider.findDataset(connection.getDataverseName(), connection.getDatasetName());
            datasets.add(ds);
        }
        org.apache.commons.lang3.tuple.Pair<JobSpecification, AlgebricksAbsolutePartitionConstraint> jobInfo = FeedOperations.buildStartFeedJob(sessionOutput, metadataProvider, feed, feedConnections, compilationProvider, storageComponentProvider, qtFactory, hcc);
        JobSpecification feedJob = jobInfo.getLeft();
        listener = new FeedEventsListener(appCtx, entityId, datasets, jobInfo.getRight().getLocations());
        activeEventHandler.registerListener(listener);
        IActiveEventSubscriber eventSubscriber = listener.subscribe(ActivityState.STARTED);
        feedJob.setProperty(ActiveJobNotificationHandler.ACTIVE_ENTITY_PROPERTY_NAME, entityId);
        JobUtils.runJob(hcc, feedJob, Boolean.valueOf(metadataProvider.getConfig().get(StartFeedStatement.WAIT_FOR_COMPLETION)));
        eventSubscriber.sync();
        LOGGER.log(Level.INFO, "Submitted");
    } catch (Exception e) {
        abort(e, e, mdTxnCtx);
        if (listener != null) {
            activeEventHandler.unregisterListener(listener);
        }
        throw e;
    } finally {
        metadataProvider.getLocks().unlock();
    }
}
Also used : FeedEventsListener(org.apache.asterix.external.feed.management.FeedEventsListener) ArrayList(java.util.ArrayList) MetadataTransactionContext(org.apache.asterix.metadata.MetadataTransactionContext) ActiveLifecycleListener(org.apache.asterix.active.ActiveLifecycleListener) ILangCompilationProvider(org.apache.asterix.compiler.provider.ILangCompilationProvider) IDataset(org.apache.asterix.common.metadata.IDataset) JobSpecification(org.apache.hyracks.api.job.JobSpecification) ActiveJobNotificationHandler(org.apache.asterix.active.ActiveJobNotificationHandler) Feed(org.apache.asterix.metadata.entities.Feed) AqlCompilationProvider(org.apache.asterix.compiler.provider.AqlCompilationProvider) IStorageComponentProvider(org.apache.asterix.common.context.IStorageComponentProvider) FeedConnection(org.apache.asterix.metadata.entities.FeedConnection) IHyracksDataset(org.apache.hyracks.api.dataset.IHyracksDataset) IDataset(org.apache.asterix.common.metadata.IDataset) Dataset(org.apache.asterix.metadata.entities.Dataset) IActiveEventSubscriber(org.apache.asterix.active.IActiveEventSubscriber) AlgebricksException(org.apache.hyracks.algebricks.common.exceptions.AlgebricksException) IStorageComponentProvider(org.apache.asterix.common.context.IStorageComponentProvider) StorageComponentProvider(org.apache.asterix.file.StorageComponentProvider) ACIDException(org.apache.asterix.common.exceptions.ACIDException) MetadataException(org.apache.asterix.metadata.MetadataException) AlgebricksException(org.apache.hyracks.algebricks.common.exceptions.AlgebricksException) HyracksDataException(org.apache.hyracks.api.exceptions.HyracksDataException) CompilationException(org.apache.asterix.common.exceptions.CompilationException) IOException(java.io.IOException) RemoteException(java.rmi.RemoteException) AsterixException(org.apache.asterix.common.exceptions.AsterixException) EntityId(org.apache.asterix.active.EntityId) StartFeedStatement(org.apache.asterix.lang.common.statement.StartFeedStatement) AlgebricksAbsolutePartitionConstraint(org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint)

Example 8 with EntityId

use of org.apache.asterix.active.EntityId in project asterixdb by apache.

the class InputHandlerTest method createInputHandler.

private FeedRuntimeInputHandler createInputHandler(IHyracksTaskContext ctx, IFrameWriter writer, FeedPolicyAccessor fpa, ConcurrentFramePool framePool) throws HyracksDataException {
    FrameTupleAccessor fta = Mockito.mock(FrameTupleAccessor.class);
    EntityId feedId = new EntityId(FeedUtils.FEED_EXTENSION_NAME, DATAVERSE, FEED);
    FeedConnectionId connectionId = new FeedConnectionId(feedId, DATASET);
    ActiveRuntimeId runtimeId = new ActiveRuntimeId(feedId, FeedRuntimeType.COLLECT.toString(), 0);
    return new FeedRuntimeInputHandler(ctx, connectionId, runtimeId, writer, fpa, fta, framePool);
}
Also used : EntityId(org.apache.asterix.active.EntityId) FeedRuntimeInputHandler(org.apache.asterix.external.feed.dataflow.FeedRuntimeInputHandler) ActiveRuntimeId(org.apache.asterix.active.ActiveRuntimeId) FeedConnectionId(org.apache.asterix.external.feed.management.FeedConnectionId) FrameTupleAccessor(org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor)

Example 9 with EntityId

use of org.apache.asterix.active.EntityId in project asterixdb by apache.

the class SubscribeFeedStatement method initialize.

public void initialize(MetadataTransactionContext mdTxnCtx) throws MetadataException {
    this.query = new Query(false);
    EntityId sourceFeedId = connectionRequest.getReceivingFeedId();
    Feed subscriberFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, connectionRequest.getReceivingFeedId().getDataverse(), connectionRequest.getReceivingFeedId().getEntityName());
    if (subscriberFeed == null) {
        throw new IllegalStateException(" Subscriber feed " + subscriberFeed + " not found.");
    }
    String feedOutputType = getOutputType(mdTxnCtx);
    StringBuilder builder = new StringBuilder();
    builder.append("use dataverse " + sourceFeedId.getDataverse() + ";\n");
    builder.append("set" + " " + FunctionUtil.IMPORT_PRIVATE_FUNCTIONS + " " + "'" + Boolean.TRUE + "'" + ";\n");
    builder.append("set" + " " + FeedActivityDetails.FEED_POLICY_NAME + " " + "'" + connectionRequest.getPolicy() + "'" + ";\n");
    builder.append("insert into dataset " + connectionRequest.getTargetDataset() + " ");
    builder.append(" (" + " for $x in feed-collect ('" + sourceFeedId.getDataverse() + "'" + "," + "'" + sourceFeedId.getEntityName() + "'" + "," + "'" + connectionRequest.getReceivingFeedId().getEntityName() + "'" + "," + "'" + connectionRequest.getSubscriptionLocation().name() + "'" + "," + "'" + connectionRequest.getTargetDataset() + "'" + "," + "'" + feedOutputType + "'" + ")");
    List<FunctionSignature> functionsToApply = connectionRequest.getFunctionsToApply();
    if ((functionsToApply != null) && functionsToApply.isEmpty()) {
        builder.append(" return $x");
    } else {
        Function function;
        String rValueName = "x";
        String lValueName = "y";
        int variableIndex = 0;
        for (FunctionSignature appliedFunction : functionsToApply) {
            function = MetadataManager.INSTANCE.getFunction(mdTxnCtx, appliedFunction);
            variableIndex++;
            switch(function.getLanguage().toUpperCase()) {
                case Function.LANGUAGE_AQL:
                    builder.append(" let " + "$" + lValueName + variableIndex + ":=" + function.getName() + "(" + "$" + rValueName + ")");
                    rValueName = lValueName + variableIndex;
                    break;
                case Function.LANGUAGE_JAVA:
                    builder.append(" let " + "$" + lValueName + variableIndex + ":=" + function.getName() + "(" + "$" + rValueName + ")");
                    rValueName = lValueName + variableIndex;
                    break;
            }
            builder.append("\n");
        }
        builder.append("return $" + lValueName + variableIndex);
    }
    builder.append(")");
    builder.append(";");
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Connect feed statement translated to\n" + builder.toString());
    }
    IParser parser = parserFactory.createParser(new StringReader(builder.toString()));
    List<Statement> statements;
    try {
        statements = parser.parse();
        query = ((InsertStatement) statements.get(INSERT_STATEMENT_POS)).getQuery();
    } catch (CompilationException pe) {
        throw new MetadataException(pe);
    }
}
Also used : CompilationException(org.apache.asterix.common.exceptions.CompilationException) Query(org.apache.asterix.lang.common.statement.Query) InsertStatement(org.apache.asterix.lang.common.statement.InsertStatement) Statement(org.apache.asterix.lang.common.base.Statement) FunctionSignature(org.apache.asterix.common.functions.FunctionSignature) MetadataException(org.apache.asterix.metadata.MetadataException) EntityId(org.apache.asterix.active.EntityId) Function(org.apache.asterix.metadata.entities.Function) StringReader(java.io.StringReader) Feed(org.apache.asterix.metadata.entities.Feed) IParser(org.apache.asterix.lang.common.base.IParser)

Aggregations

EntityId (org.apache.asterix.active.EntityId)9 MetadataException (org.apache.asterix.metadata.MetadataException)8 CompilationException (org.apache.asterix.common.exceptions.CompilationException)7 IOException (java.io.IOException)6 RemoteException (java.rmi.RemoteException)6 ActiveJobNotificationHandler (org.apache.asterix.active.ActiveJobNotificationHandler)6 ActiveLifecycleListener (org.apache.asterix.active.ActiveLifecycleListener)6 ACIDException (org.apache.asterix.common.exceptions.ACIDException)6 AsterixException (org.apache.asterix.common.exceptions.AsterixException)6 MetadataTransactionContext (org.apache.asterix.metadata.MetadataTransactionContext)6 AlgebricksException (org.apache.hyracks.algebricks.common.exceptions.AlgebricksException)6 HyracksDataException (org.apache.hyracks.api.exceptions.HyracksDataException)6 Feed (org.apache.asterix.metadata.entities.Feed)5 FunctionSignature (org.apache.asterix.common.functions.FunctionSignature)3 FeedEventsListener (org.apache.asterix.external.feed.management.FeedEventsListener)3 AlgebricksAbsolutePartitionConstraint (org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint)3 JobSpecification (org.apache.hyracks.api.job.JobSpecification)3 ArrayList (java.util.ArrayList)2 IActiveEventSubscriber (org.apache.asterix.active.IActiveEventSubscriber)2 IDataset (org.apache.asterix.common.metadata.IDataset)2