Examples with ClusterPartition org.apache.asterix.common.cluster.ClusterPartition used on opensource projects

Example 11 with ClusterPartition

use of org.apache.asterix.common.cluster.ClusterPartition in project asterixdb by apache.

the class AutoFaultToleranceStrategy method prepareFailbackPlan.

private synchronized void prepareFailbackPlan(String failingBackNodeId) {
    NodeFailbackPlan plan = NodeFailbackPlan.createPlan(failingBackNodeId);
    pendingProcessingFailbackPlans.add(plan);
    planId2FailbackPlanMap.put(plan.getPlanId(), plan);
    //get all partitions this node requires to resync
    ICcApplicationContext appCtx = (ICcApplicationContext) serviceCtx.getApplicationContext();
    ReplicationProperties replicationProperties = appCtx.getReplicationProperties();
    Set<String> nodeReplicas = replicationProperties.getNodeReplicasIds(failingBackNodeId);
    clusterManager.getClusterPartitons();
    for (String replicaId : nodeReplicas) {
        ClusterPartition[] nodePartitions = clusterManager.getNodePartitions(replicaId);
        for (ClusterPartition partition : nodePartitions) {
            plan.addParticipant(partition.getActiveNodeId());
            /*
                 * if the partition original node is the returning node,
                 * add it to the list of the partitions which will be failed back
                 */
            if (partition.getNodeId().equals(failingBackNodeId)) {
                plan.addPartitionToFailback(partition.getPartitionId(), partition.getActiveNodeId());
            }
        }
    }
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Prepared Failback plan: " + plan.toString());
    }
    processPendingFailbackPlans();
}

Example 12 with ClusterPartition

use of org.apache.asterix.common.cluster.ClusterPartition in project asterixdb by apache.

the class AutoFaultToleranceStrategy method getNodeAssignedPartitions.

public synchronized List<ClusterPartition> getNodeAssignedPartitions(String nodeId) {
    List<ClusterPartition> nodePartitions = new ArrayList<>();
    ClusterPartition[] clusterPartitons = clusterManager.getClusterPartitons();
    Map<Integer, ClusterPartition> clusterPartitionsMap = new HashMap<>();
    for (ClusterPartition partition : clusterPartitons) {
        clusterPartitionsMap.put(partition.getPartitionId(), partition);
    }
    for (ClusterPartition partition : clusterPartitons) {
        if (nodeId.equals(partition.getActiveNodeId())) {
            nodePartitions.add(partition);
        }
    }
    /*
         * if there is any pending takeover request this node was supposed to handle,
         * it needs to be sent to a different replica
         */
    List<Long> failedTakeoverRequests = new ArrayList<>();
    for (TakeoverPartitionsRequestMessage request : pendingTakeoverRequests.values()) {
        if (request.getNodeId().equals(nodeId)) {
            for (Integer partitionId : request.getPartitions()) {
                nodePartitions.add(clusterPartitionsMap.get(partitionId));
            }
            failedTakeoverRequests.add(request.getRequestId());
        }
    }
    //remove failed requests
    for (Long requestId : failedTakeoverRequests) {
        pendingTakeoverRequests.remove(requestId);
    }
    return nodePartitions;
}

Example 13 with ClusterPartition

use of org.apache.asterix.common.cluster.ClusterPartition in project asterixdb by apache.

the class AutoFaultToleranceStrategy method requestMetadataNodeTakeover.

private synchronized void requestMetadataNodeTakeover() {
    //need a new node to takeover metadata node
    ICcApplicationContext appCtx = (ICcApplicationContext) serviceCtx.getApplicationContext();
    ClusterPartition metadataPartiton = appCtx.getMetadataProperties().getMetadataPartition();
    //request the metadataPartition node to register itself as the metadata node
    TakeoverMetadataNodeRequestMessage takeoverRequest = new TakeoverMetadataNodeRequestMessage();
    try {
        messageBroker.sendApplicationMessageToNC(takeoverRequest, metadataPartiton.getActiveNodeId());
        // Since the metadata node will be changed, we need to rebind the proxy object
        MetadataManager.INSTANCE.rebindMetadataNode();
    } catch (Exception e) {
        /*
             * if we fail to send the request, it means the NC we tried to send the request to
             * has failed. When the failure notification arrives, a new NC will be assigned to
             * the metadata partition and a new metadata node takeover request will be sent to it.
             */
        LOGGER.log(Level.WARNING, "Failed to send metadata node takeover request to: " + metadataPartiton.getActiveNodeId(), e);
    }
}

Example 14 with ClusterPartition

use of org.apache.asterix.common.cluster.ClusterPartition in project asterixdb by apache.

the class TestTypedAdapterFactory method createAdapter.

@Override
public IDataSourceAdapter createAdapter(IHyracksTaskContext ctx, int partition) throws HyracksDataException {
    final String nodeId = ctx.getJobletContext().getServiceContext().getNodeId();
    final ITupleParserFactory tupleParserFactory = new ITupleParserFactory() {

        private static final long serialVersionUID = 1L;

        @Override
        public ITupleParser createTupleParser(IHyracksTaskContext ctx) throws HyracksDataException {
            ADMDataParser parser;
            ITupleForwarder forwarder;
            ArrayTupleBuilder tb;
            IApplicationContext appCtx = (IApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
            ClusterPartition nodePartition = appCtx.getMetadataProperties().getNodePartitions().get(nodeId)[0];
            parser = new ADMDataParser(outputType, true);
            forwarder = DataflowUtils.getTupleForwarder(configuration, FeedUtils.getFeedLogManager(ctx, FeedUtils.splitsForAdapter(ExternalDataUtils.getDataverse(configuration), ExternalDataUtils.getFeedName(configuration), nodeId, nodePartition)));
            tb = new ArrayTupleBuilder(1);
            return new ITupleParser() {

                @Override
                public void parse(InputStream in, IFrameWriter writer) throws HyracksDataException {
                    try {
                        parser.setInputStream(in);
                        forwarder.initialize(ctx, writer);
                        while (true) {
                            tb.reset();
                            if (!parser.parse(tb.getDataOutput())) {
                                break;
                            }
                            tb.addFieldEndOffset();
                            forwarder.addTuple(tb);
                        }
                        forwarder.close();
                    } catch (Exception e) {
                        throw new HyracksDataException(e);
                    }
                }
            };
        }
    };
    try {
        return new TestTypedAdapter(tupleParserFactory, outputType, ctx, configuration, partition);
    } catch (IOException e) {
        throw new HyracksDataException(e);
    }
}

Example 15 with ClusterPartition

use of org.apache.asterix.common.cluster.ClusterPartition in project asterixdb by apache.

the class NCAppRuntimeContext method initialize.

@Override
public void initialize(boolean initialRun) throws IOException, ACIDException {
    ioManager = getServiceContext().getIoManager();
    threadExecutor = new ThreadExecutor(getServiceContext().getThreadFactory());
    fileMapManager = new FileMapManager(ioManager);
    ICacheMemoryAllocator allocator = new HeapBufferAllocator();
    IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
    IPageReplacementStrategy prs = new ClockPageReplacementStrategy(allocator, storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages());
    AsynchronousScheduler.INSTANCE.init(getServiceContext().getThreadFactory());
    lsmIOScheduler = AsynchronousScheduler.INSTANCE;
    metadataMergePolicyFactory = new PrefixMergePolicyFactory();
    ILocalResourceRepositoryFactory persistentLocalResourceRepositoryFactory = new PersistentLocalResourceRepositoryFactory(ioManager, getServiceContext().getNodeId(), metadataProperties);
    localResourceRepository = (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory.createRepository();
    IAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AppRuntimeContextProviderForRecovery(this);
    txnSubsystem = new TransactionSubsystem(getServiceContext(), getServiceContext().getNodeId(), asterixAppRuntimeContextProvider, txnProperties);
    IRecoveryManager recoveryMgr = txnSubsystem.getRecoveryManager();
    SystemState systemState = recoveryMgr.getSystemState();
    if (initialRun || systemState == SystemState.PERMANENT_DATA_LOSS) {
        //delete any storage data before the resource factory is initialized
        localResourceRepository.deleteStorageData(true);
    }
    datasetLifecycleManager = new DatasetLifecycleManager(storageProperties, localResourceRepository, MetadataIndexImmutableProperties.FIRST_AVAILABLE_USER_DATASET_ID, txnSubsystem.getLogManager(), ioManager.getIODevices().size());
    isShuttingdown = false;
    activeManager = new ActiveManager(threadExecutor, getServiceContext().getNodeId(), activeProperties.getMemoryComponentGlobalBudget(), compilerProperties.getFrameSize());
    if (replicationProperties.isParticipant(getServiceContext().getNodeId())) {
        String nodeId = getServiceContext().getNodeId();
        replicaResourcesManager = new ReplicaResourcesManager(localResourceRepository, metadataProperties);
        replicationManager = new ReplicationManager(nodeId, replicationProperties, replicaResourcesManager, txnSubsystem.getLogManager(), asterixAppRuntimeContextProvider);
        //pass replication manager to replication required object
        //LogManager to replicate logs
        txnSubsystem.getLogManager().setReplicationManager(replicationManager);
        //PersistentLocalResourceRepository to replicate metadata files and delete backups on drop index
        localResourceRepository.setReplicationManager(replicationManager);
        /*
             * add the partitions that will be replicated in this node as inactive partitions
             */
        //get nodes which replicate to this node
        Set<String> remotePrimaryReplicas = replicationProperties.getRemotePrimaryReplicasIds(nodeId);
        for (String clientId : remotePrimaryReplicas) {
            //get the partitions of each client
            ClusterPartition[] clientPartitions = metadataProperties.getNodePartitions().get(clientId);
            for (ClusterPartition partition : clientPartitions) {
                localResourceRepository.addInactivePartition(partition.getPartitionId());
            }
        }
        //initialize replication channel
        replicationChannel = new ReplicationChannel(nodeId, replicationProperties, txnSubsystem.getLogManager(), replicaResourcesManager, replicationManager, getServiceContext(), asterixAppRuntimeContextProvider);
        remoteRecoveryManager = new RemoteRecoveryManager(replicationManager, this, replicationProperties);
        bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager, storageProperties.getBufferCacheMaxOpenFiles(), getServiceContext().getThreadFactory(), replicationManager);
    } else {
        bufferCache = new BufferCache(ioManager, prs, pcp, fileMapManager, storageProperties.getBufferCacheMaxOpenFiles(), getServiceContext().getThreadFactory());
    }
    /*
         * The order of registration is important. The buffer cache must registered before recovery and transaction
         * managers. Notes: registered components are stopped in reversed order
         */
    ILifeCycleComponentManager lccm = getServiceContext().getLifeCycleComponentManager();
    lccm.register((ILifeCycleComponent) bufferCache);
    /*
         * LogManager must be stopped after RecoveryManager, DatasetLifeCycleManager, and ReplicationManager
         * to process any logs that might be generated during stopping these components
         */
    lccm.register((ILifeCycleComponent) txnSubsystem.getLogManager());
    /*
         * ReplicationManager must be stopped after indexLifecycleManager and recovery manager
         * so that any logs/files generated during closing datasets or checkpoints are sent to remote replicas
         */
    if (replicationManager != null) {
        lccm.register(replicationManager);
    }
    lccm.register((ILifeCycleComponent) txnSubsystem.getRecoveryManager());
    /*
         * Stopping indexLifecycleManager will flush and close all datasets.
         */
    lccm.register((ILifeCycleComponent) datasetLifecycleManager);
    lccm.register((ILifeCycleComponent) txnSubsystem.getTransactionManager());
    lccm.register((ILifeCycleComponent) txnSubsystem.getLockManager());
    lccm.register(txnSubsystem.getCheckpointManager());
}