Examples with StringCodec com.datatorrent.api.StringCodec used on opensource projects

Example 6 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class StringCodecs method check.

public static void check() {
    if (classLoaders.putIfAbsent(Thread.currentThread().getContextClassLoader(), Boolean.TRUE) == null) {
        loadDefaultConverters();
        for (Map.Entry<Class<?>, Class<? extends StringCodec<?>>> entry : codecs.entrySet()) {
            try {
                final StringCodec<?> codecInstance = entry.getValue().newInstance();
                ConvertUtils.register(new Converter() {

                    @Override
                    public Object convert(Class type, Object value) {
                        return value == null ? null : codecInstance.fromString(value.toString());
                    }
                }, entry.getKey());
            } catch (Exception ex) {
                throw new RuntimeException(ex);
            }
        }
    }
}

Example 7 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class StreamingContainer method setup.

@SuppressWarnings("unchecked")
public void setup(StreamingContainerContext ctx) {
    containerContext = ctx;
    /* add a request factory local to this container */
    this.requestFactory = new RequestFactory();
    ctx.attributes.put(ContainerContext.REQUEST_FACTORY, requestFactory);
    heartbeatIntervalMillis = ctx.getValue(Context.DAGContext.HEARTBEAT_INTERVAL_MILLIS);
    firstWindowMillis = ctx.startWindowMillis;
    windowWidthMillis = ctx.getValue(Context.DAGContext.STREAMING_WINDOW_SIZE_MILLIS);
    checkpointWindowCount = ctx.getValue(Context.DAGContext.CHECKPOINT_WINDOW_COUNT);
    fastPublisherSubscriber = ctx.getValue(LogicalPlan.FAST_PUBLISHER_SUBSCRIBER);
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = ctx.getValue(Context.DAGContext.STRING_CODECS);
    StringCodecs.loadConverters(codecs);
    try {
        if (ctx.deployBufferServer) {
            eventloop.start();
            int bufferServerRAM = ctx.getValue(ContainerContext.BUFFER_SERVER_MB);
            logger.debug("buffer server memory {}", bufferServerRAM);
            int blockCount;
            int blocksize;
            if (bufferServerRAM < ContainerContext.BUFFER_SERVER_MB.defaultValue) {
                blockCount = 8;
                blocksize = bufferServerRAM / blockCount;
                if (blocksize < 1) {
                    blocksize = 1;
                }
            } else {
                blocksize = 64;
                blockCount = bufferServerRAM / blocksize;
            }
            // start buffer server, if it was not set externally
            bufferServer = new Server(eventloop, 0, blocksize * 1024 * 1024, blockCount);
            bufferServer.setAuthToken(ctx.getValue(StreamingContainerContext.BUFFER_SERVER_TOKEN));
            if (ctx.getValue(Context.DAGContext.BUFFER_SPOOLING)) {
                bufferServer.setSpoolStorage(new DiskStorage());
            }
            bufferServerAddress = NetUtils.getConnectAddress(bufferServer.run());
            logger.debug("Buffer server started: {}", bufferServerAddress);
        }
    } catch (IOException ex) {
        logger.warn("deploy request failed due to {}", ex);
        throw new IllegalStateException("Failed to deploy buffer server", ex);
    }
    for (Class<?> clazz : ContainerEvent.CONTAINER_EVENTS_LISTENERS) {
        try {
            Object newInstance = clazz.newInstance();
            singletons.put(clazz.getName(), newInstance);
            if (newInstance instanceof Component) {
                components.add((Component<ContainerContext>) newInstance);
            }
            eventBus.subscribe(newInstance);
        } catch (InstantiationException ex) {
            logger.warn("Container Event Listener Instantiation", ex);
        } catch (IllegalAccessException ex) {
            logger.warn("Container Event Listener Instantiation", ex);
        }
    }
    operateListeners(ctx, true);
}

Example 8 with StringCodec

use of com.datatorrent.api.StringCodec in project apex-core by apache.

the class StreamingAppMasterService method serviceInit.

@Override
protected void serviceInit(Configuration conf) throws Exception {
    LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());
    FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
    try {
        this.dag = LogicalPlan.read(fis);
    } finally {
        fis.close();
    }
    // "debug" simply dumps all data using LOG.info
    if (dag.isDebug()) {
        dumpOutDebugInfo();
    }
    dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
    FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
    this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
    dag = this.dnmgr.getLogicalPlan();
    this.appContext = new ClusterAppContextImpl(dag.getAttributes());
    Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
    StringCodecs.loadConverters(codecs);
    LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());
    // Setup security configuration such as that for web security
    SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));
    if (UserGroupInformation.isSecurityEnabled()) {
        // TODO :- Need to perform token renewal
        delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
    }
    this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
    addService(nmClient);
    this.amRmClient = AMRMClient.createAMRMClient();
    addService(amRmClient);
    // start RPC server
    int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
    this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
    addService(heartbeatListener);
    AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
    if (appDataPushTransport != null) {
        this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
        addService(this.appDataPushAgent);
    }
    initApexPluginDispatcher();
    // Initialize all services added above
    super.serviceInit(conf);
}