Examples with MantisSSEJob io.mantisrx.client.MantisSSEJob used on opensource projects

Example 1 with MantisSSEJob

use of io.mantisrx.client.MantisSSEJob in project mantis by Netflix.

the class ConnectToNamedJob method main.

public static void main(String[] args) {
    // SinkParameters params = new SinkParameters.Builder().withParameter("filter", "windows8").build();
    final AtomicLong eventCounter = new AtomicLong(0L);
    System.setProperty("log4j.logger.io", "DEBUG");
    try {
        Args.parse(ConnectToNamedJob.class, args);
    } catch (IllegalArgumentException e) {
        Args.usage(SubmitEphemeralJob.class);
        System.exit(1);
    }
    Properties properties = new Properties();
    System.out.println("propfile=" + propFile);
    try (InputStream inputStream = new FileInputStream(propFile)) {
        properties.load(inputStream);
    } catch (IOException e) {
        e.printStackTrace();
    }
    final CountDownLatch latch = new CountDownLatch(1);
    MantisSSEJob job = null;
    try {
        job = new MantisSSEJob.Builder(properties).name(jobName).onConnectionReset(new Action1<Throwable>() {

            @Override
            public void call(Throwable throwable) {
                System.err.println("Reconnecting due to error: " + throwable.getMessage());
            }
        }).sinkConnectionsStatusObserver(new Observer<SinkConnectionsStatus>() {

            @Override
            public void onCompleted() {
                System.out.println("ConnectionStatusObserver completed");
            }

            @Override
            public void onError(Throwable e) {
                System.err.println("ConnectionStatusObserver error: " + e.getMessage());
            }

            @Override
            public void onNext(SinkConnectionsStatus status) {
                System.out.println("ConnectionStatusObserver: receiving from " + status.getRecevingDataFrom() + ", connected to " + status.getNumConnected() + " of " + status.getTotal());
            }
        }).sinkDataRecvTimeoutSecs(11).buildJobConnector();
    } catch (Exception e) {
        e.printStackTrace();
    }
    // try{Thread.sleep(3000);}catch(InterruptedException ie){}
    System.out.println("Subscribing now");
    Subscription subscription = job.connectAndGet().doOnNext(new Action1<Observable<MantisServerSentEvent>>() {

        @Override
        public void call(Observable<MantisServerSentEvent> o) {
            o.doOnNext(new Action1<MantisServerSentEvent>() {

                @Override
                public void call(MantisServerSentEvent data) {
                    logger.info("Got event:  + " + data);
                    latch.countDown();
                // if(eventCounter.incrementAndGet()>4)
                // throw new RuntimeException("Test exception");
                }
            }).subscribe();
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            logger.error(throwable.getMessage());
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            System.out.println("Completed");
            System.exit(0);
        }
    }).subscribe();
    // .subscribe();
    try {
        boolean await = latch.await(30, TimeUnit.SECONDS);
        if (await)
            System.out.println("PASSED");
        else
            System.err.println("FAILED!");
        Thread.sleep(5000000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    subscription.unsubscribe();
    System.out.println("Unsubscribed");
    try {
        Thread.sleep(80000);
    } catch (InterruptedException ie) {
    }
    System.exit(0);
}

Example 2 with MantisSSEJob

use of io.mantisrx.client.MantisSSEJob in project mantis by Netflix.

the class SubmitWithRuntimeLimit method main.

public static void main(String[] args) {
    try {
        Args.parse(SubmitWithRuntimeLimit.class, args);
    } catch (IllegalArgumentException e) {
        Args.usage(SubmitEphemeralJob.class);
        System.exit(1);
    }
    Properties properties = new Properties();
    try (InputStream inputStream = new FileInputStream(propFile)) {
        properties.load(inputStream);
    } catch (IOException e) {
        e.printStackTrace();
    }
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicBoolean completed = new AtomicBoolean(false);
    final long runtimeLimitSecs = 30;
    MantisSSEJob job = new MantisSSEJob.Builder(properties).name(jobName).jobSla(new JobSla(runtimeLimitSecs, 0L, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Perpetual, "")).onConnectionReset(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            System.err.println("Reconnecting due to error: " + throwable.getMessage());
        }
    }).buildJobSubmitter();
    final Observable<Observable<MantisServerSentEvent>> observable = job.submitAndGet();
    final Subscription subscription = observable.flatMap(new Func1<Observable<MantisServerSentEvent>, Observable<?>>() {

        @Override
        public Observable<?> call(Observable<MantisServerSentEvent> eventObservable) {
            return eventObservable.doOnNext(new Action1<MantisServerSentEvent>() {

                @Override
                public void call(MantisServerSentEvent event) {
                    if (completed.get())
                        System.out.println("FAILURE");
                    System.out.println("Got: " + event.getEventAsString());
                }
            });
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            latch.countDown();
        }
    }).subscribe();
    try {
        // add a buffer for job launch time
        Thread.sleep((runtimeLimitSecs + 10) * 1000);
        // set expectation of complete
        completed.set(true);
        // give some time to see if we still get data, which would be a failure
        Thread.sleep(20000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    subscription.unsubscribe();
    System.exit(0);
}

Example 3 with MantisSSEJob

use of io.mantisrx.client.MantisSSEJob in project mantis by Netflix.

the class MantisSSEJobTest method submitAndConnectTest.

// @Test
public void submitAndConnectTest() {
    CountDownLatch latch = new CountDownLatch(2);
    CountDownLatch connectionStatusReceived = new CountDownLatch(1);
    MantisSSEJob job = new MantisSSEJob.Builder(zkProps).name("mantis-examples-sine-function").sinkConnectionsStatusObserver(new Observer<SinkConnectionsStatus>() {

        @Override
        public void onCompleted() {
        // TODO Auto-generated method stub
        }

        @Override
        public void onError(Throwable e) {
        // TODO Auto-generated method stub
        }

        @Override
        public void onNext(SinkConnectionsStatus t) {
            connectionStatusReceived.countDown();
        }
    }).onConnectionReset(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            System.err.println("Reconnecting due to error: " + throwable.getMessage());
        }
    }).onCloseKillJob().buildJobSubmitter();
    job.submitAndGet().flatMap((t) -> {
        return t.map((mmsse) -> {
            return mmsse.getEventAsString();
        });
    }).take(2).doOnNext((d) -> {
        latch.countDown();
    }).toBlocking().subscribe((data) -> {
        System.out.println("Got data -> " + data);
    });
    ;
    try {
        latch.await(20, TimeUnit.SECONDS);
        connectionStatusReceived.await(10, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        fail();
    }
}

Example 4 with MantisSSEJob

use of io.mantisrx.client.MantisSSEJob in project mantis by Netflix.

the class JobSource method call.

@Override
public Observable<Observable<MantisServerSentEvent>> call(Context context, Index index) {
    if (targets.isEmpty()) {
        targets = parseInputParameters(context);
    }
    Observable<Observable<MantisServerSentEvent>> sourceObs = null;
    int workerNo = context.getWorkerInfo().getWorkerNumber();
    targets = enforceClientIdConsistency(targets, context.getJobId());
    for (TargetInfo targetInfo : targets) {
        MantisSSEJob job;
        String sourceJobName = targetInfo.sourceJobName;
        String criterion = targetInfo.criterion;
        int samplePerSec = targetInfo.samplePerSec;
        boolean enableMetaMessages = targetInfo.enableMetaMessages;
        LOGGER.info("Processing job " + sourceJobName);
        boolean singleton = false;
        SinkConnectionStatusObserver obs = DefaultSinkConnectionStatusObserver.getInstance(singleton);
        MultiSinkConnectionStatusObserver.INSTANCE.addSinkConnectionObserver(sourceJobName, obs);
        String clientId = targetInfo.clientId;
        if (targetInfo.isBroadcastMode) {
            clientId = clientId + "_" + workerNo;
        }
        boolean enableCompressedBinary = targetInfo.enableCompressedBinary;
        job = getSourceJob(sourceJobName, criterion, clientId, samplePerSec, enableMetaMessages, enableCompressedBinary, obs, Optional.<SinkParameters>empty());
        if (sourceObs == null) {
            sourceObs = job.connectAndGet();
        } else {
            if (job != null) {
                Observable<Observable<MantisServerSentEvent>> clientObs = job.connectAndGet();
                if (clientObs != null) {
                    sourceObs = sourceObs.mergeWith(clientObs);
                } else {
                    LOGGER.error("Could not connect to job " + sourceJobName);
                }
            } else {
                LOGGER.error("Could not connect to job " + sourceJobName);
            }
        }
    }
    return sourceObs;
}

Example 5 with MantisSSEJob

use of io.mantisrx.client.MantisSSEJob in project mantis by Netflix.

the class SubmitEphemeralJob method main.

public static void main(String[] args) {
    try {
        Args.parse(SubmitEphemeralJob.class, args);
    } catch (IllegalArgumentException e) {
        Args.usage(SubmitEphemeralJob.class);
        System.exit(1);
    }
    Properties properties = new Properties();
    try (InputStream inputStream = new FileInputStream(propFile)) {
        properties.load(inputStream);
    } catch (IOException e) {
        e.printStackTrace();
    }
    final CountDownLatch latch = new CountDownLatch(1);
    // AutoCloseable job will terminate job if onCloseKillJob() called when building it
    Subscription subscription1 = null;
    // Subscription s2=null;
    MantisSSEJob job1 = new MantisSSEJob.Builder(properties).name(jobName).onCloseKillJob().onConnectionReset(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            System.err.println("Reconnecting due to error: " + throwable.getMessage());
        }
    }).buildJobSubmitter();
    // .buildJobSubmitter();
    try {
        Observable<Observable<MantisServerSentEvent>> observable = job1.submitAndGet();
        subscription1 = observable.doOnNext(new Action1<Observable<MantisServerSentEvent>>() {

            @Override
            public void call(Observable<MantisServerSentEvent> o) {
                o.doOnNext(new Action1<MantisServerSentEvent>() {

                    @Override
                    public void call(MantisServerSentEvent data) {
                        logger.info("Got event: " + data);
                        latch.countDown();
                    }
                }).subscribe();
            }
        }).doOnCompleted(new Action0() {

            @Override
            public void call() {
                System.out.println("Observable completed!!!");
            }
        }).subscribe();
        if (latch.await(50, TimeUnit.SECONDS))
            System.out.println("SUCCESS");
        else
            System.out.println("FAILURE");
        // .subscribe();
        if (latch.await(50, TimeUnit.SECONDS))
            System.out.println("SUCCESS");
        else
            System.out.println("FAILURE");
        Thread.sleep(30000000);
        // unsubscribe to close connection to sink
        subscription1.unsubscribe();
        // s2.unsubscribe();
        job1.close();
    // job2.close();
    } catch (Exception e) {
        e.printStackTrace();
    }
    System.exit(0);
}