Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 21 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestResponseProcessors method testTargetSCNExceptionAtStart.

@Test
public void testTargetSCNExceptionAtStart() throws Exception {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
    BootstrapTargetScnHttpResponseProcessor processor = new BootstrapTargetScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler, null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();
    processor.channelException(new Exception("dummy exception"));
    processor.startResponse(httpResponse);
    processor.addChunk(httpChunk);
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();
    Assert.assertEquals("Error Handled", true, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_FINISHED, processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage", "TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage) (queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.TARGETSCN_RESPONSE_ERROR, gotMsg._state);
    Assert.assertEquals("No response", true, (null == stateMsg._cp));
}

Example 22 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestResponseProcessors method testTargetSCNExceptionAfterStartCase2.

@Test
public void testTargetSCNExceptionAfterStartCase2() throws Exception {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);
    BootstrapTargetScnHttpResponseProcessor processor = new BootstrapTargetScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler, null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();
    processor.startResponse(httpResponse);
    processor.addChunk(httpChunk);
    processor.channelException(new Exception("dummy exception"));
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();
    Assert.assertEquals("Error Handled", true, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_FINISHED, processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage", "TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage) (queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.TARGETSCN_RESPONSE_ERROR, gotMsg._state);
    Assert.assertEquals("No response", true, (null == stateMsg._cp));
}

Example 23 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestResponseProcessors method testStartSCNExceptionAtStart.

@Test
public void testStartSCNExceptionAtStart() throws Exception {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    HttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor processor = new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler, null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();
    processor.channelException(new Exception("dummy exception"));
    processor.startResponse(httpResponse);
    processor.addChunk(httpChunk);
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();
    Assert.assertEquals("Error Handled", true, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_FINISHED, processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage", "TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage) (queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.STARTSCN_RESPONSE_ERROR, gotMsg._state);
    Assert.assertEquals("No response", true, (null == stateMsg._cp));
}

Example 24 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestResponseProcessors method testStartSCNHappyPathChunked.

@Test
public void testStartSCNHappyPathChunked() throws Exception {
    TestAbstractQueue queue = new TestAbstractQueue();
    TestConnectionStateMessage stateMsg = new TestConnectionStateMessage();
    DefaultHttpResponse httpResponse = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
    TestRemoteExceptionHandler remoteExHandler = new TestRemoteExceptionHandler();
    Checkpoint cp = new Checkpoint();
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    BootstrapStartScnHttpResponseProcessor processor = new BootstrapStartScnHttpResponseProcessor(null, queue, stateMsg, cp, remoteExHandler, null);
    ChannelBuffer buf = getScnResponse();
    HttpChunk httpChunk = new DefaultHttpChunk(buf);
    HttpChunkTrailer httpChunkTrailer = new DefaultHttpChunkTrailer();
    processor.startResponse(httpResponse);
    processor.addChunk(httpChunk);
    processor.addTrailer(httpChunkTrailer);
    processor.finishResponse();
    Assert.assertEquals("Error Handled", false, processor._errorHandled);
    Assert.assertEquals("Processor Response State", AbstractHttpResponseProcessorDecorator.ResponseStatus.CHUNKS_FINISHED, processor._responseStatus);
    Assert.assertEquals("Actor Queue Size", 1, queue.getMessages().size());
    Assert.assertEquals("Expected ConnectionStateMessage", "TestConnectionStateMessage", queue.getMessages().get(0).getClass().getSimpleName());
    TestConnectionStateMessage gotMsg = (TestConnectionStateMessage) (queue.getMessages().get(0));
    Assert.assertEquals("Expected ConnectionStateMessage State", TestConnectionStateMessage.State.STARTSCN_RESPONSE_SUCCESS, gotMsg._state);
    Assert.assertEquals("StartSCN Response Id Check", new Long(5678912), cp.getBootstrapStartScn());
}

Example 25 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DatabusClusterUtil method main.

/**
 * @param args
 *            DbusClusterUtil -z <zookeper-server> -c <cluster-name> [-p
 *            <partitionNumber] partitions readSCN writeSCN SCN remove
 *            clients
 */
public static void main(String[] args) {
    try {
        GnuParser cmdLineParser = new GnuParser();
        Options options = new Options();
        options.addOption("z", true, "zk-server").addOption("c", true, "cluster-name ").addOption("p", true, "partition").addOption("l", false, "legacy").addOption("h", false, "help");
        CommandLine cmdLineArgs = cmdLineParser.parse(options, args, false);
        if (cmdLineArgs.hasOption('h')) {
            usage();
            System.exit(0);
        }
        if (!cmdLineArgs.hasOption('c')) {
            usage();
            System.exit(1);
        }
        String clusterName = cmdLineArgs.getOptionValue('c');
        String zkServer = cmdLineArgs.getOptionValue('z');
        boolean isLegacyChkptLocation = cmdLineArgs.hasOption('l');
        if (zkServer == null || zkServer.isEmpty()) {
            zkServer = "localhost:2181";
        }
        String partitionStr = cmdLineArgs.getOptionValue('p');
        String partition = partitionStr;
        if ((partition != null) && partition.equals("all")) {
            partition = "";
        }
        String[] fns = cmdLineArgs.getArgs();
        if (fns.length < 1) {
            usage();
            System.exit(1);
        }
        DatabusClusterUtilHelper clusterState = new DatabusClusterUtilHelper(zkServer, clusterName);
        String function = fns[0];
        String arg1 = (fns.length > 1) ? fns[1] : null;
        String arg2 = (fns.length > 2) ? fns[2] : null;
        boolean clusterExists = clusterState.existsCluster();
        if (function.equals("create")) {
            if (!clusterExists) {
                if (arg1 == null) {
                    throw new DatabusClusterUtilException("create: please provide the number of partitions");
                }
                int part = Integer.parseInt(arg1);
                clusterState.createCluster(part);
                return;
            } else {
                throw new DatabusClusterUtilException("Cluster " + clusterName + " already exists");
            }
        }
        if (!clusterExists) {
            throw new DatabusClusterUtilException("Cluster doesn't exist! ");
        }
        if (function.equals("delete")) {
            clusterState.removeCluster();
        } else if (function.equals("partitions")) {
            int numParts = clusterState.getNumPartitions();
            System.out.println(numParts);
        } else {
            // all these functions require the notion of partition;
            Set<Integer> partitions = getPartitions(partition, clusterState.getNumPartitions());
            if (function.equals("sources")) {
                DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, null, partitions, isLegacyChkptLocation);
                Set<String> sources = ckptMgr.getSourcesFromCheckpoint();
                if (sources != null) {
                    for (String s : sources) {
                        System.out.println(s);
                    }
                } else {
                    throw new DatabusClusterUtilException("sources: Sources not found for cluster " + clusterName);
                }
            } else if (function.equals("clients")) {
                clusterState.getClusterInfo();
                for (Integer p : partitions) {
                    String client = clusterState.getInstanceForPartition(p);
                    System.out.println(p + "\t" + client);
                }
            } else if (function.equals("readSCN")) {
                List<String> sources = getSources(arg1);
                if ((sources != null) && !sources.isEmpty()) {
                    DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                    Map<Integer, Checkpoint> ckpts = ckptMgr.readCheckpoint();
                    char delim = '\t';
                    for (Map.Entry<Integer, Checkpoint> mkPair : ckpts.entrySet()) {
                        StringBuilder output = new StringBuilder(64);
                        output.append(mkPair.getKey());
                        output.append(delim);
                        Checkpoint cp = mkPair.getValue();
                        if (cp == null) {
                            output.append(-1);
                            output.append(delim);
                            output.append(-1);
                        } else {
                            if (cp.getConsumptionMode() == DbusClientMode.ONLINE_CONSUMPTION) {
                                output.append(cp.getWindowScn());
                                output.append(delim);
                                output.append(cp.getWindowOffset());
                            } else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP) {
                                output.append(cp.getWindowScn());
                                output.append(delim);
                                output.append(cp.getWindowOffset());
                            } else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
                                output.append(cp.getBootstrapSinceScn());
                                output.append(delim);
                                output.append(-1);
                            }
                        }
                        System.out.println(output.toString());
                    }
                } else {
                    throw new DatabusClusterUtilException("readSCN: please specify non-empty sources");
                }
            } else if (function.equals("checkpoint")) {
                List<String> sources = getSources(arg1);
                if ((sources != null) && !sources.isEmpty()) {
                    DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                    Map<Integer, Checkpoint> ckpts = ckptMgr.readCheckpoint();
                    char delim = '\t';
                    for (Map.Entry<Integer, Checkpoint> mkPair : ckpts.entrySet()) {
                        StringBuilder output = new StringBuilder(64);
                        output.append(mkPair.getKey());
                        output.append(delim);
                        Checkpoint cp = mkPair.getValue();
                        if (cp == null) {
                            output.append("null");
                        } else {
                            output.append(cp.toString());
                        }
                        System.out.println(output.toString());
                    }
                } else {
                    throw new DatabusClusterUtilException("readSCN: please specify non-empty sources");
                }
            } else if (function.equals("writeSCN")) {
                String scnStr = arg1;
                Long scn = Long.parseLong(scnStr);
                if (partitionStr != null) {
                    List<String> sources = getSources(arg2);
                    if ((sources != null) && !sources.isEmpty()) {
                        DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                        ckptMgr.writeCheckpoint(scn);
                    } else {
                        throw new DatabusClusterUtilException("writeSCN: please specify non-empty sources");
                    }
                } else {
                    throw new DatabusClusterUtilException("writeSCN: to write the SCN to all partitions please use '-p all'");
                }
            } else if (function.equals("removeSCN")) {
                if (partitionStr != null) {
                    List<String> sources = getSources(arg1);
                    if ((sources != null) && !sources.isEmpty()) {
                        DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                        ckptMgr.remove();
                    } else {
                        throw new DatabusClusterUtilException("remove: please specify non-empty sources");
                    }
                } else {
                    throw new DatabusClusterUtilException("remove: to remove SCN from all partitions please use '-p all'");
                }
            } else {
                usage();
                System.exit(1);
            }
        }
    } catch (ParseException e) {
        usage();
        System.exit(1);
    } catch (DatabusClusterUtilException e) {
        System.err.println("Error! " + e.toString());
        System.exit(1);
    }
}