Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 51 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DummyRemoteExceptionHandler method runServerStreamReqDisconnectIteration.

private void runServerStreamReqDisconnectIteration(final Logger log, TestingConnectionCallback callback, DummyRemoteExceptionHandler remoteExceptionHandler, final NettyHttpDatabusRelayConnection conn) throws JsonGenerationException, JsonMappingException, IOException, ScnNotFoundException, OffsetNotFoundException {
    //connect to server and send /sources
    TestResponseProcessors.TestConnectionStateMessage msg = new TestResponseProcessors.TestConnectionStateMessage();
    conn.requestSources(msg);
    waitForServerConnection(conn, log);
    //introspect connection to server
    Channel channel = conn._channel;
    SocketAddress clientAddr = channel.getLocalAddress();
    Channel serverChannel = _dummyServer.getChildChannel(clientAddr);
    ChannelPipeline serverPipeline = serverChannel.getPipeline();
    SimpleObjectCaptureHandler objCapture = (SimpleObjectCaptureHandler) serverPipeline.get("3");
    //verify server gets the /source request
    HttpResponse sourcesResp = runHappyPathSources(log, callback, remoteExceptionHandler, clientAddr, objCapture);
    //send /register
    runHappyPathRegister(log, callback, remoteExceptionHandler, conn, msg, clientAddr, objCapture, sourcesResp);
    //send partial /stream
    callback.clearLastMsg();
    objCapture.clear();
    serverChannel.close();
    Checkpoint cp = new Checkpoint();
    cp.setFlexible();
    CheckpointMult cpm = new CheckpointMult();
    cpm.addCheckpoint(PhysicalPartition.ANY_PHYSICAL_PARTITION, cp);
    conn.requestStream("1", null, 1000, cpm, null, msg);
    waitForCallback(callback, TestResponseProcessors.TestConnectionStateMessage.State.STREAM_REQUEST_ERROR, log);
    Assert.assertNull(remoteExceptionHandler.getLastException());
    Assert.assertEquals(1, callback.getAllMsgs().size());
    callback.clearLastMsg();
    objCapture.clear();
}

Example 52 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class CheckpointSerializerMain method main.

public static void main(String[] args) throws Exception {
    parseArgs(args);
    PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c{1}} %m%n");
    ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout);
    Logger.getRootLogger().removeAllAppenders();
    Logger.getRootLogger().addAppender(defaultAppender);
    Logger.getRootLogger().setLevel(Level.INFO);
    Logger.getRootLogger().info("NOTE. This tool works only with V2/V1 checkpoints");
    CheckpointPersistenceProvider cp3 = null;
    if (null != _cp3Props) {
        CheckpointPersistenceStaticConfigBuilder cp3ConfBuilder = new CheckpointPersistenceStaticConfigBuilder();
        ConfigLoader<CheckpointPersistenceStaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.CheckpointPersistenceStaticConfig>(_propPrefix, cp3ConfBuilder);
        configLoader.loadConfig(_cp3Props);
        CheckpointPersistenceStaticConfig cp3Conf = cp3ConfBuilder.build();
        if (cp3Conf.getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + cp3Conf.getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(cp3Conf.getFileSystem(), 2);
    } else if (null != _clientProps) {
        DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
        ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>(_propPrefix, clientConfBuilder);
        configLoader.loadConfig(_clientProps);
        DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
        if (clientConf.getCheckpointPersistence().getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + clientConf.getCheckpointPersistence().getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(clientConf.getCheckpointPersistence().getFileSystem(), 2);
    }
    List<String> sourceList = Arrays.asList(_sources);
    Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint();
    Checkpoint cpNew;
    if (Action.PRINT == _action) {
        cpNew = updateCheckpoint(cpOld);
    } else if (Action.CHANGE == _action) {
        cpNew = updateCheckpoint(cpOld);
        cp3.storeCheckpoint(sourceList, cpNew);
        //reread as a sanity check
        cpNew = cp3.loadCheckpoint(sourceList);
    } else if (Action.DELETE == _action) {
        cp3.removeCheckpoint(sourceList);
        cpNew = cp3.loadCheckpoint(sourceList);
    } else {
        throw new RuntimeException("don't know what to do with action: " + _action);
    }
    if (null != cpOld)
        System.out.println("old: " + cpOld.toString());
    else
        System.out.println("old: null");
    if (null != cpNew)
        System.out.println("new: " + cpNew.toString());
    else
        System.out.println("new: null");
}

Example 53 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class CheckpointSerializerMain method updateCheckpoint.

private static Checkpoint updateCheckpoint(Checkpoint cpOld) throws JsonParseException, JsonMappingException, IOException {
    Checkpoint cpNew = null != cpOld ? new Checkpoint(cpOld.toString()) : new Checkpoint();
    if (null != _scn) {
        if (-1L != _scn) {
            cpNew.setWindowScn(_scn);
            cpNew.setWindowOffset(0);
        } else {
            cpNew.setFlexible();
        }
    }
    if (null != _startScn) {
        cpNew.setBootstrapStartScn(_startScn);
    }
    if (null != _targetScn) {
        cpNew.setBootstrapTargetScn(_targetScn);
    }
    if (null != _cpType) {
        cpNew.setConsumptionMode(_cpType);
        switch(_cpType) {
            case ONLINE_CONSUMPTION:
                cpNew.setWindowOffset(0);
                break;
            /*
         * TODO Disabling as the bootstrap checkpoint creation leaves out important
         * information (e.g. catchup/snashot source index) out of the checkpoint
         * and thus is incorrect. We have to figure out what types of bootstrap
         * checkpoints it makes sense to create.
        case BOOTSTRAP_CATCHUP:
        {
          if (null != _bootstrapSource) cpNew.setCatchupSource(_bootstrapSource);
          cpNew.setCatchupOffset(-1);
          break;
        }*/
            case BOOTSTRAP_SNAPSHOT:
                {
                    BootstrapCheckpointHandler handler = new BootstrapCheckpointHandler(_sources);
                    cpNew = handler.createInitialBootstrapCheckpoint(cpNew, _sinceScn);
                    //if (null != _bootstrapSource) cpNew.setSnapshotSource(_bootstrapSource);
                    cpNew.setSnapshotOffset(-1);
                    break;
                }
            default:
                throw new DatabusRuntimeException("unsupported checkpoint type: " + _cpType);
        }
    }
    return cpNew;
}

Example 54 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class CheckpointSerializerMain method parseArgs.

private static void parseArgs(String[] args) throws Exception {
    CommandLineParser cliParser = new GnuParser();
    Options options = createOptions();
    CommandLine cmd = null;
    try {
        cmd = cliParser.parse(options, args);
    } catch (ParseException pe) {
        throw new RuntimeException("failed to parse command-line options.", pe);
    }
    if (cmd.hasOption(HELP_OPT_CHAR) || 0 == cmd.getOptions().length) {
        printCliHelp(options);
        System.exit(0);
    }
    try {
        _action = Action.valueOf(cmd.getOptionValue(ACTION_OPT_CHAR).toUpperCase());
    } catch (Exception e) {
        throw new RuntimeException("invalid action: " + cmd.getOptionValue(ACTION_OPT_CHAR), e);
    }
    if (!cmd.hasOption(SOURCES_OPT_CHAR)) {
        throw new RuntimeException("expected sources list; see --help for more info");
    }
    String sourcesListStr = cmd.getOptionValue(SOURCES_OPT_CHAR);
    _sources = sourcesListStr.split(",");
    if (null == _sources || 0 == _sources.length) {
        throw new RuntimeException("empty sources list");
    }
    for (int i = 0; i < _sources.length; ++i) _sources[i] = _sources[i].trim();
    if (Action.PRINT != _action && !cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR) && !cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR)) {
        throw new RuntimeException("expected client or CP3 configuration; see --help for more info");
    }
    String defaultPropPrefix = null;
    if (cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR)) {
        try {
            _clientProps = loadProperties(cmd.getOptionValue(CLIENT_PROPS_FILE_OPT_CHAR));
            defaultPropPrefix = "databus2.client";
        } catch (Exception e) {
            throw new RuntimeException("unable to load client properties", e);
        }
    } else if (cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR)) {
        try {
            _cp3Props = loadProperties(cmd.getOptionValue(CP3_PROPS_FILE_OPT_CHAR));
            defaultPropPrefix = "databus2.client.checkpointPersistence";
        } catch (Exception e) {
            throw new RuntimeException("unable to load CP3 properties", e);
        }
    }
    _propPrefix = cmd.hasOption(PROPS_PREFIX_OPT_CHAR) ? cmd.getOptionValue(PROPS_PREFIX_OPT_CHAR) : defaultPropPrefix;
    if (null != _propPrefix && !_propPrefix.endsWith(".")) {
        _propPrefix = _propPrefix + ".";
    }
    if (!cmd.hasOption(ACTION_OPT_CHAR)) {
        throw new RuntimeException("action expected; see --help for more info");
    }
    _scn = parseLongOption(cmd, SCN_OPT_CHAR, "sequence number");
    _sinceScn = parseLongOption(cmd, SINCE_SCN_OPT_CHAR, "last sequence number");
    _startScn = parseLongOption(cmd, START_SCN_OPT_CHAR, "start sequence number");
    _targetScn = parseLongOption(cmd, TARGET_SCN_OPT_CHAR, "target sequence number");
    if (cmd.hasOption(TYPE_OPT_CHAR)) {
        try {
            _cpType = DbusClientMode.valueOf(cmd.getOptionValue(TYPE_OPT_CHAR).toUpperCase());
        } catch (Exception e) {
            throw new RuntimeException("invalid checkpoint type:" + cmd.getOptionValue(TYPE_OPT_CHAR), e);
        }
    }
    if (cmd.hasOption(BOOTSTRAP_SOURCE_OPT_CHAR)) {
        _bootstrapSource = cmd.getOptionValue(BOOTSTRAP_SOURCE_OPT_CHAR);
    }
}

Example 55 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DatabusClusterUtil method main.

/**
     * @param args
     *            DbusClusterUtil -z <zookeper-server> -c <cluster-name> [-p
     *            <partitionNumber] partitions readSCN writeSCN SCN remove
     *            clients
     */
public static void main(String[] args) {
    try {
        GnuParser cmdLineParser = new GnuParser();
        Options options = new Options();
        options.addOption("z", true, "zk-server").addOption("c", true, "cluster-name ").addOption("p", true, "partition").addOption("l", false, "legacy").addOption("h", false, "help");
        CommandLine cmdLineArgs = cmdLineParser.parse(options, args, false);
        if (cmdLineArgs.hasOption('h')) {
            usage();
            System.exit(0);
        }
        if (!cmdLineArgs.hasOption('c')) {
            usage();
            System.exit(1);
        }
        String clusterName = cmdLineArgs.getOptionValue('c');
        String zkServer = cmdLineArgs.getOptionValue('z');
        boolean isLegacyChkptLocation = cmdLineArgs.hasOption('l');
        if (zkServer == null || zkServer.isEmpty()) {
            zkServer = "localhost:2181";
        }
        String partitionStr = cmdLineArgs.getOptionValue('p');
        String partition = partitionStr;
        if ((partition != null) && partition.equals("all")) {
            partition = "";
        }
        String[] fns = cmdLineArgs.getArgs();
        if (fns.length < 1) {
            usage();
            System.exit(1);
        }
        DatabusClusterUtilHelper clusterState = new DatabusClusterUtilHelper(zkServer, clusterName);
        String function = fns[0];
        String arg1 = (fns.length > 1) ? fns[1] : null;
        String arg2 = (fns.length > 2) ? fns[2] : null;
        boolean clusterExists = clusterState.existsCluster();
        if (function.equals("create")) {
            if (!clusterExists) {
                if (arg1 == null) {
                    throw new DatabusClusterUtilException("create: please provide the number of partitions");
                }
                int part = Integer.parseInt(arg1);
                clusterState.createCluster(part);
                return;
            } else {
                throw new DatabusClusterUtilException("Cluster " + clusterName + " already exists");
            }
        }
        if (!clusterExists) {
            throw new DatabusClusterUtilException("Cluster doesn't exist! ");
        }
        if (function.equals("delete")) {
            clusterState.removeCluster();
        } else if (function.equals("partitions")) {
            int numParts = clusterState.getNumPartitions();
            System.out.println(numParts);
        } else {
            // all these functions require the notion of partition;
            Set<Integer> partitions = getPartitions(partition, clusterState.getNumPartitions());
            if (function.equals("sources")) {
                DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, null, partitions, isLegacyChkptLocation);
                Set<String> sources = ckptMgr.getSourcesFromCheckpoint();
                if (sources != null) {
                    for (String s : sources) {
                        System.out.println(s);
                    }
                } else {
                    throw new DatabusClusterUtilException("sources: Sources not found for cluster " + clusterName);
                }
            } else if (function.equals("clients")) {
                clusterState.getClusterInfo();
                for (Integer p : partitions) {
                    String client = clusterState.getInstanceForPartition(p);
                    System.out.println(p + "\t" + client);
                }
            } else if (function.equals("readSCN")) {
                List<String> sources = getSources(arg1);
                if ((sources != null) && !sources.isEmpty()) {
                    DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                    Map<Integer, Checkpoint> ckpts = ckptMgr.readCheckpoint();
                    char delim = '\t';
                    for (Map.Entry<Integer, Checkpoint> mkPair : ckpts.entrySet()) {
                        StringBuilder output = new StringBuilder(64);
                        output.append(mkPair.getKey());
                        output.append(delim);
                        Checkpoint cp = mkPair.getValue();
                        if (cp == null) {
                            output.append(-1);
                            output.append(delim);
                            output.append(-1);
                        } else {
                            if (cp.getConsumptionMode() == DbusClientMode.ONLINE_CONSUMPTION) {
                                output.append(cp.getWindowScn());
                                output.append(delim);
                                output.append(cp.getWindowOffset());
                            } else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_CATCHUP) {
                                output.append(cp.getWindowScn());
                                output.append(delim);
                                output.append(cp.getWindowOffset());
                            } else if (cp.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
                                output.append(cp.getBootstrapSinceScn());
                                output.append(delim);
                                output.append(-1);
                            }
                        }
                        System.out.println(output.toString());
                    }
                } else {
                    throw new DatabusClusterUtilException("readSCN: please specify non-empty sources");
                }
            } else if (function.equals("checkpoint")) {
                List<String> sources = getSources(arg1);
                if ((sources != null) && !sources.isEmpty()) {
                    DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                    Map<Integer, Checkpoint> ckpts = ckptMgr.readCheckpoint();
                    char delim = '\t';
                    for (Map.Entry<Integer, Checkpoint> mkPair : ckpts.entrySet()) {
                        StringBuilder output = new StringBuilder(64);
                        output.append(mkPair.getKey());
                        output.append(delim);
                        Checkpoint cp = mkPair.getValue();
                        if (cp == null) {
                            output.append("null");
                        } else {
                            output.append(cp.toString());
                        }
                        System.out.println(output.toString());
                    }
                } else {
                    throw new DatabusClusterUtilException("readSCN: please specify non-empty sources");
                }
            } else if (function.equals("writeSCN")) {
                String scnStr = arg1;
                Long scn = Long.parseLong(scnStr);
                if (partitionStr != null) {
                    List<String> sources = getSources(arg2);
                    if ((sources != null) && !sources.isEmpty()) {
                        DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                        ckptMgr.writeCheckpoint(scn);
                    } else {
                        throw new DatabusClusterUtilException("writeSCN: please specify non-empty sources");
                    }
                } else {
                    throw new DatabusClusterUtilException("writeSCN: to write the SCN to all partitions please use '-p all'");
                }
            } else if (function.equals("removeSCN")) {
                if (partitionStr != null) {
                    List<String> sources = getSources(arg1);
                    if ((sources != null) && !sources.isEmpty()) {
                        DatabusClusterCkptManager ckptMgr = new DatabusClusterCkptManager(zkServer, clusterName, sources, partitions, isLegacyChkptLocation);
                        ckptMgr.remove();
                    } else {
                        throw new DatabusClusterUtilException("remove: please specify non-empty sources");
                    }
                } else {
                    throw new DatabusClusterUtilException("remove: to remove SCN from all partitions please use '-p all'");
                }
            } else {
                usage();
                System.exit(1);
            }
        }
    } catch (ParseException e) {
        usage();
        System.exit(1);
    } catch (DatabusClusterUtilException e) {
        System.err.println("Error! " + e.toString());
        System.exit(1);
    }
}