Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 26 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class CheckpointSerializerMain method updateCheckpoint.

private static Checkpoint updateCheckpoint(Checkpoint cpOld) throws JsonParseException, JsonMappingException, IOException {
    Checkpoint cpNew = null != cpOld ? new Checkpoint(cpOld.toString()) : new Checkpoint();
    if (null != _scn) {
        if (-1L != _scn) {
            cpNew.setWindowScn(_scn);
            cpNew.setWindowOffset(0);
        } else {
            cpNew.setFlexible();
        }
    }
    if (null != _startScn) {
        cpNew.setBootstrapStartScn(_startScn);
    }
    if (null != _targetScn) {
        cpNew.setBootstrapTargetScn(_targetScn);
    }
    if (null != _cpType) {
        cpNew.setConsumptionMode(_cpType);
        switch(_cpType) {
            case ONLINE_CONSUMPTION:
                cpNew.setWindowOffset(0);
                break;
            /*
         * TODO Disabling as the bootstrap checkpoint creation leaves out important
         * information (e.g. catchup/snashot source index) out of the checkpoint
         * and thus is incorrect. We have to figure out what types of bootstrap
         * checkpoints it makes sense to create.
        case BOOTSTRAP_CATCHUP:
        {
          if (null != _bootstrapSource) cpNew.setCatchupSource(_bootstrapSource);
          cpNew.setCatchupOffset(-1);
          break;
        }*/
            case BOOTSTRAP_SNAPSHOT:
                {
                    BootstrapCheckpointHandler handler = new BootstrapCheckpointHandler(_sources);
                    cpNew = handler.createInitialBootstrapCheckpoint(cpNew, _sinceScn);
                    // if (null != _bootstrapSource) cpNew.setSnapshotSource(_bootstrapSource);
                    cpNew.setSnapshotOffset(-1);
                    break;
                }
            default:
                throw new DatabusRuntimeException("unsupported checkpoint type: " + _cpType);
        }
    }
    return cpNew;
}

Example 27 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class CheckpointSerializerMain method parseArgs.

private static void parseArgs(String[] args) throws Exception {
    CommandLineParser cliParser = new GnuParser();
    Options options = createOptions();
    CommandLine cmd = null;
    try {
        cmd = cliParser.parse(options, args);
    } catch (ParseException pe) {
        throw new RuntimeException("failed to parse command-line options.", pe);
    }
    if (cmd.hasOption(HELP_OPT_CHAR) || 0 == cmd.getOptions().length) {
        printCliHelp(options);
        System.exit(0);
    }
    try {
        _action = Action.valueOf(cmd.getOptionValue(ACTION_OPT_CHAR).toUpperCase());
    } catch (Exception e) {
        throw new RuntimeException("invalid action: " + cmd.getOptionValue(ACTION_OPT_CHAR), e);
    }
    if (!cmd.hasOption(SOURCES_OPT_CHAR)) {
        throw new RuntimeException("expected sources list; see --help for more info");
    }
    String sourcesListStr = cmd.getOptionValue(SOURCES_OPT_CHAR);
    _sources = sourcesListStr.split(",");
    if (null == _sources || 0 == _sources.length) {
        throw new RuntimeException("empty sources list");
    }
    for (int i = 0; i < _sources.length; ++i) _sources[i] = _sources[i].trim();
    if (Action.PRINT != _action && !cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR) && !cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR)) {
        throw new RuntimeException("expected client or CP3 configuration; see --help for more info");
    }
    String defaultPropPrefix = null;
    if (cmd.hasOption(CLIENT_PROPS_FILE_OPT_CHAR)) {
        try {
            _clientProps = loadProperties(cmd.getOptionValue(CLIENT_PROPS_FILE_OPT_CHAR));
            defaultPropPrefix = "databus2.client";
        } catch (Exception e) {
            throw new RuntimeException("unable to load client properties", e);
        }
    } else if (cmd.hasOption(CP3_PROPS_FILE_OPT_CHAR)) {
        try {
            _cp3Props = loadProperties(cmd.getOptionValue(CP3_PROPS_FILE_OPT_CHAR));
            defaultPropPrefix = "databus2.client.checkpointPersistence";
        } catch (Exception e) {
            throw new RuntimeException("unable to load CP3 properties", e);
        }
    }
    _propPrefix = cmd.hasOption(PROPS_PREFIX_OPT_CHAR) ? cmd.getOptionValue(PROPS_PREFIX_OPT_CHAR) : defaultPropPrefix;
    if (null != _propPrefix && !_propPrefix.endsWith(".")) {
        _propPrefix = _propPrefix + ".";
    }
    if (!cmd.hasOption(ACTION_OPT_CHAR)) {
        throw new RuntimeException("action expected; see --help for more info");
    }
    _scn = parseLongOption(cmd, SCN_OPT_CHAR, "sequence number");
    _sinceScn = parseLongOption(cmd, SINCE_SCN_OPT_CHAR, "last sequence number");
    _startScn = parseLongOption(cmd, START_SCN_OPT_CHAR, "start sequence number");
    _targetScn = parseLongOption(cmd, TARGET_SCN_OPT_CHAR, "target sequence number");
    if (cmd.hasOption(TYPE_OPT_CHAR)) {
        try {
            _cpType = DbusClientMode.valueOf(cmd.getOptionValue(TYPE_OPT_CHAR).toUpperCase());
        } catch (Exception e) {
            throw new RuntimeException("invalid checkpoint type:" + cmd.getOptionValue(TYPE_OPT_CHAR), e);
        }
    }
    if (cmd.hasOption(BOOTSTRAP_SOURCE_OPT_CHAR)) {
        _bootstrapSource = cmd.getOptionValue(BOOTSTRAP_SOURCE_OPT_CHAR);
    }
}

Example 28 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class CheckpointSerializerMain method main.

public static void main(String[] args) throws Exception {
    parseArgs(args);
    PatternLayout defaultLayout = new PatternLayout("%d{ISO8601} +%r [%t] (%p) {%c{1}} %m%n");
    ConsoleAppender defaultAppender = new ConsoleAppender(defaultLayout);
    Logger.getRootLogger().removeAllAppenders();
    Logger.getRootLogger().addAppender(defaultAppender);
    Logger.getRootLogger().setLevel(Level.INFO);
    Logger.getRootLogger().info("NOTE. This tool works only with V2/V1 checkpoints");
    CheckpointPersistenceProvider cp3 = null;
    if (null != _cp3Props) {
        CheckpointPersistenceStaticConfigBuilder cp3ConfBuilder = new CheckpointPersistenceStaticConfigBuilder();
        ConfigLoader<CheckpointPersistenceStaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.CheckpointPersistenceStaticConfig>(_propPrefix, cp3ConfBuilder);
        configLoader.loadConfig(_cp3Props);
        CheckpointPersistenceStaticConfig cp3Conf = cp3ConfBuilder.build();
        if (cp3Conf.getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + cp3Conf.getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(cp3Conf.getFileSystem(), 2);
    } else if (null != _clientProps) {
        DatabusHttpClientImpl.Config clientConfBuilder = new DatabusHttpClientImpl.Config();
        ConfigLoader<DatabusHttpClientImpl.StaticConfig> configLoader = new ConfigLoader<DatabusHttpClientImpl.StaticConfig>(_propPrefix, clientConfBuilder);
        configLoader.loadConfig(_clientProps);
        DatabusHttpClientImpl.StaticConfig clientConf = clientConfBuilder.build();
        if (clientConf.getCheckpointPersistence().getType() != CheckpointPersistenceStaticConfig.ProviderType.FILE_SYSTEM) {
            throw new RuntimeException("don't know what to do with cp3 type:" + clientConf.getCheckpointPersistence().getType());
        }
        cp3 = new FileSystemCheckpointPersistenceProvider(clientConf.getCheckpointPersistence().getFileSystem(), 2);
    }
    List<String> sourceList = Arrays.asList(_sources);
    Checkpoint cpOld = null != cp3 ? cp3.loadCheckpoint(sourceList) : new Checkpoint();
    Checkpoint cpNew;
    if (Action.PRINT == _action) {
        cpNew = updateCheckpoint(cpOld);
    } else if (Action.CHANGE == _action) {
        cpNew = updateCheckpoint(cpOld);
        cp3.storeCheckpoint(sourceList, cpNew);
        // reread as a sanity check
        cpNew = cp3.loadCheckpoint(sourceList);
    } else if (Action.DELETE == _action) {
        cp3.removeCheckpoint(sourceList);
        cpNew = cp3.loadCheckpoint(sourceList);
    } else {
        throw new RuntimeException("don't know what to do with action: " + _action);
    }
    if (null != cpOld)
        System.out.println("old: " + cpOld.toString());
    else
        System.out.println("old: null");
    if (null != cpNew)
        System.out.println("new: " + cpNew.toString());
    else
        System.out.println("new: null");
}

Example 29 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockBootstrapConnection method testTransition_ResumeCkptMissingStartScn.

@Test
public /**
 * Test Bootstrap transition: Connection Factory returned null with resumeCkpt startScn not set
 */
void testTransition_ResumeCkptMissingStartScn() throws Exception {
    BootstrapPullThread bsPuller = createBootstrapPullThread(true, false, false);
    Checkpoint cp = _ckptHandlerTwoSources.createInitialBootstrapCheckpoint(null, 100L);
    bsPuller.getComponentStatus().start();
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.BOOTSTRAP, "SUSPEND_ON_ERROR", cp);
}

Example 30 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class MockBootstrapConnection method testTransition_StartScnResponseSuccessHappyPath.

@Test
public /**
 * Test bootstrap transition: StartSCN_Response_Success : Happy path
 */
void testTransition_StartScnResponseSuccessHappyPath() throws Exception {
    BootstrapPullThread bsPuller = createBootstrapPullThread(false, false, false);
    Checkpoint cp = _ckptHandlerTwoSources.createInitialBootstrapCheckpoint(null, 1000L);
    cp.setBootstrapServerInfo(_serverInfoName);
    bsPuller.getComponentStatus().start();
    ConnectionState connState = bsPuller.getConnectionState();
    connState.switchToBootstrap(cp);
    testTransitionCase(bsPuller, StateId.BOOTSTRAP, StateId.REQUEST_START_SCN, cp);
    bsPuller.getMessageQueue().clear();
    testTransitionCase(bsPuller, StateId.REQUEST_START_SCN, StateId.START_SCN_RESPONSE_SUCCESS, null);
    bsPuller.getMessageQueue().clear();
    Map<Long, List<RegisterResponseEntry>> entries = new HashMap<Long, List<RegisterResponseEntry>>();
    entries.put(1L, new ArrayList<RegisterResponseEntry>());
    connState.setSourcesSchemas(entries);
    testTransitionCase(bsPuller, StateId.START_SCN_RESPONSE_SUCCESS, StateId.REQUEST_STREAM, null);
}