Examples with Checkpoint com.linkedin.databus.core.Checkpoint used on opensource projects

Example 76 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class ClusterCheckpointPersistenceProvider method loadCheckpointLegacy.

@Deprecated
public /**
     * read legacy checkpoint without migration
     * 
     * @param sources
     * @return checkpoint if found or null otherwise
     */
Checkpoint loadCheckpointLegacy(List<String> sources) {
    String key = makeKeyOld(sources);
    Checkpoint cp = getCheckpoint(key);
    return cp;
}

Example 77 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DbusClientClusterUtil method main.

/**
	 * @param args
	 * DbusClientClusterUtil -s <serverList> -n <namespace> -g <group> -d <dir>     members
	 * 													   	 		leader 
	 * 													  	 		keys 
	 * 													  	 		readSCN <key> 
	 * 													  	 		writeSCN <key> SCN [OFFSET]
	 * 																remove <key>
	 * 																readLastTS
	 * 																writeLastTS TIMESTAMP				
	 */
public static void main(String[] args) {
    try {
        GnuParser cmdLineParser = new GnuParser();
        Options options = new Options();
        options.addOption("n", true, "Zookeeper namespace  [/DatabusClient").addOption("g", true, "Groupname [default-group-name] ").addOption("d", true, "Shared directory name [shareddata] ").addOption("s", true, "Zookeeper server list [localhost:2181] ").addOption("h", false, "help");
        CommandLine cmdLineArgs = cmdLineParser.parse(options, args, false);
        if (cmdLineArgs.hasOption('h')) {
            usage();
            System.exit(0);
        }
        String namespace = cmdLineArgs.getOptionValue('n');
        if (namespace == null || namespace.isEmpty()) {
            namespace = "/DatabusClient";
        }
        String groupname = cmdLineArgs.getOptionValue('g');
        if (groupname == null || groupname.isEmpty()) {
            groupname = "default-group-name";
        }
        String sharedDir = cmdLineArgs.getOptionValue('d');
        if (sharedDir == null || sharedDir.isEmpty()) {
            sharedDir = "shareddata";
        }
        String serverList = cmdLineArgs.getOptionValue('s');
        if (serverList == null || serverList.isEmpty()) {
            serverList = "localhost:2181";
        }
        String[] fns = cmdLineArgs.getArgs();
        if (fns.length < 1) {
            usage();
            System.exit(1);
        }
        String function = fns[0];
        String arg1 = (fns.length > 1) ? fns[1] : null;
        String arg2 = (fns.length > 2) ? fns[2] : null;
        try {
            String memberName = "cmd-line-tool";
            DatabusClientNode clusterNode = new DatabusClientNode(new DatabusClientNode.StaticConfig(true, serverList, 2000, 5000, namespace, groupname, memberName, false, sharedDir));
            DatabusClientGroupMember member = clusterNode.getMember(namespace, groupname, memberName);
            if (member == null || !member.joinWithoutLeadershipDuties()) {
                System.err.println("Initialization failed for: " + member);
                System.exit(1);
            }
            if (function.equals("members")) {
                List<String> mlist = member.getMembers();
                for (String m : mlist) {
                    System.out.println(m);
                }
            } else if (function.equals("leader")) {
                String leader = member.getLeader();
                System.out.println(leader);
            } else if (function.equals("keys")) {
                List<String> keyList = member.getSharedKeys();
                if (keyList != null) {
                    for (String m : keyList) {
                        System.out.println(m);
                    }
                }
            } else if (function.equals("readSCN")) {
                List<String> keyList;
                if (arg1 == null) {
                    keyList = member.getSharedKeys();
                } else {
                    keyList = new ArrayList<String>();
                    keyList.add(arg1);
                }
                if (keyList != null) {
                    for (String k : keyList) {
                        if (!k.equals(DatabusClientDSCUpdater.DSCKEY)) {
                            Checkpoint cp = (Checkpoint) member.readSharedData(k);
                            if (cp != null) {
                                System.out.println(k + " " + cp.getWindowScn() + " " + cp.getWindowOffset());
                            } else {
                                System.err.println(k + " null null");
                            }
                        }
                    }
                }
            } else if (function.equals("writeSCN")) {
                if (arg1 != null && arg2 != null) {
                    Checkpoint cp = new Checkpoint();
                    cp.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);
                    cp.setWindowScn(Long.parseLong(arg2));
                    if (fns.length > 3) {
                        cp.setWindowOffset(Integer.parseInt(fns[3]));
                    } else {
                        cp.setWindowOffset(-1);
                    }
                    if (member.writeSharedData(arg1, cp)) {
                        System.out.println(arg1 + " " + cp.getWindowScn() + " " + cp.getWindowOffset());
                    } else {
                        System.err.println("Write failed! " + member + " couldn't write key=" + arg1);
                        System.exit(1);
                    }
                } else {
                    usage();
                    System.exit(1);
                }
            } else if (function.equals("readLastTs")) {
                Long timeInMs = (Long) member.readSharedData(DatabusClientDSCUpdater.DSCKEY);
                if (timeInMs != null) {
                    System.out.println(DatabusClientDSCUpdater.DSCKEY + " " + timeInMs.longValue());
                } else {
                    System.err.println(DatabusClientDSCUpdater.DSCKEY + " null");
                }
            } else if (function.equals("writeLastTs")) {
                if (arg1 != null) {
                    Long ts = Long.parseLong(arg1);
                    if (member.writeSharedData(DatabusClientDSCUpdater.DSCKEY, ts)) {
                        System.out.println(DatabusClientDSCUpdater.DSCKEY + " " + ts);
                    } else {
                        System.err.println("Write failed! " + member + " couldn't write key=" + DatabusClientDSCUpdater.DSCKEY);
                        System.exit(1);
                    }
                } else {
                    usage();
                    System.exit(1);
                }
            } else if (function.equals("remove")) {
                if (!member.removeSharedData(arg1)) {
                    System.err.println("Remove failed! " + arg1);
                    System.exit(1);
                }
            } else if (function.equals("create")) {
                if (!member.createPaths()) {
                    System.err.println("Create path failed!");
                    System.exit(1);
                }
            } else {
                usage();
                System.exit(1);
            }
        } catch (InvalidConfigException e) {
            e.printStackTrace();
            usage();
            System.exit(1);
        }
    } catch (ParseException e) {
        usage();
        System.exit(1);
    }
}

Example 78 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DbusEventBufferWriter method run.

@Override
public //run in a thread please;
void run() {
    _stop = false;
    _expectedEvents = -1;
    _count = 0;
    try {
        Checkpoint cp = new Checkpoint();
        //is there anything from the checkpoint I can infer that it's end of stream? control message?
        cp.setFlexible();
        do {
            int streamedEvents = 0;
            StreamEventsArgs args = new StreamEventsArgs(_batchsize).setStatsCollector(_stats);
            while ((streamedEvents = _buffer.streamEvents(cp, _channel, args).getNumEventsStreamed()) > 0) {
                _count += streamedEvents;
            }
        //the writer hangs around - cannot count events; cp provides current window and window offset;
        //and streamedEvents has the count of all events - not just data events
        } while (!_stop && !endOfEvents());
    } catch (ScnNotFoundException e) {
        LOG.error("SCN not found! " + e);
    } catch (OffsetNotFoundException e) {
        LOG.error("Offset not found! " + e);
    } finally {
        _stop = false;
    }
}

Example 79 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class DbusEventAppender method addBootstrapCheckpointEventToBuffer.

public void addBootstrapCheckpointEventToBuffer(long lastScn, long dataEventCount, int numCheckpoints) {
    Checkpoint cp = (_bootstrapCheckpoint == null) ? _bstCheckpointHandler.createInitialBootstrapCheckpoint(null, 0L) : _bootstrapCheckpoint;
    if (cp.getBootstrapStartScn() == Checkpoint.UNSET_BOOTSTRAP_START_SCN) {
        cp.setBootstrapStartScn(0L);
    }
    cp.setWindowScn(lastScn);
    cp.setSnapshotOffset(dataEventCount);
    DbusEventInternalReadable ev = new DbusEventV2Factory().createCheckpointEvent(cp);
    ByteBuffer b = ev.value();
    byte[] bytes = new byte[b.remaining()];
    b.get(bytes);
    for (int i = 0; i < numCheckpoints; ++i) {
        _buffer.appendEvent(new DbusEventKey(ev.key()), ev.physicalPartitionId(), ev.logicalPartitionId(), ev.timestampInNanos(), ev.srcId(), ev.schemaId(), bytes, false, _stats);
    }
}

Example 80 with Checkpoint

use of com.linkedin.databus.core.Checkpoint in project databus by linkedin.

the class TestRelayCommandsLocal method doTestOneDataClientVerStreamCommand.

/** Validates the version checks in the stream calls. */
private void doTestOneDataClientVerStreamCommand(int ver, boolean expectFail) throws Exception {
    //try to read it
    Checkpoint cp = Checkpoint.createFlexibleCheckpoint();
    String maxev = "&" + DatabusHttpHeaders.MAX_EVENT_VERSION + "=" + ver;
    // protocol version 2 (versions >= 3 use "subs=")
    String streamRequest = "/stream?sources=100&size=100000&output=json&checkPoint=" + cp.toString() + maxev;
    HttpRequest httpRequest = new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, streamRequest);
    SimpleTestHttpClient httpClient = SimpleTestHttpClient.createLocal(TimeoutPolicy.ALL_TIMEOUTS);
    SimpleHttpResponseHandler respHandler = httpClient.sendRequest(_serverAddress, httpRequest);
    assertTrue("failed to get a response", respHandler.awaitResponseUninterruptedly(1, TimeUnit.SECONDS));
    HttpResponse respObj = respHandler.getResponse();
    if (expectFail) {
        assertNotNull("/stream failed to return expected error", respObj.getHeader(DatabusHttpHeaders.DATABUS_ERROR_CLASS_HEADER));
    } else {
        assertNull("/stream returned unexpected error", respObj.getHeader(DatabusHttpHeaders.DATABUS_ERROR_CLASS_HEADER));
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("/stream response:" + new String(respHandler.getReceivedBytes()));
    }
    ObjectMapper objMapper = new ObjectMapper();
    ByteArrayInputStream in = new ByteArrayInputStream(respHandler.getReceivedBytes());
    objMapper.readValue(in, new TypeReference<Map<String, String>>() {
    });
}