Examples with SchemaId com.linkedin.databus2.schemas.SchemaId used on opensource projects

Example 11 with SchemaId

use of com.linkedin.databus2.schemas.SchemaId in project databus by linkedin.

the class GenericDispatcher method doDispatchEvents.

protected void doDispatchEvents() {
    boolean debugEnabled = _log.isDebugEnabled();
    boolean traceEnabled = _log.isTraceEnabled();
    //need to remove eventually but for now I want to avoid a nasty diff
    final DispatcherState curState = _internalState;
    if (!_stopDispatch.get() && !curState.getEventsIterator().hasNext() && !checkForShutdownRequest()) {
        if (debugEnabled)
            _log.debug("Waiting for events");
        curState.getEventsIterator().await(50, TimeUnit.MILLISECONDS);
    }
    boolean success = true;
    boolean hasQueuedEvents = false;
    while (success && !_stopDispatch.get() && curState.getStateId() != DispatcherState.StateId.STOP_DISPATCH_EVENTS && null != curState.getEventsIterator() && curState.getEventsIterator().hasNext() && !checkForShutdownRequest() && //exit the event processing loop if there are other queued notifications
    !hasMessages()) {
        DbusEventInternalReadable nextEvent = curState.getEventsIterator().next();
        _currentWindowSizeInBytes += nextEvent.size();
        if (traceEnabled)
            _log.trace("Got event:" + nextEvent);
        Long eventSrcId = (long) nextEvent.srcId();
        if (curState.isSCNRegress()) {
            SingleSourceSCN scn = new SingleSourceSCN(nextEvent.physicalPartitionId(), nextEvent.sequence());
            _log.info("We are regressing to SCN: " + scn);
            curState.switchToRollback();
            doRollback(curState, scn, false, false);
            curState.setSCNRegress(false);
            curState.switchToExpectEventWindow();
        }
        if (null != getAsyncCallback().getStats())
            getAsyncCallback().getStats().registerWindowSeen(nextEvent.timestampInNanos(), nextEvent.sequence());
        if (nextEvent.isControlMessage()) {
            //control event
            if (nextEvent.isEndOfPeriodMarker()) {
                if (curState.isEventsSeen()) {
                    if (null != curState.getCurrentSource()) {
                        curState.switchToEndStreamSource();
                        success = doEndStreamSource(curState);
                    }
                    SCN endWinScn = null;
                    if (success) {
                        _lastWindowScn = nextEvent.sequence();
                        _lastEowTsNsecs = nextEvent.timestampInNanos();
                        endWinScn = new SingleSourceSCN(nextEvent.physicalPartitionId(), _lastWindowScn);
                        curState.switchToEndStreamEventWindow(endWinScn);
                        success = doEndStreamEventWindow(curState);
                    }
                    if (success) {
                        try {
                            //end of period event
                            Checkpoint cp = createCheckpoint(curState, nextEvent);
                            success = doStoreCheckpoint(curState, nextEvent, cp, endWinScn);
                        } catch (SharedCheckpointException e) {
                            //shutdown
                            return;
                        }
                    }
                } else {
                    //empty window
                    success = true;
                    if (_log.isDebugEnabled()) {
                        _log.debug("skipping empty window: " + nextEvent.sequence());
                    }
                    //write a checkpoint; takes care of slow sources ; but skip storing the first control eop with 0 scn
                    if (nextEvent.sequence() > 0) {
                        _lastWindowScn = nextEvent.sequence();
                        //The reason is that the eop's timestamp is the max timestamp of all data events seen so far.
                        if (nextEvent.timestampInNanos() > 0) {
                            _lastEowTsNsecs = nextEvent.timestampInNanos();
                        }
                        Checkpoint ckpt = createCheckpoint(curState, nextEvent);
                        try {
                            success = doStoreCheckpoint(curState, nextEvent, ckpt, new SingleSourceSCN(nextEvent.physicalPartitionId(), nextEvent.sequence()));
                        } catch (SharedCheckpointException e) {
                            //shutdown
                            return;
                        }
                    } else {
                        _log.warn("EOP with scn=" + nextEvent.sequence());
                    }
                }
                if (success) {
                    curState.switchToExpectEventWindow();
                    //we have recovered from the error  and it's not the dummy window
                    if (nextEvent.sequence() > 0) {
                        if (!getStatus().isRunningStatus())
                            getStatus().resume();
                    }
                }
            } else if (nextEvent.isErrorEvent()) {
                _log.info("Error event: " + nextEvent.sequence());
                success = processErrorEvent(curState, nextEvent);
            } else {
                //control event
                success = processSysEvent(curState, nextEvent);
                if (success) {
                    if (nextEvent.isCheckpointMessage()) {
                        Checkpoint sysCheckpt = createCheckpoint(curState, nextEvent);
                        try {
                            long scn = sysCheckpt.getConsumptionMode() == DbusClientMode.ONLINE_CONSUMPTION ? nextEvent.sequence() : sysCheckpt.getBootstrapSinceScn();
                            //ensure that control event with 0 scn doesn't get saved unless it is during snapshot of bootstrap
                            if (scn > 0 || sysCheckpt.getConsumptionMode() == DbusClientMode.BOOTSTRAP_SNAPSHOT) {
                                success = doStoreCheckpoint(curState, nextEvent, sysCheckpt, new SingleSourceSCN(nextEvent.physicalPartitionId(), scn));
                            }
                        } catch (SharedCheckpointException e) {
                            //shutdown
                            return;
                        }
                    }
                }
            }
        } else {
            curState.setEventsSeen(true);
            //not a control event
            if (curState.getStateId().equals(StateId.EXPECT_EVENT_WINDOW) || curState.getStateId().equals(StateId.REPLAY_DATA_EVENTS)) {
                SCN startScn = new SingleSourceSCN(nextEvent.physicalPartitionId(), nextEvent.sequence());
                curState.switchToStartStreamEventWindow(startScn);
                success = doStartStreamEventWindow(curState);
                if (success && (eventSrcId.longValue() >= 0)) {
                    success = doCheckStartSource(curState, eventSrcId, new SchemaId(nextEvent.schemaId()));
                }
            } else {
                if (null != curState.getCurrentSource() && !eventSrcId.equals(curState.getCurrentSource().getId())) {
                    curState.switchToEndStreamSource();
                    success = doEndStreamSource(curState);
                }
                if (success) {
                    //Check if schemas of the source exist.
                    //Also check if the exact schema id present in event exists in the client. This is worthwhile if there's a
                    //guarantee that the entire window is written with the same schemaId, which is the case if the relay does not use a new schema
                    //mid-window
                    success = doCheckStartSource(curState, eventSrcId, new SchemaId(nextEvent.schemaId()));
                }
            }
            if (success) {
                //finally: process data event
                success = processDataEvent(curState, nextEvent);
                if (success) {
                    hasQueuedEvents = true;
                    if (hasCheckpointThresholdBeenExceeded()) {
                        _log.info("Attempting to checkpoint (only if the consumer callback for onCheckpoint returns SUCCESS), because " + getCurrentWindowSizeInBytes() + " bytes reached without checkpoint ");
                        success = processDataEventsBatch(curState);
                        if (success) {
                            hasQueuedEvents = false;
                            //checkpoint: for bootstrap it's the right checkpoint; that has been lazily created by a checkpoint event
                            // checkpoint: for relay: create a checkpoint that has the prevScn
                            Checkpoint cp = createCheckpoint(curState, nextEvent);
                            // DDSDBUS-1889 : scn for bootstrap is bootstrapSinceSCN
                            // scn for online consumption is : currentWindow
                            SCN lastScn = cp.getConsumptionMode() == DbusClientMode.ONLINE_CONSUMPTION ? curState.getStartWinScn() : new SingleSourceSCN(nextEvent.physicalPartitionId(), cp.getBootstrapSinceScn());
                            try {
                                // Even if storeCheckpoint fails, we
                                // should continue (hoping for the best)
                                success = doStoreCheckpoint(curState, nextEvent, cp, lastScn);
                            } catch (SharedCheckpointException e) {
                                // shutdown
                                return;
                            }
                            curState.switchToExpectStreamDataEvents();
                            if (!getStatus().isRunningStatus())
                                getStatus().resume();
                        }
                    }
                }
            }
        }
        if (success) {
            // before next successful checkpoint
            if (hasCheckpointThresholdBeenExceeded()) {
                //drain events just in case it hasn't been drained before; mainly control events that are not checkpoint events
                success = processDataEventsBatch(curState);
                if (success) {
                    _log.warn("Checkpoint not stored, but removing older events from buffer to guarantee progress (checkpoint threshold has" + " exceeded), consider checkpointing more frequently. Triggered on control-event=" + nextEvent.isControlMessage());
                    // guarantee progress: risk being unable to rollback by
                    // removing events, but hope for the best
                    removeEvents(curState);
                }
            }
        }
    }
    if (!_stopDispatch.get() && !checkForShutdownRequest()) {
        if (success) {
            if (hasQueuedEvents) {
                success = processDataEventsBatch(curState);
                if (!success) {
                    _log.error("Unable to flush partial window");
                }
            }
            if (debugEnabled)
                _log.debug("doDispatchEvents to " + curState.toString());
        }
        if (!success) {
            curState.switchToRollback();
            doRollback(curState);
        }
        //loop around -- let any other messages be processed
        enqueueMessage(curState);
    }
}

Example 12 with SchemaId

use of com.linkedin.databus2.schemas.SchemaId in project databus by linkedin.

the class OpenReplicatorAvroEventFactory method createAndAppendEvent.

public int createAndAppendEvent(DbChangeEntry changeEntry, DbusEventBufferAppendable eventBuffer, boolean enableTracing, DbusEventsStatisticsCollector dbusEventsStatisticsCollector) throws EventCreationException, UnsupportedKeyException, DatabusException {
    Object keyObj = obtainKey(changeEntry);
    //Construct the Databus Event key, determine the key type and construct the key
    DbusEventKey eventKey = new DbusEventKey(keyObj);
    short lPartitionId = _partitionFunction.getPartition(eventKey);
    //Get the md5 for the schema
    SchemaId schemaId = SchemaId.createWithMd5(changeEntry.getSchema());
    byte[] payload = serializeEvent(changeEntry.getRecord());
    DbusEventInfo eventInfo = new DbusEventInfo(changeEntry.getOpCode(), changeEntry.getScn(), (short) _pSourceId, lPartitionId, changeEntry.getTimestampInNanos(), (short) _sourceId, schemaId.getByteArray(), payload, enableTracing, false);
    boolean success = eventBuffer.appendEvent(eventKey, eventInfo, dbusEventsStatisticsCollector);
    return success ? payload.length : -1;
}

Example 13 with SchemaId

use of com.linkedin.databus2.schemas.SchemaId in project databus by linkedin.

the class TestRegisterRequestProcessor method testV4RegisterRequestProcessor.

// Test the happy path where there are 2 versions of document schema of a table and one version of metadata
// schema in the registry.
@Test
public void testV4RegisterRequestProcessor() throws Exception {
    Properties params = new Properties();
    final int protoVersion = 4;
    final int srcId1 = 101;
    final String srcName1 = "source-101";
    final String docSchema1 = "docSchema1";
    final String docSchema2 = "docSchema2";
    final String metadataSchema1 = makeMetadataSchema(1);
    final String metadataSchema2 = makeMetadataSchema(2);
    final byte[] metaSchemaDigest1 = new byte[] { 32, 33, 34, 35 };
    final byte[] metaSchemaDigest2 = new byte[] { 35, 34, 33, 32 };
    final short docSchemaV1 = 1;
    final short docSchemaV2 = 2;
    final short metaSchemaV1 = 1;
    final short metaSchemaV2 = 2;
    params.setProperty(DatabusHttpHeaders.PROTOCOL_VERSION_PARAM, Integer.toString(protoVersion));
    params.setProperty(RegisterRequestProcessor.SOURCES_PARAM, Integer.toString(srcId1));
    final StringBuilder responseStr = new StringBuilder();
    ChunkedWritableByteChannel chunkedWritableByteChannel = EasyMock.createMock(ChunkedWritableByteChannel.class);
    chunkedWritableByteChannel.addMetadata(EasyMock.eq(DatabusHttpHeaders.DBUS_CLIENT_RELAY_PROTOCOL_VERSION_HDR), EasyMock.eq(protoVersion));
    chunkedWritableByteChannel.write(EasyMock.anyObject(ByteBuffer.class));
    EasyMock.expectLastCall().andAnswer(new IAnswer<Object>() {

        @Override
        public Object answer() throws Throwable {
            Charset charset = Charset.forName("UTF-8");
            CharsetDecoder decoder = charset.newDecoder();
            responseStr.append(decoder.decode((ByteBuffer) EasyMock.getCurrentArguments()[0]));
            return responseStr.length();
        }
    });
    EasyMock.replay(chunkedWritableByteChannel);
    DatabusRequest mockReq = EasyMock.createMock(DatabusRequest.class);
    EasyMock.expect(mockReq.getParams()).andReturn(params).anyTimes();
    EasyMock.expect(mockReq.getResponseContent()).andReturn(chunkedWritableByteChannel);
    EasyMock.replay(mockReq);
    LogicalSource lsrc1 = new LogicalSource(srcId1, srcName1);
    SourceIdNameRegistry mockSrcIdReg = EasyMock.createMock(SourceIdNameRegistry.class);
    EasyMock.expect(mockSrcIdReg.getSource(srcId1)).andReturn(lsrc1).anyTimes();
    EasyMock.replay(mockSrcIdReg);
    Map<Short, String> srcSchemaVersions = new HashMap<Short, String>();
    srcSchemaVersions.put(docSchemaV1, docSchema1);
    srcSchemaVersions.put(docSchemaV2, docSchema2);
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, metaSchemaV1, new SchemaId(metaSchemaDigest1), metadataSchema1, true);
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, metaSchemaV2, new SchemaId(metaSchemaDigest2), metadataSchema2, true);
    SchemaRegistryService mockSchemaReg = EasyMock.createMock(SchemaRegistryService.class);
    EasyMock.expect(mockSchemaReg.fetchAllSchemaVersionsBySourceName(srcName1)).andReturn(srcSchemaVersions).anyTimes();
    EasyMock.expect(mockSchemaReg.fetchAllMetadataSchemaVersions()).andReturn(metadataSchemaSet).anyTimes();
    EasyMock.replay(mockSchemaReg);
    HttpRelay mockRelay = EasyMock.createMock(HttpRelay.class);
    EasyMock.expect(mockRelay.getHttpStatisticsCollector()).andReturn(null).anyTimes();
    EasyMock.expect(mockRelay.getSourcesIdNameRegistry()).andReturn(mockSrcIdReg).anyTimes();
    EasyMock.expect(mockRelay.getSchemaRegistryService()).andReturn(mockSchemaReg).anyTimes();
    EasyMock.replay(mockRelay);
    RegisterRequestProcessor reqProcessor = new RegisterRequestProcessor(null, mockRelay);
    reqProcessor.process(mockReq);
    // Decode
    ObjectMapper mapper = new ObjectMapper();
    HashMap<String, List<Object>> responseMap = mapper.readValue(responseStr.toString(), new TypeReference<HashMap<String, List<Object>>>() {
    });
    Map<Long, List<RegisterResponseEntry>> sourcesSchemasMap = RegisterResponseEntry.createFromResponse(responseMap, RegisterResponseEntry.SOURCE_SCHEMAS_KEY, false);
    // There should be one entry in the map, which is a  list.
    Assert.assertEquals(1, sourcesSchemasMap.size());
    Assert.assertEquals(2, sourcesSchemasMap.get(new Long(srcId1)).size());
    for (RegisterResponseEntry r : sourcesSchemasMap.get(new Long(srcId1))) {
        Assert.assertEquals(srcId1, r.getId());
        if (r.getVersion() == docSchemaV1) {
            Assert.assertEquals(docSchema1, r.getSchema());
        } else {
            Assert.assertEquals(docSchema2, r.getSchema());
        }
    }
    Map<Long, List<RegisterResponseEntry>> keysSchemasMap = RegisterResponseEntry.createFromResponse(responseMap, RegisterResponseEntry.KEY_SCHEMAS_KEY, true);
    Assert.assertNull(keysSchemasMap);
    List<RegisterResponseMetadataEntry> metadataSchemasList = RegisterResponseMetadataEntry.createFromResponse(responseMap, RegisterResponseMetadataEntry.METADATA_SCHEMAS_KEY, true);
    // The response should contain the exact string that the schema registry has.
    Assert.assertEquals(2, metadataSchemasList.size());
    for (RegisterResponseMetadataEntry r : metadataSchemasList) {
        if (r.getVersion() == 1) {
            Assert.assertEquals(metadataSchema1, r.getSchema());
            Assert.assertTrue(Arrays.equals(metaSchemaDigest1, r.getCrc32()));
        } else {
            Assert.assertEquals(metadataSchema2, r.getSchema());
            Assert.assertTrue(Arrays.equals(metaSchemaDigest2, r.getCrc32()));
        }
    }
    EasyMock.verify(mockRelay);
    EasyMock.verify(mockReq);
    EasyMock.verify(mockSchemaReg);
    EasyMock.verify(mockSrcIdReg);
}

Example 14 with SchemaId

use of com.linkedin.databus2.schemas.SchemaId in project databus by linkedin.

the class TestInternalMetadata method testGetMetadata_UnhappyPath_BadSchema.

/**
   * Verifies that getMetadata() returns null if there's a mismatch between the event's metadata
   * and the metadata schema whose signature/checksum is specified in the event header.
   */
@Test
public void testGetMetadata_UnhappyPath_BadSchema() throws Exception {
    LOG.info("starting testGetMetadata_UnhappyPath_BadSchema()");
    // build the event's metadata and then the event
    DbusEventPart metadataPart = createMetadataPart();
    DbusEvent event = createEvent(metadataPart);
    // create a metadata schema set with a schema that claims to match the event's
    // metadata but doesn't actually
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, // METADATA_SCHEMA_VERSION
    metadataPart.getSchemaVersion(), // METADATA_SCHEMA_CHECKSUM
    new SchemaId(metadataPart.getSchemaDigest()), INCORRECT_METADATA_SCHEMA, // preserve original string
    true);
    // now create the decoder and attempt to use it to extract and decode the event's metadata
    DbusEventAvroDecoder eventDecoder = createDecoder(metadataSchemaSet);
    try {
        GenericRecord reuse = null;
        GenericRecord decodedMetadata = eventDecoder.getMetadata(event, reuse);
        Assert.assertNull(decodedMetadata, "getMetadata() should have returned null;");
    } catch (Exception ex) {
        Assert.fail("getMetadata() should not have thrown exception: " + ex);
    }
    LOG.info("leaving testGetMetadata_UnhappyPath_BadSchema()");
}

Example 15 with SchemaId

use of com.linkedin.databus2.schemas.SchemaId in project databus by linkedin.

the class TestInternalMetadata method testGetMetadata_HappyPath.

/**
   * Verifies that getMetadata() returns the expected GenericRecord for the event's
   * metadata and that it has the expected fields and values in it.
   */
@Test
public void testGetMetadata_HappyPath() throws Exception {
    LOG.info("starting testGetMetadata_HappyPath()");
    // build the event's metadata and then the event
    DbusEventPart metadataPart = createMetadataPart();
    DbusEvent event = createEvent(metadataPart);
    // create a metadata schema set that correctly corresponds to the metadata
    VersionedSchemaSet metadataSchemaSet = new VersionedSchemaSet();
    metadataSchemaSet.add(SchemaRegistryService.DEFAULT_METADATA_SCHEMA_SOURCE, // METADATA_SCHEMA_VERSION
    metadataPart.getSchemaVersion(), // METADATA_SCHEMA_CHECKSUM
    new SchemaId(metadataPart.getSchemaDigest()), CORRECT_METADATA_SCHEMA, // preserve original string
    true);
    // now create the decoder and use it to extract and decode the event's metadata
    DbusEventAvroDecoder eventDecoder = createDecoder(metadataSchemaSet);
    try {
        GenericRecord reuse = null;
        GenericRecord decodedMetadata = eventDecoder.getMetadata(event, reuse);
        Assert.assertNotNull(decodedMetadata, "getMetadata() returned null GenericRecord;");
        Utf8 etag = (Utf8) decodedMetadata.get("etag");
        Assert.assertEquals(etag.toString(), "dunno what an etag is");
        Integer flags = (Integer) decodedMetadata.get("flags");
        Assert.assertEquals(flags, null, "expected flags to be null");
        Long expires = (Long) decodedMetadata.get("expires");
        Assert.assertNotNull(expires, "expected expires to have a value;");
        Assert.assertEquals(expires.longValue(), 1366150681);
        Utf8 nonexistentField = (Utf8) decodedMetadata.get("nonexistentField");
        Assert.assertNull(nonexistentField, "unexpected value for 'nonexistentField';");
    } catch (Exception ex) {
        Assert.fail("unexpected error decoding metadata: " + ex);
    }
    LOG.info("leaving testGetMetadata_HappyPath()");
}