Examples with KeyTypeNotImplementedException com.linkedin.databus.core.KeyTypeNotImplementedException used on opensource projects

Example 1 with KeyTypeNotImplementedException

use of com.linkedin.databus.core.KeyTypeNotImplementedException in project databus by linkedin.

the class BootstrapDBSeeder method appendEvent.

public boolean appendEvent(DbusEventKey key, DbusEventKey seederChunkKey, DbusEventInfo eventInfo, DbusEventsStatisticsCollector statsCollector) {
    boolean isDebugEnabled = LOG.isDebugEnabled();
    boolean ret = false;
    PreparedStatement stmt = null;
    /*
     * Track the startSCN
     */
    if (_startSCN == -1)
        _startSCN = eventInfo.getSequenceId();
    short srcId = eventInfo.getSrcId();
    long sequenceId = eventInfo.getSequenceId();
    try {
        //long start = System.nanoTime();
        if (seederChunkKey.getKeyType() == KeyType.LONG) {
            _lastSeenKey = "" + seederChunkKey.getLongKey();
        } else {
            _lastSeenKey = seederChunkKey.getStringKey();
        }
        if (isDebugEnabled)
            LOG.debug("Seeder Chunk Key is: " + seederChunkKey + ",EventInfo :" + eventInfo + ", Key is :" + key);
        eventInfo.setOpCode(null);
        eventInfo.setAutocommit(true);
        if (eventInfo.getValueLength() >= FIFTY_MB_IN_BYTES) {
            LOG.fatal("Event Size larger than 50 MB. For Key :" + key + ", avro record size is : " + eventInfo.getValueLength());
        }
        DbusEventFactory.serializeEvent(key, _buf, eventInfo);
        long end = System.nanoTime();
        stmt = _statementMap.get(srcId);
        if (null == stmt) {
            stmt = prepareInsertStatement(srcId);
            _statementMap.put(srcId, stmt);
        }
        if (isDebugEnabled) {
            LOG.debug("Number of Bytes in serialized format:" + _buf.position());
            LOG.debug("Key is :" + ((key.getKeyType() == KeyType.LONG) ? key.getLongKey() : key.getStringKey()));
        }
        stmt.setLong(1, sequenceId);
        String keyStr = null;
        if (key.getKeyType() == DbusEventKey.KeyType.LONG) {
            keyStr = key.getLongKey().toString();
        } else {
            keyStr = key.getStringKey();
        }
        stmt.setString(2, keyStr);
        // Reuse the iStream to set the blob
        _bufStream.reset();
        stmt.setBlob(3, _bufStream, _buf.position());
        //stmt.setBytes(3, _buf.array());
        stmt.setLong(4, sequenceId);
        // Reuse the iStream to se the blob
        _bufStream2.reset();
        stmt.setBlob(5, _bufStream2, _buf.position());
        //stmt.setBytes(5, _buf.array());
        stmt.executeUpdate();
        long end2 = System.nanoTime();
        _totLatency += (end2 - end);
        _currSrcId = srcId;
        ret = true;
    } catch (SQLException sqlEx) {
        LOG.error("Error occured while inserting record for key:" + key.getStringKey() + "(" + key.getLongKey() + ") with sequenceId:" + sequenceId, sqlEx);
        throw new RuntimeException(sqlEx);
    } catch (KeyTypeNotImplementedException ex) {
        LOG.error("KeyNotImplemented error while inserting record for key:" + key.getStringKey() + "(" + key.getLongKey() + ") with sequenceId:" + sequenceId, ex);
        throw new RuntimeException(ex);
    } finally {
        _buf.clear();
    }
    return ret;
}

Example 2 with KeyTypeNotImplementedException

use of com.linkedin.databus.core.KeyTypeNotImplementedException in project databus by linkedin.

the class TestUnifiedClientStats method createEvent.

private DbusEvent createEvent(long timestampNs) {
    DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, // SCN
    6004L, // physical partition ID
    (short) 1, // logical partition ID
    (short) 1, timestampNs, // srcId
    (short) 1, // payloadSchemaMd5
    SOURCE1_SCHEMAID, // payload
    SOURCE1_PAYLOAD, // enableTracing
    false, // autocommit
    true);
    DbusEventKey key = new DbusEventKey("myKey".getBytes(Charset.forName("UTF-8")));
    ByteBuffer buf = ByteBuffer.allocate(1000).order(ByteOrder.BIG_ENDIAN);
    try {
        DbusEventFactory.serializeEvent(key, buf, eventInfo);
    } catch (KeyTypeNotImplementedException ex) {
        fail("string key type not supported by DbusEventV2Factory?!? " + ex.getLocalizedMessage());
    }
    return _eventFactory.createReadOnlyDbusEventFromBuffer(buf, 0);
}

Example 3 with KeyTypeNotImplementedException

use of com.linkedin.databus.core.KeyTypeNotImplementedException in project databus by linkedin.

the class DatabusEventRandomProducer method run.

@Override
public void run() {
    _stopGeneration.set(false);
    _suspendGeneration.set(false);
    long startTime = System.nanoTime();
    long startTime0 = startTime;
    long realStartTime = System.currentTimeMillis() * 1000000L;
    _numEventsGenerated = 0;
    int eventBatchSize = 10;
    long currTime = startTime;
    long sleepTimes = 0;
    long sleepTime = 0;
    long firstScn = _startScn;
    long maxScn;
    long numEventsGeneratedAfterResume = 0;
    // get the free space at the beginning as the buffer size
    long eventBufferSize = 0;
    long sizeDataEventsBeforeResume = 0;
    long currSizeDataEvents = _statsCollector.getTotalStats().getSizeDataEvents() * _statsCollector.getTotalStats().getNumDataEvents();
    maxScn = firstScn;
    /* fix firstSCN setting and figure minimum free buffers size available */
    long minSpace = Long.MAX_VALUE;
    for (DbusEventBuffer buf : _dbusEventBuffer.bufIterable()) {
        if (buf.getMinScn() < 0)
            buf.start(firstScn - 1);
        long tmpMinSpace = buf.getBufferFreeSpace();
        if (tmpMinSpace < minSpace)
            minSpace = tmpMinSpace;
    }
    if (_dbusEventBuffer.bufsNum() > 0)
        eventBufferSize = minSpace;
    try {
        _rateMonitor = new RateMonitor("RandomProducer");
        _rateMonitor.start();
        while (!_stopGeneration.get()) {
            LOG.info("Resume. currTime (ms) = " + currTime / MILLISECONDS_IN_NANOS + ", startTime (ms) = " + startTime / MILLISECONDS_IN_NANOS + ", elapseTime = " + (currTime - startTime) / MILLISECONDS_IN_NANOS + ", _duration=" + _duration + ", numEventsToGenerate = " + _numEventsToGenerate.get() + ", StartScn = " + firstScn + ", _minEventsPerWindow=" + _minEventsPerWindow + ", _maxEventsPerWindow =" + _maxEventsPerWindow + ", _keyMin =" + _keyMin + ", _keyMax =" + _keyMax + ", _minLength=" + _minLength + ", _maxLength=" + _maxLength + ", numEvents = " + numEventsGeneratedAfterResume + ", eventBufferSize = " + eventBufferSize + ", sources = " + _sources.size());
            while (!_stopGeneration.get() && !_suspendGeneration.get() && (currTime - startTime < _duration * MILLISECONDS_IN_NANOS) && (numEventsGeneratedAfterResume < _numEventsToGenerate.get()) && (currSizeDataEvents - sizeDataEventsBeforeResume < _percentOfBufferToGenerate.get() / 100.0 * eventBufferSize)) {
                eventBatchSize = _minEventsPerWindow + RngUtils.randomPositiveShort(_realRng) % (_maxEventsPerWindow - _minEventsPerWindow);
                long before = System.nanoTime();
                _rateMonitor.resume();
                maxScn = produceNRandomEvents(firstScn, realStartTime + (currTime - startTime0), eventBatchSize, _sources, _keyMin.get(), _keyMax.get(), _minLength, _maxLength, _schemas);
                assert (maxScn >= firstScn);
                _rateMonitor.ticks(eventBatchSize + 1);
                _rateMonitor.suspend();
                currTime = System.nanoTime();
                currSizeDataEvents = _statsCollector.getTotalStats().getSizeDataEvents() * _statsCollector.getTotalStats().getNumDataEvents();
                firstScn = maxScn + 1;
                _numEventsGenerated += eventBatchSize;
                numEventsGeneratedAfterResume += eventBatchSize;
                _totalGenerationTime += (currTime - before);
                long nextTime = (long) (startTime + numEventsGeneratedAfterResume * _tickInNanos);
                if (nextTime > currTime) {
                    try {
                        ++sleepTimes;
                        sleepTime += (nextTime - currTime);
                        long milliseconds = (nextTime - currTime) / MILLISECONDS_IN_NANOS;
                        int nanoseconds = (int) ((nextTime - currTime) - (milliseconds * MILLISECONDS_IN_NANOS));
                        Thread.sleep(milliseconds, nanoseconds);
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                        return;
                    }
                    currTime = System.nanoTime();
                }
            }
            LOG.info("Suspended. currTime (ms) = " + currTime / MILLISECONDS_IN_NANOS + ", startTime (ms) = " + startTime / MILLISECONDS_IN_NANOS + ", elapseTime = " + (currTime - startTime) / MILLISECONDS_IN_NANOS + ", numEvents = " + numEventsGeneratedAfterResume + ", eventBufferSize = " + eventBufferSize + ", currEventSize = " + currSizeDataEvents + ", startEventSize = " + sizeDataEventsBeforeResume + ", EventSizeDelta = " + (currSizeDataEvents - sizeDataEventsBeforeResume));
            LOG.info(getRateOfProduction());
            //Suspend till resumed
            synchronized (this) {
                if (!_stopGeneration.get()) {
                    boolean doWait = false;
                    // Checking again for suspend inside synchronized block to make sure we didn't miss resume outside the synchronized block.
                    if (!_suspendGeneration.get()) {
                        if (currTime - startTime >= _duration * MILLISECONDS_IN_NANOS || (numEventsGeneratedAfterResume >= _numEventsToGenerate.get()) || (currSizeDataEvents - sizeDataEventsBeforeResume >= _percentOfBufferToGenerate.get() / 100.0 * eventBufferSize)) {
                            // Completed this round. Suspending myself till someone calls resume or stop
                            doWait = true;
                            _suspendGeneration.set(true);
                        } else {
                            // case when someone called suspend and resumed immediately before I reached here
                            doWait = false;
                        }
                    } else {
                        // User requested suspension
                        doWait = true;
                    }
                    while (doWait && (!_stopGeneration.get()) && (_suspendGeneration.get())) {
                        try {
                            this.wait();
                        } catch (InterruptedException ie) {
                            LOG.info("Got Interrupted during suspension: " + ie.getMessage());
                        }
                    }
                    // reset startTime to wall clock
                    startTime = System.nanoTime();
                    currTime = startTime;
                    // reset the event size and number of events
                    numEventsGeneratedAfterResume = 0;
                    sizeDataEventsBeforeResume = currSizeDataEvents;
                }
            }
        }
        _stopGeneration.set(true);
    } catch (RuntimeException e) {
        LOG.error("event generation error:" + e.getMessage(), e);
    } catch (KeyTypeNotImplementedException e) {
        LOG.error("event generation error:" + e.getMessage(), e);
    } finally {
        LOG.info("Produced " + _numEventsGenerated + " events in " + (currTime - startTime) + " nanoseconds.");
        LOG.info("Slept a total of " + sleepTimes + " times for a duration of " + sleepTime + " nanoseconds.");
        LOG.info("Busy time = " + (currTime - startTime - sleepTime) + " nanoseconds.");
        _rateMonitor.stop();
        _generationStopped.countDown();
    }
}

Example 4 with KeyTypeNotImplementedException

use of com.linkedin.databus.core.KeyTypeNotImplementedException in project databus by linkedin.

the class RngUtils method randomEvent.

public static DbusEvent randomEvent(short srcId) {
    ByteBuffer serBuf = ByteBuffer.allocate(1000).order(_eventFactory.getByteOrder());
    try {
        DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, // Sequence number
        0L, // physical Partition
        (short) 0, randomPositiveShort(), System.currentTimeMillis(), srcId, schemaMd5, randomString(20).getBytes(Charset.defaultCharset()), randomPositiveShort() % 100 <= 1, false);
        // make this explicit
        eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V1);
        DbusEventFactory.serializeEvent(new DbusEventKey(randomLong()), serBuf, eventInfo);
    } catch (KeyTypeNotImplementedException e1) {
        throw new RuntimeException(e1);
    }
    serBuf.rewind();
    DbusEventInternalReadable e = _eventFactory.createReadOnlyDbusEventFromBuffer(serBuf, serBuf.position());
    return e;
}

Example 5 with KeyTypeNotImplementedException

use of com.linkedin.databus.core.KeyTypeNotImplementedException in project databus by linkedin.

the class DbusEventGenerator method generateEvents.

/**
   * Generate specified number of constant sized events.
   *
   * @param numEvents : Number of events desired
   * @param windowSize : Max window size (transaction size)
   * @param maxEventSize : maximum event size expected
   * @param payloadSize : payload size in bytes
   * @param useLastEventAsScn : if true, use count of last event of window as window scn; else use i/windowSize+1
   * @param eventVector : output container that is populated with the events
   * @return last window number generated
   */
public long generateEvents(int numEvents, int windowSize, int maxEventSize, int payloadSize, boolean useLastEventAsScn, Vector<DbusEvent> eventVector) {
    long lastScn = 0;
    try {
        long beginningOfTime = System.currentTimeMillis() / 1000;
        beginningOfTime *= 1000;
        short srcId = 1;
        for (int i = 0; i < numEvents; ++i) {
            if (_srcIdList != null && _srcIdList.size() > 0) {
                int srcIdIndex = RngUtils.randomPositiveInt() % _srcIdList.size();
                srcId = _srcIdList.get(srcIdIndex);
            } else {
                srcId = RngUtils.randomPositiveShort();
            }
            if (srcId == 0) {
                //0 srcId not allowed
                srcId = 1;
            }
            //assumption: serialized event fits in maxEventSize
            ByteBuffer buf = ByteBuffer.allocate(maxEventSize).order(_eventFactory.getByteOrder());
            DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, // sequence number
            0L, // physical Partition
            (short) 0, RngUtils.randomPositiveShort(), (beginningOfTime - ((numEvents - i) * 1000)) * 1000 * 1000, //nanoseconds ; first event is numEvents seconds ago
            srcId, RngUtils.schemaMd5, RngUtils.randomString(payloadSize).getBytes(Charset.defaultCharset()), false, false);
            // make this explicit
            eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V1);
            DbusEventFactory.serializeEvent(new DbusEventKey(RngUtils.randomLong()), buf, eventInfo);
            lastScn = (useLastEventAsScn) ? _startScn + ((i / windowSize) + 1) * (long) windowSize : _startScn + (i / windowSize) + 1;
            eventVector.add((DbusEvent) DbusEventFactoryForTesting.createReadOnlyDbusEventFromBuffer(buf, 0, lastScn, DbusEventFactory.DBUS_EVENT_V1));
        }
    } catch (KeyTypeNotImplementedException e) {
    // ignore
    }
    return lastScn;
}