Examples with DatabusException com.linkedin.databus2.core.DatabusException used on opensource projects

Example 51 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class GGEventGenerationFactory method convertToSimpleType.

public static Object convertToSimpleType(String fieldValue, Schema.Field avroField) throws DatabusException {
    String databaseFieldType = SchemaHelper.getMetaField(avroField, "dbFieldType");
    String recordFieldName = avroField.name();
    //return int
    if (databaseFieldType.equalsIgnoreCase("INTEGER")) {
        return new Integer(fieldValue);
    } else //return long
    if (databaseFieldType.equalsIgnoreCase("LONG")) {
        return new Long(fieldValue);
    } else if (databaseFieldType.equalsIgnoreCase("DATE")) {
        return ggDateStringToLong(fieldValue);
    } else if (databaseFieldType.equalsIgnoreCase("TIMESTAMP")) {
        return ggTimeStampStringToMilliSeconds(fieldValue);
    } else //return float
    if (databaseFieldType.equalsIgnoreCase("FLOAT")) {
        return new Float(fieldValue);
    } else //return double
    if (databaseFieldType.equalsIgnoreCase("DOUBLE")) {
        return new Double(fieldValue);
    } else //return string
    if (databaseFieldType.equalsIgnoreCase("CLOB")) {
        return fieldValue;
    } else if (databaseFieldType.equalsIgnoreCase("VARCHAR")) {
        return fieldValue;
    } else if (databaseFieldType.equalsIgnoreCase("VARCHAR2")) {
        return fieldValue;
    } else if (databaseFieldType.equalsIgnoreCase("NVARCHAR")) {
        return fieldValue;
    } else if (databaseFieldType.equalsIgnoreCase("NVARCHAR2")) {
        return fieldValue;
    } else if (databaseFieldType.equalsIgnoreCase("XMLTYPE")) {
        return fieldValue;
    } else if (databaseFieldType.equalsIgnoreCase("CHAR")) {
        return fieldValue;
    } else //return bytes
    if (databaseFieldType.equalsIgnoreCase("BLOB") || databaseFieldType.equalsIgnoreCase("RAW")) {
        if (fieldValue.length() == 0) {
            return fieldValue.getBytes(Charset.defaultCharset());
        }
        if (fieldValue.length() <= 2) {
            throw new DatabusException("Unable to decode the string because length is less than 2");
        }
        if (!isStringHex(fieldValue)) {
            throw new DatabusException("Unable to decode the string because it is not hex-encoded");
        }
        try {
            return stringToHex(fieldValue.substring(2, fieldValue.length() - 1));
        } catch (DecoderException e) {
            throw new DatabusException("Unable to decode a " + databaseFieldType + " field: " + recordFieldName);
        }
    } else //return array
    if (databaseFieldType.equalsIgnoreCase("ARRAY")) {
        //TODO add support for array
        throw new DatabusException("ARRAY type still not implemented!");
    } else //return record
    if (databaseFieldType.equalsIgnoreCase("TABLE")) {
        //TODO add support for table
        throw new DatabusException("TABLE type still not implemented!");
    } else {
        throw new DatabusException("unknown field type: " + recordFieldName + ":" + databaseFieldType);
    }
}

Example 52 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class TestGoldenGateEventProducer method testAddEventToBuffer.

@Test
public void testAddEventToBuffer() throws InvalidConfigException, UnsupportedKeyException, DatabusException {
    // No rate control
    long rate = 0;
    PhysicalSourceStaticConfig pssc = buildPssc(rate, 0L);
    long scn = 10;
    DbusEventBuffer mb = (DbusEventBuffer) createBufMult(pssc);
    GoldenGateEventProducer gg = new GoldenGateEventProducer(pssc, null, mb, null, null);
    List<TransactionState.PerSourceTransactionalUpdate> dbUpdates = new ArrayList<TransactionState.PerSourceTransactionalUpdate>(10);
    int sourceId = 505;
    HashSet<DBUpdateImage> db = new HashSet<DBUpdateImage>();
    Object key = new String("name");
    Schema.Type keyType = Schema.Type.RECORD;
    ColumnsState.KeyPair kp = new ColumnsState.KeyPair(key, keyType);
    ArrayList<ColumnsState.KeyPair> keyPairs = new ArrayList<ColumnsState.KeyPair>(1);
    keyPairs.add(kp);
    Schema s = Schema.parse(avroSchema);
    GenericRecord gr = new GenericData.Record(s);
    gr.put("name", "phani");
    DBUpdateImage dbi = new DBUpdateImage(keyPairs, scn, gr, s, DbUpdateState.DBUpdateImage.OpType.INSERT, false);
    db.add(dbi);
    TransactionState.PerSourceTransactionalUpdate dbUpdate = new TransactionState.PerSourceTransactionalUpdate(sourceId, db);
    dbUpdates.add(dbUpdate);
    long timestamp = System.nanoTime();
    gg.addEventToBuffer(dbUpdates, new TransactionInfo(0, 0, timestamp, scn));
    Assert.assertEquals(gg.getRateControl().getNumSleeps(), 0);
    DbusEventIterator iter = mb.acquireIterator("test");
    int count = 0;
    long eventTs = 0;
    while (iter.hasNext()) {
        DbusEvent e = iter.next();
        if (count == 1) {
            // first event prev control event
            eventTs = e.timestampInNanos();
        }
        count++;
    }
    Assert.assertEquals("Event timestamp in Ns", timestamp, eventTs);
    Assert.assertEquals("Got events ", 3, count);
    return;
}

Example 53 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class TestDatabusRelayEvents method testV2Events.

/**
   * Stuffs an event buffer with both a v1 and a v2 event, then reads the buffer two ways:
   * first accepting only v1 events (verifying conversion of the v2 event to v1); then accepting
   * both v1 and v2 events.
   *
   * Note that the version of the _EOP_ events must match the version of the event factory,
   * regardless of the versions of any preceding "real" events.  (This matches DbusEventBuffer
   * behavior; see the serializeLongKeyEndOfPeriodMarker() call in endEvents() for details.)
   */
@Test
public void testV2Events() throws KeyTypeNotImplementedException, InvalidEventException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testV2Events");
    log.setLevel(Level.DEBUG);
    String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
    short srcId = 2;
    short pId = 1;
    int relayPort = Utils.getAvailablePort(11993);
    // create relay
    final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        //EventProducer[] producers = relay1.getProducers();
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        PhysicalPartition pPartition = new PhysicalPartition((int) pId, pSourceName);
        DbusEventBuffer buf = (DbusEventBuffer) bufMult.getDbusEventBufferAppendable(pPartition);
        log.info("create some events");
        long windowScn = 100L;
        ByteBuffer serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V2);
        ReadableByteChannel channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        int readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        windowScn = 101L;
        serializationBuffer = addEvent(windowScn, srcId, relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray(), pId, DbusEventFactory.DBUS_EVENT_V1);
        channel = Channels.newChannel(new ByteBufferInputStream(serializationBuffer));
        readEvents = buf.readEvents(channel);
        log.info("successfully read in " + readEvents + " events ");
        channel.close();
        log.info("starting relay on port " + relayPort);
        r1.start();
        //TestUtil.sleep(10*1000);
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 30000, LOG);
        log.info("now create client");
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        log.info("starting client");
        cr.start();
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer.getNumDataEvents();
                LOG.info("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 2);
        log.info("shutdown first client");
        clientConn.stop();
        cr.shutdown();
        TestUtil.sleep(1000);
        cr = null;
        log.info("start another client who understands V2");
        final EventsCountingConsumer countingConsumer1 = new EventsCountingConsumer();
        clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer1, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V2, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("wait till client gets the event");
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                int events = countingConsumer1.getNumDataEvents();
                LOG.debug("client got " + events + " events");
                return events == 2;
            }
        }, "Consumer didn't get 2 events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(countingConsumer1.getNumDataEvents(), 2);
        Assert.assertEquals(countingConsumer1.getNumWindows(), 2);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1), 1);
        Assert.assertEquals(countingConsumer1.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V2), 1);
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 54 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class TestDatabusRelayEvents method createRelay.

/**
   * create a test relay with event producer turned off
   */
private DatabusRelayMain createRelay(int relayPort, int pId, String[] srcs) throws IOException, DatabusException {
    // create main relay with random generator
    PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcs.length];
    String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
    PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) pId, pSourceName, "mock", 500, 0, srcs);
    srcConfigs[0] = src1;
    HttpRelay.Config httpRelayConfig = DatabusRelayTestUtil.createHttpRelayConfig(1002, relayPort, 3024);
    // do not produce any events
    httpRelayConfig.setStartDbPuller("false");
    httpRelayConfig.getSchemaRegistry().getFileSystem().setSchemaDir("TestDatabusRelayEvents_schemas");
    final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelay(srcConfigs, httpRelayConfig);
    Assert.assertNotNull(relay1);
    return relay1;
}

Example 55 with DatabusException

use of com.linkedin.databus2.core.DatabusException in project databus by linkedin.

the class TestDatabusRelayEvents method testEventConversion.

@Test
public /**
   * append event V2 to the buffer and stream it to the client
   * which only accepts events V1. Make sure it got converted
   */
void testEventConversion() throws InterruptedException, IOException, DatabusException {
    final Logger log = Logger.getLogger("TestDatabusRelayEvents.testEventConversion");
    log.setLevel(Level.INFO);
    DatabusRelayTestUtil.RelayRunner r1 = null;
    ClientRunner cr = null;
    try {
        String[] srcs = { "com.linkedin.events.example.fake.FakeSchema" };
        int pId = 1;
        int srcId = 2;
        int relayPort = Utils.getAvailablePort(11994);
        ;
        final DatabusRelayMain relay1 = createRelay(relayPort, pId, srcs);
        Assert.assertNotNull(relay1);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("Relay created");
        DbusEventBufferMult bufMult = relay1.getEventBuffer();
        String pSourceName = DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]);
        PhysicalPartition pPartition = new PhysicalPartition(pId, pSourceName);
        DbusEventBufferAppendable buf = bufMult.getDbusEventBufferAppendable(pPartition);
        DbusEventKey key = new DbusEventKey(123L);
        byte[] schemaId = relay1.getSchemaRegistryService().fetchSchemaIdForSourceNameAndVersion(srcs[0], 2).getByteArray();
        byte[] payload = RngUtils.randomString(100).getBytes(Charset.defaultCharset());
        DbusEventInfo eventInfo = new DbusEventInfo(DbusOpcode.UPSERT, 100L, (short) pId, (short) pId, 897L, (short) srcId, schemaId, payload, false, true);
        eventInfo.setEventSerializationVersion(DbusEventFactory.DBUS_EVENT_V2);
        buf.startEvents();
        buf.appendEvent(key, eventInfo, null);
        buf.endEvents(100L, null);
        r1.start();
        log.info("Relay started");
        // wait until relay comes up
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return relay1.isRunningStatus();
            }
        }, "Relay hasn't come up completely ", 7000, LOG);
        // now create client:
        String srcSubscriptionString = TestUtil.join(srcs, ",");
        String serverName = "localhost:" + relayPort;
        final EventsCountingConsumer countingConsumer = new EventsCountingConsumer();
        int id = (RngUtils.randomPositiveInt() % 10000) + 1;
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", id, serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true, DatabusClientNettyThreadPools.createNettyThreadPools(id), 0, DbusEventFactory.DBUS_EVENT_V1, 0);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("Consumer started");
        // wait till client gets the event
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                return countingConsumer.getNumDataEvents() == 1;
            }
        }, "Consumer didn't get any events ", 64 * 1024, LOG);
        // asserts
        Assert.assertEquals(1, countingConsumer.getNumDataEvents());
        Assert.assertEquals(1, countingConsumer.getNumWindows());
        Assert.assertEquals(1, countingConsumer.getNumDataEvents(DbusEventFactory.DBUS_EVENT_V1));
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}