Examples with Logger org.apache.log4j.Logger used on opensource projects

Example 96 with Logger

use of org.apache.log4j.Logger in project databus by linkedin.

the class TestDatabusRelayMain method testRelayEventGenerator.

@Test
public void testRelayEventGenerator() throws InterruptedException, InvalidConfigException {
    DatabusRelayTestUtil.RelayRunner r1 = null;
    //Logger.getRootLogger().setLevel(Level.INFO);
    final Logger log = Logger.getLogger("TestDatabusRelayMain.testRelayEventGenerator");
    //log.setLevel(Level.DEBUG);
    ClientRunner cr = null;
    try {
        String[][] srcNames = { { "com.linkedin.events.example.fake.FakeSchema", "com.linkedin.events.example.person.Person" } };
        log.info("create main relay with random generator");
        PhysicalSourceConfig[] srcConfigs = new PhysicalSourceConfig[srcNames.length];
        int i = 0;
        int eventRatePerSec = 20;
        for (String[] srcs : srcNames) {
            PhysicalSourceConfig src1 = DatabusRelayTestUtil.createPhysicalConfigBuilder((short) (i + 1), DatabusRelayTestUtil.getPhysicalSrcName(srcs[0]), "mock", 500, eventRatePerSec, srcs);
            srcConfigs[i++] = src1;
        }
        int relayPort = Utils.getAvailablePort(11993);
        final DatabusRelayMain relay1 = DatabusRelayTestUtil.createDatabusRelayWithSchemaReg(1001, relayPort, 10 * 1024 * 1024, srcConfigs, SCHEMA_REGISTRY_DIR);
        Assert.assertNotEquals(relay1, null);
        r1 = new DatabusRelayTestUtil.RelayRunner(relay1);
        log.info("async starts");
        r1.start();
        log.info("start client in parallel");
        String srcSubscriptionString = TestUtil.join(srcNames[0], ",");
        String serverName = "localhost:" + relayPort;
        final CountingConsumer countingConsumer = new CountingConsumer();
        DatabusSourcesConnection clientConn = RelayEventProducer.createDatabusSourcesConnection("testProducer", serverName, srcSubscriptionString, countingConsumer, 1 * 1024 * 1024, 50000, 30 * 1000, 100, 15 * 1000, 1, true);
        cr = new ClientRunner(clientConn);
        cr.start();
        log.info("terminating conditions");
        final DbusEventsTotalStats stats = relay1.getInboundEventStatisticsCollector().getTotalStats();
        long totalRunTime = 5000;
        long startTime = System.currentTimeMillis();
        do {
            log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
            Thread.sleep(1000);
        } while ((System.currentTimeMillis() - startTime) < totalRunTime);
        r1.pause();
        log.info("Sending pause to relay!");
        log.info("numDataEvents=" + stats.getNumDataEvents() + " numWindows=" + stats.getNumSysEvents() + " size=" + stats.getSizeDataEvents());
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                boolean success = true;
                for (EventProducer p : relay1.getProducers()) {
                    if (!(success = success && p.isPaused()))
                        break;
                }
                return success;
            }
        }, "waiting for producers to pause", 4000, log);
        TestUtil.assertWithBackoff(new ConditionCheck() {

            @Override
            public boolean check() {
                log.debug("countingConsumer.getNumWindows()=" + countingConsumer.getNumWindows());
                return countingConsumer.getNumWindows() == stats.getNumSysEvents();
            }
        }, "wait until client got all events or for maxTimeout", 64 * 1024, log);
        log.info("Client stats=" + countingConsumer);
        log.info("Event windows generated=" + stats.getNumSysEvents());
        cr.shutdown(2000, log);
        log.info("Client cr stopped");
        Assert.assertEquals(countingConsumer.getNumDataEvents(), stats.getNumDataEvents());
        boolean stopped = r1.shutdown(2000);
        Assert.assertTrue(stopped);
        log.info("Relay r1 stopped");
    } finally {
        cleanup(new DatabusRelayTestUtil.RelayRunner[] { r1 }, cr);
    }
}

Example 97 with Logger

use of org.apache.log4j.Logger in project databus by linkedin.

the class TestTrailFilePositionSetter method testFailureMode_MalformedTransactionForExactScn.

/**
   * Verify that a non-corrupted SCN value within a corrupted transaction causes an exception to be thrown
   * when the exact SCN is requested.
   */
@Test
public void testFailureMode_MalformedTransactionForExactScn() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedTransactionForExactScn");
    log.info("starting");
    File dir = createTempDir();
    // first SCN is hardcoded always to be 100
    // this adds a non-XML string near the end of the dbupdate for SCN 377, but the SCN value itself is clean:
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150, /* numTxns, 24 lines each */
    1250, /* numLinesPerFile */
    1, /* numLinesPerNewline */
    "\n", 0, 377, /* corrupt a txn in 3rd file */
    "", true, "kilroy");
    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;
    // with a full XML parser, this should throw an exception; with regexQuery(), it's fine:
    finder.reset();
    res = posSetter.locateFilePosition(377, finder);
    assertFilePositionResult(res, dir, 377, FilePositionResult.Status.FOUND);
    log.info(DONE_STRING);
}

Example 98 with Logger

use of org.apache.log4j.Logger in project databus by linkedin.

the class TestTrailFilePositionSetter method testFailureMode_MalformedLastTransactionInLastFile.

/**
   * Verify that corruption in the very last transaction causes USE_LATEST_SCN to return
   * the last SCN from the previous transaction.
   */
@Test
public void testFailureMode_MalformedLastTransactionInLastFile() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedLastTransactionInLastFile");
    log.info("starting");
    File dir = createTempDir();
    // first SCN is hardcoded always to be 100
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150, /* numTxns, 24 lines and 2 SCNs each */
    1250, /* numLinesPerFile */
    1, /* numLinesPerNewline */
    "\n", 0, 399, /* corrupt last SCN in 3rd file */
    "quux", false, "");
    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;
    // corruption at SCN 399 => should get 398 back (same transaction, but again, regexQuery() doesn't fully validate)
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);
    assertFilePositionResult(res, dir, 398, FilePositionResult.Status.FOUND);
    log.info(DONE_STRING);
}

Example 99 with Logger

use of org.apache.log4j.Logger in project databus by linkedin.

the class TestTrailFilePositionSetter method testScnPositionSetter2.

/**
   * Case where XML snippets have newlines at different positions.
   * NOTE:  This test is EXCEEDINGLY slow...
   *
   * @throws Exception
   */
@Test
public void testScnPositionSetter2() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionSetter2");
    log.info("starting");
    String[] newLines = { "\r", "\n", "\r\n" };
    for (String newLine : newLines) {
        // Iterate by Number of Tags per line
        for (int j = 10; j < 32; j++) {
            log.info("NumPerLine: " + j);
            File dir = createTempDir();
            createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 100, 200, j, newLine, 0, -1, "", false, "");
            log.info("Directory is: " + dir);
            TrailFilePositionSetter posSetter = null;
            //GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
            GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
            //less than minScn
            log.info("less than MinScn case started !!");
            for (long i = 0; i < 100; i++) {
                if (true)
                    break;
                posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
                //finder = new GoldenGateTransactionSCNFinder();
                finder = new GGXMLTrailTransactionFinder();
                FilePositionResult res = posSetter.getFilePosition(i, finder);
                Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status");
            }
            log.info("less than MinScn case passed !!");
            //Found Case
            for (long i = 100; i < 300; i++) {
                posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
                finder = new GGXMLTrailTransactionFinder();
                log.info("SCN:  " + i);
                FilePositionResult res = posSetter.getFilePosition(i, finder);
                log.info("For scn (" + i + "):  the result is:  " + res);
                if (i % 2 == 0)
                    assertFilePositionResult(res, dir, i + 1, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
                else
                    assertFilePositionResult(res, dir, i, FilePositionResult.Status.FOUND);
            }
            //Found Case
            FilePositionResult res = null;
            for (long i = 300; i < 320; i++) {
                posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
                //finder = new GoldenGateTransactionSCNFinder();
                finder = new GGXMLTrailTransactionFinder();
                res = posSetter.getFilePosition(i, finder);
                //log.info("For scn (" + i + "):  the result is:  "  + res);
                assertFilePositionResult(res, dir, 299, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
            }
            // USE Latest SCN (-1)
            posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
            //finder = new GoldenGateTransactionSCNFinder();
            finder = new GGXMLTrailTransactionFinder();
            res = posSetter.getFilePosition(-1, finder);
            //log.info("For scn (" + i + "):  the result is:  "  + res);
            assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND);
        }
    }
    log.info(DONE_STRING);
}

Example 100 with Logger

use of org.apache.log4j.Logger in project databus by linkedin.

the class TestTrailFilePositionSetter method testRepeatSCN.

@Test
public void testRepeatSCN() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testRepeatSCN");
    File dir = createTempDir();
    long startScn = 100;
    int numTxnsPerFile = 10;
    /**
     * Create trail files with the following SCN range
     *
     * Trail-File       Number of Txns          SCN Range
     *   x300               10                  100-100
     *   x301               10                  100-100
     *   x302               10                  101-101
     *   x303               10                  101-101
     *   x304               10                  102-102
     *   x305               10                  102-102
     *   x306               10                  103-103
     *   x307               10                  103-103
     *   x308               10                  104-104
     *   x309               10                  104-104
     *
     */
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 100L, _txnPattern.length * numTxnsPerFile, 1L, "\n", 0, _txnPattern, // Transactions in 2 files have the same SCN
    2 * numTxnsPerFile * 2, startScn);
    log.info("Directory is: " + dir);
    TrailFilePositionSetter posSetter = null;
    // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    // SCN 100 is not found because this is the first SCN in the trail file.
    FilePositionResult res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status for SCN: " + 100 + ", Result: " + res);
    // SCN 101 is found
    res = posSetter.locateFilePosition(101, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x302", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 101, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 101, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
    // SCN 102 is found
    res = posSetter.locateFilePosition(102, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x304", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 102, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 102, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
    // SCN 103 is found
    res = posSetter.locateFilePosition(103, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x306", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 103, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 103, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
    // SCN 104 is found
    res = posSetter.locateFilePosition(104, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x308", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 104, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 104, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
}