Examples with GGXMLTrailTransactionFinder com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder used on opensource projects

Example 1 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testFailureMode_MalformedTransactionForExactScn.

/**
 * Verify that a non-corrupted SCN value within a corrupted transaction causes an exception to be thrown
 * when the exact SCN is requested.
 */
@Test
public void testFailureMode_MalformedTransactionForExactScn() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedTransactionForExactScn");
    log.info("starting");
    File dir = createTempDir();
    // first SCN is hardcoded always to be 100
    // this adds a non-XML string near the end of the dbupdate for SCN 377, but the SCN value itself is clean:
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150, /* numTxns, 24 lines each */
    1250, /* numLinesPerFile */
    1, /* numLinesPerNewline */
    "\n", 0, 377, /* corrupt a txn in 3rd file */
    "", true, "kilroy");
    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;
    // with a full XML parser, this should throw an exception; with regexQuery(), it's fine:
    finder.reset();
    res = posSetter.locateFilePosition(377, finder);
    assertFilePositionResult(res, dir, 377, FilePositionResult.Status.FOUND);
    log.info(DONE_STRING);
}

Example 2 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testRepeatSCN.

@Test
public void testRepeatSCN() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testRepeatSCN");
    File dir = createTempDir();
    long startScn = 100;
    int numTxnsPerFile = 10;
    /**
     * Create trail files with the following SCN range
     *
     * Trail-File       Number of Txns          SCN Range
     *   x300               10                  100-100
     *   x301               10                  100-100
     *   x302               10                  101-101
     *   x303               10                  101-101
     *   x304               10                  102-102
     *   x305               10                  102-102
     *   x306               10                  103-103
     *   x307               10                  103-103
     *   x308               10                  104-104
     *   x309               10                  104-104
     */
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 100L, _txnPattern.length * numTxnsPerFile, 1L, "\n", 0, _txnPattern, // Transactions in 2 files have the same SCN
    2 * numTxnsPerFile * 2, startScn);
    log.info("Directory is: " + dir);
    TrailFilePositionSetter posSetter = null;
    // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    // SCN 100 is not found because this is the first SCN in the trail file.
    FilePositionResult res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status for SCN: " + 100 + ", Result: " + res);
    // SCN 101 is found
    res = posSetter.locateFilePosition(101, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x302", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 101, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 101, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
    // SCN 102 is found
    res = posSetter.locateFilePosition(102, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x304", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 102, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 102, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
    // SCN 103 is found
    res = posSetter.locateFilePosition(103, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x306", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 103, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 103, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
    // SCN 104 is found
    res = posSetter.locateFilePosition(104, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.FOUND, "Result Status");
    Assert.assertEquals(res.getTxnPos().getFile(), "x308", "File found");
    Assert.assertEquals(res.getTxnPos().getFileOffset(), 0, "File offset found");
    Assert.assertEquals(res.getTxnPos().getLineNumber(), 1, "File line number found");
    Assert.assertEquals(res.getTxnPos().getMinScn(), 104, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getMaxScn(), 104, "MinScn check");
    Assert.assertEquals(res.getTxnPos().getTxnRank(), 10, "Txn Rank");
}

Example 3 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testRequestedScnOlderThanOldestTransaction.

/**
 * Verify that requesting an SCN that's older than the oldest trail file causes ERROR
 * to be returned.  Also check various valid states:  SCN that isn't the max for its
 * transaction; SCN that is the max for its transaction; earliest SCN; etc.
 */
@Test
public void testRequestedScnOlderThanOldestTransaction() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testRequestedScnOlderThanOldestTransaction");
    // log.setLevel(Level.DEBUG);
    log.info("starting");
    File dir = createTempDir();
    // first SCN is hardcoded always to be 100:
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150, /* numTxns, 24 lines each */
    1250, /* numLinesPerFile */
    1, /* numLinesPerNewline */
    "\n", 0, -1, /* no corruption */
    "", false, "");
    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;
    // SCN 99 is before the earliest SCN present in the trail files (100), so expect ERROR:
    res = posSetter.locateFilePosition(99, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's too old.");
    // "Transaction-SCN" semantics:  max SCN is 121 in this transaction and 119 in the
    // previous one, so those are the values used for most of the "is it found?" logic.
    // Since 120 falls between the two but doesn't equal either one, it's not considered
    // to be found exactly (even though it's actually present), but the state is still
    // valid, so processing can continue (i.e., it doesn't matter).
    finder.reset();
    res = posSetter.locateFilePosition(120, finder);
    assertFilePositionResult(res, dir, 120, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's too old.");
    // Related, weird corner case:  USE_EARLIEST_SCN uses the min SCN in the oldest transaction,
    // so even though an exact match for 100 doesn't return FOUND, this does:
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 100, FilePositionResult.Status.FOUND);
    // For SCN <= the earliest transactions maxSCN, we throw error
    finder.reset();
    res = posSetter.locateFilePosition(101, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's too old.");
    log.info(DONE_STRING);
}

Example 4 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testScnPositionSetter2.

/**
 * Case where XML snippets have newlines at different positions.
 * NOTE:  This test is EXCEEDINGLY slow...
 *
 * @throws Exception
 */
@Test
public void testScnPositionSetter2() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testScnPositionSetter2");
    log.info("starting");
    String[] newLines = { "\r", "\n", "\r\n" };
    for (String newLine : newLines) {
        // Iterate by Number of Tags per line
        for (int j = 10; j < 32; j++) {
            log.info("NumPerLine: " + j);
            File dir = createTempDir();
            createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 100, 200, j, newLine, 0, -1, "", false, "");
            log.info("Directory is: " + dir);
            TrailFilePositionSetter posSetter = null;
            // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
            GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
            // less than minScn
            log.info("less than MinScn case started !!");
            for (long i = 0; i < 100; i++) {
                if (true)
                    break;
                posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
                // finder = new GoldenGateTransactionSCNFinder();
                finder = new GGXMLTrailTransactionFinder();
                FilePositionResult res = posSetter.getFilePosition(i, finder);
                Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status");
            }
            log.info("less than MinScn case passed !!");
            // Found Case
            for (long i = 100; i < 300; i++) {
                posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
                finder = new GGXMLTrailTransactionFinder();
                log.info("SCN:  " + i);
                FilePositionResult res = posSetter.getFilePosition(i, finder);
                log.info("For scn (" + i + "):  the result is:  " + res);
                if (i % 2 == 0)
                    assertFilePositionResult(res, dir, i + 1, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
                else
                    assertFilePositionResult(res, dir, i, FilePositionResult.Status.FOUND);
            }
            // Found Case
            FilePositionResult res = null;
            for (long i = 300; i < 320; i++) {
                posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
                // finder = new GoldenGateTransactionSCNFinder();
                finder = new GGXMLTrailTransactionFinder();
                res = posSetter.getFilePosition(i, finder);
                // log.info("For scn (" + i + "):  the result is:  "  + res);
                assertFilePositionResult(res, dir, 299, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
            }
            // USE Latest SCN (-1)
            posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
            // finder = new GoldenGateTransactionSCNFinder();
            finder = new GGXMLTrailTransactionFinder();
            res = posSetter.getFilePosition(-1, finder);
            // log.info("For scn (" + i + "):  the result is:  "  + res);
            assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND);
        }
    }
    log.info(DONE_STRING);
}

Example 5 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testFailureMode_MalformedLastTransactionInLastFile.

/**
 * Verify that corruption in the very last transaction causes USE_LATEST_SCN to return
 * the last SCN from the previous transaction.
 */
@Test
public void testFailureMode_MalformedLastTransactionInLastFile() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedLastTransactionInLastFile");
    log.info("starting");
    File dir = createTempDir();
    // first SCN is hardcoded always to be 100
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150, /* numTxns, 24 lines and 2 SCNs each */
    1250, /* numLinesPerFile */
    1, /* numLinesPerNewline */
    "\n", 0, 399, /* corrupt last SCN in 3rd file */
    "quux", false, "");
    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;
    // corruption at SCN 399 => should get 398 back (same transaction, but again, regexQuery() doesn't fully validate)
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_LATEST_SCN, finder);
    assertFilePositionResult(res, dir, 398, FilePositionResult.Status.FOUND);
    log.info(DONE_STRING);
}