Examples with GGXMLTrailTransactionFinder com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder used on opensource projects

Example 6 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testFailureMode_MalformedLastTransactionInMiddleFile.

/**
 * Verify that corruption that occurs after the desired SCN/transaction doesn't cause errors
 * when requesting an exact SCN.
 */
@Test
public void testFailureMode_MalformedLastTransactionInMiddleFile() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedLastTransactionInMiddleFile");
    log.info("starting");
    File dir = createTempDir();
    // first SCN is hardcoded always to be 100
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150, /* numTxns, 24 lines each */
    1250, /* numLinesPerFile */
    1, /* numLinesPerNewline */
    "\n", 0, 307, /* corrupt last SCN in 2nd file */
    "plugh", false, "");
    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;
    // corruption at SCN 307 occurs after SCN 299, so latter should be found OK:
    res = posSetter.locateFilePosition(299, finder);
    assertFilePositionResult(res, dir, 299, FilePositionResult.Status.FOUND);
    // SCN 306 is in same transaction as 307, but regexQuery() doesn't fully validate XML => OK
    finder.reset();
    res = posSetter.locateFilePosition(306, finder);
    assertFilePositionResult(res, dir, 306, FilePositionResult.Status.FOUND);
    log.info(DONE_STRING);
}

Example 7 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class GoldenGateEventProducer method locateScnInTrailFile.

/**
 * Given an xml directory and prefix, the method identifies the file which has the scn (_scn from event producer class)
 * and returns an inputstream reader pointing to the scn location. If the scn is not found:
 * 1. If scn less than what is present in the trail file directory (minimum) - throws a fatal exception.
 * 2. If exact scn is not found, but it's greater than the minimum scn in the trail file directory, it returns the closest scn greater than _scn (from the event producer class).
 * This methods reads and modifies the _scn from the event producer class.
 * @param xmlDir The directory where the trail files are located
 * @param xmlPrefix The prefix of the xml trail files, eg. x4
 * @return
 * @throws IOException
 * @throws DatabusException
 */
private ConcurrentAppendableCompositeFileInputStream locateScnInTrailFile(String xmlDir, String xmlPrefix) throws Exception {
    ConcurrentAppendableCompositeFileInputStream compositeInputStream = null;
    TrailFilePositionSetter.FilePositionResult filePositionResult = null;
    TrailFilePositionSetter trailFilePositionSetter = null;
    while (compositeInputStream == null) {
        _log.info("Requesting trail file position setter for scn: " + _scn.get());
        trailFilePositionSetter = new TrailFilePositionSetter(xmlDir, xmlPrefix, getName());
        filePositionResult = trailFilePositionSetter.locateFilePosition(_scn.get(), new GGXMLTrailTransactionFinder());
        _log.info("File position at : " + filePositionResult);
        switch(filePositionResult.getStatus()) {
            case ERROR:
                _log.fatal("Unable to locate the scn in the trail file.");
                throw new DatabusException("Unable to find the given scn " + _scn.get() + " in the trail files");
            case NO_TXNS_FOUND:
                // If the latest scn is not found in the trail files, then use the earliest scn.
                if (_scn.get() == TrailFilePositionSetter.USE_LATEST_SCN) {
                    _log.info("Switching from USE_LATEST_SCN to USE_EARLIEST_SCN because no trail files were not found");
                    _scn.set(TrailFilePositionSetter.USE_EARLIEST_SCN);
                }
                // TODO sleep get configuration for sleep time
                long noTxnsFoundSleepTime = 500;
                _log.info("NO_TXNS_FOUND, sleeping for " + noTxnsFoundSleepTime + " ms before retrying");
                Thread.sleep(noTxnsFoundSleepTime);
                break;
            case EXACT_SCN_NOT_FOUND:
                {
                    _log.info("Exact SCN was not found, the closest scn found was: " + filePositionResult.getTxnPos().getMinScn());
                    compositeInputStream = new ConcurrentAppendableCompositeFileInputStream(xmlDir, filePositionResult.getTxnPos().getFile(), filePositionResult.getTxnPos().getFileOffset(), new TrailFilePositionSetter.FileFilter(new File(xmlDir), xmlPrefix), false);
                    long foundScn = filePositionResult.getTxnPos().getMaxScn();
                    /**
                     * If exact scn is not found, the trail file position setter returns the next immediate available scn, i.e., the contract guarantees
                     * a scn always greater than the given scn (foundscn > _scn). We use the _scn (requested scn to be found) as the prevScn to start the event buffer.
                     * And the scn found as the current scn(first event in the relay).
                     */
                    if (foundScn <= _scn.get())
                        throw new DatabusException("EXACT_SCN_NOT_FOUND, but foundScn is <= _scn ");
                    _startPrevScn.set(_scn.get());
                    _log.info("Changing current scn from " + _scn.get() + " to " + foundScn);
                    _log.info("Planning to use prevScn " + _startPrevScn);
                    _scn.set(foundScn);
                    break;
                }
            case FOUND:
                {
                    _log.info("Exact SCN was  found" + filePositionResult.getTxnPos().getMaxScn());
                    compositeInputStream = new ConcurrentAppendableCompositeFileInputStream(xmlDir, filePositionResult.getTxnPos().getFile(), filePositionResult.getTxnPos().getFileOffset(), new TrailFilePositionSetter.FileFilter(new File(xmlDir), xmlPrefix), false);
                    /**
                     * The trail file position setter returns FOUND in two cases:
                     * 1. MaxScn was given as input.
                     * 2. Earliest or Latest scn was given as input.
                     * For both the cases, we set the prevScn to the foundScn-1 and the foundScn as the currentScn.
                     */
                    long foundScn = filePositionResult.getTxnPos().getMaxScn();
                    // Assert that if maxScn was requested, the trail file position setter has returned the exact scn (It has returned FOUND).
                    if (_scn.get() >= 0 && _scn.get() != foundScn) {
                        throw new DatabusException("The exact scn was not found, but the trail file position setter has returned FOUND!");
                    }
                    _startPrevScn.set(foundScn - 1);
                    _scn.set(foundScn);
                    break;
                }
            default:
                throw new DatabusException("Unhandled file position result in switch case, terminating producer.");
        }
    }
    if (filePositionResult == null) {
        _log.info(trailFilePositionSetter);
        throw new DatabusException("file position Result returned by TrailFilePositionSetter is null!");
    }
    if (_scn.get() <= 0) {
        _log.info("The scn is <=0, using scn from file position setter:" + filePositionResult);
        _scn.set(filePositionResult.getTxnPos().getMaxScn());
    }
    return compositeInputStream;
}

Example 8 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testScnPositionLocator.

private void testScnPositionLocator(int startLine, int startScn, int endScn, int beginFoundScn, Logger log) throws Exception {
    File dir = createTempDir();
    // TODO/FIXME:  why are startScn and endScn being used for numTxns and numLinesPerFile, respectively?
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, startScn, endScn, 1, "\n", startLine, -1, "", false, "");
    log.info("Directory is: " + dir);
    TrailFilePositionSetter posSetter = null;
    // GoldenGateTransactionSCNFinder finder;
    GGXMLTrailTransactionFinder finder;
    // less than minScn
    for (long i = 0; i < beginFoundScn; i++) {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        // finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();
        FilePositionResult res = posSetter.locateFilePosition(i, finder);
        Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status for SCN: " + i + ", Result: " + res);
    }
    // Found Case
    for (long i = beginFoundScn; i < (startScn + endScn); i++) {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        // finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();
        FilePositionResult res = posSetter.locateFilePosition(i, finder);
        log.info("For scn (" + i + "):  the result is:  " + res);
        if (i % 2 == 0)
            assertFilePositionResult(res, dir, i + 1, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
        else
            assertFilePositionResult(res, dir, i, FilePositionResult.Status.FOUND);
    }
    // Found Case
    for (long i = (startScn + endScn); i < (startScn + endScn) + 20; i++) {
        posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
        // finder = new GoldenGateTransactionSCNFinder();
        finder = new GGXMLTrailTransactionFinder();
        FilePositionResult res = posSetter.locateFilePosition(i, finder);
        log.info("For scn (" + i + "):  the result is:  " + res);
        assertFilePositionResult(res, dir, 299, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
    }
}

Example 9 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testFailureMode_MalformedFirstTransactionInFirstFile.

/**
 * Verify that corruption that occurs early in the file causes USE_EARLIEST_SCN to return the
 * first SCN of the first uncorrupted transaction.
 */
@Test
public void testFailureMode_MalformedFirstTransactionInFirstFile() throws Exception {
    final Logger log = Logger.getLogger("TestTrailFilePositionSetter.testFailureMode_MalformedFirstTransactionInFirstFile");
    log.info("starting");
    File dir = createTempDir();
    // first SCN is hardcoded always to be 100
    createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, 150, /* numTxns, 24 lines each */
    1250, /* numLinesPerFile */
    1, /* numLinesPerNewline */
    "\n", 0, 100, /* corrupt first SCN */
    "xyzzy", false, "");
    TrailFilePositionSetter posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
    GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
    FilePositionResult res;
    // SCN 100 is corrupted, so 101 is the effective oldest SCN => 100 treated as error:
    res = posSetter.locateFilePosition(100, finder);
    Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "expected error for exact-match SCN that's corrupted and oldest in all trail files.");
    // SCN 101 is OK (regexQuery() doesn't fully validate XML):
    finder.reset();
    res = posSetter.locateFilePosition(TrailFilePositionSetter.USE_EARLIEST_SCN, finder);
    assertFilePositionResult(res, dir, 101, FilePositionResult.Status.FOUND);
    log.info(DONE_STRING);
}

Example 10 with GGXMLTrailTransactionFinder

use of com.linkedin.databus2.producers.db.GGXMLTrailTransactionFinder in project databus by linkedin.

the class TestTrailFilePositionSetter method testScnPositionSetter.

private void testScnPositionSetter(int startLine, int startScn, int endScn, int beginFoundScn, Logger log) throws Exception {
    String[] newLines = { "\n", "\r", "\r\n" };
    for (String newLine : newLines) {
        File dir = createTempDir();
        createTrailFiles(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX, startScn, endScn, 1, newLine, startLine, -1, "", false, "");
        log.info("Directory is: " + dir);
        TrailFilePositionSetter posSetter = null;
        // GoldenGateTransactionSCNFinder finder = new GoldenGateTransactionSCNFinder();
        GGXMLTrailTransactionFinder finder = new GGXMLTrailTransactionFinder();
        // less than minScn
        for (long i = 0; i < beginFoundScn; i++) {
            posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
            Logger log2 = posSetter._log;
            log2.setLevel(Level.INFO);
            log2.info("Created a TrailFilePositionSetter with suffix x3");
            // finder = new GoldenGateTransactionSCNFinder();
            finder = new GGXMLTrailTransactionFinder();
            FilePositionResult res = posSetter.getFilePosition(i, finder);
            Assert.assertEquals(res.getStatus(), FilePositionResult.Status.ERROR, "Result Status for SCN: " + i + ", Result: " + res);
        }
        // Found Case
        for (long i = beginFoundScn; i < (startScn + endScn); i++) {
            posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
            // finder = new GoldenGateTransactionSCNFinder();
            finder = new GGXMLTrailTransactionFinder();
            FilePositionResult res = posSetter.getFilePosition(i, finder);
            log.info("For scn (" + i + "):  the result is:  " + res);
            if (i % 2 == 0)
                assertFilePositionResult(res, dir, i + 1, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
            else
                assertFilePositionResult(res, dir, i, FilePositionResult.Status.FOUND);
        }
        // Found Case
        for (long i = (startScn + endScn); i < (startScn + endScn) + 20; i++) {
            posSetter = new TrailFilePositionSetter(dir.getAbsolutePath(), TRAIL_FILENAME_PREFIX);
            // finder = new GoldenGateTransactionSCNFinder();
            finder = new GGXMLTrailTransactionFinder();
            FilePositionResult res = posSetter.getFilePosition(i, finder);
            log.info("For scn (" + i + "):  the result is:  " + res);
            assertFilePositionResult(res, dir, 299, FilePositionResult.Status.EXACT_SCN_NOT_FOUND);
        }
    }
}