Examples with PerformanceLogTable org.pentaho.di.core.logging.PerformanceLogTable used on opensource projects

Example 1 with PerformanceLogTable

use of org.pentaho.di.core.logging.PerformanceLogTable in project pentaho-kettle by pentaho.

the class Trans method writeStepPerformanceLogRecords.

/**
 * Write step performance log records.
 *
 * @param startSequenceNr the start sequence numberr
 * @param status          the logging status. If this is End, perform cleanup
 * @return the new sequence number
 * @throws KettleException if any errors occur during logging
 */
private int writeStepPerformanceLogRecords(int startSequenceNr, LogStatus status) throws KettleException {
    int lastSeqNr = 0;
    Database ldb = null;
    PerformanceLogTable performanceLogTable = transMeta.getPerformanceLogTable();
    if (!performanceLogTable.isDefined() || !transMeta.isCapturingStepPerformanceSnapShots() || stepPerformanceSnapShots == null || stepPerformanceSnapShots.isEmpty()) {
        // nothing to do here!
        return 0;
    }
    try {
        ldb = new Database(this, performanceLogTable.getDatabaseMeta());
        ldb.shareVariablesWith(this);
        ldb.connect();
        ldb.setCommit(logCommitSize);
        // Write to the step performance log table...
        // 
        RowMetaInterface rowMeta = performanceLogTable.getLogRecord(LogStatus.START, null, null).getRowMeta();
        ldb.prepareInsert(rowMeta, performanceLogTable.getActualSchemaName(), performanceLogTable.getActualTableName());
        synchronized (stepPerformanceSnapShots) {
            Iterator<List<StepPerformanceSnapShot>> iterator = stepPerformanceSnapShots.values().iterator();
            while (iterator.hasNext()) {
                List<StepPerformanceSnapShot> snapshots = iterator.next();
                synchronized (snapshots) {
                    Iterator<StepPerformanceSnapShot> snapshotsIterator = snapshots.iterator();
                    while (snapshotsIterator.hasNext()) {
                        StepPerformanceSnapShot snapshot = snapshotsIterator.next();
                        if (snapshot.getSeqNr() >= startSequenceNr && snapshot.getSeqNr() <= lastStepPerformanceSnapshotSeqNrAdded) {
                            RowMetaAndData row = performanceLogTable.getLogRecord(LogStatus.START, snapshot, null);
                            ldb.setValuesInsert(row.getRowMeta(), row.getData());
                            ldb.insertRow(true);
                        }
                        lastSeqNr = snapshot.getSeqNr();
                    }
                }
            }
        }
        ldb.insertFinished(true);
        // 
        if (status.equals(LogStatus.END)) {
            ldb.cleanupLogRecords(performanceLogTable, getName());
        }
    } catch (Exception e) {
        throw new KettleException(BaseMessages.getString(PKG, "Trans.Exception.ErrorWritingStepPerformanceLogRecordToTable"), e);
    } finally {
        if (ldb != null) {
            ldb.disconnect();
        }
    }
    return lastSeqNr + 1;
}

Example 2 with PerformanceLogTable

use of org.pentaho.di.core.logging.PerformanceLogTable in project pentaho-kettle by pentaho.

the class SpoonExportXmlTest method savingTransToXmlNotChangesLogTables.

@Test
public void savingTransToXmlNotChangesLogTables() {
    TransMeta transMeta = new TransMeta();
    initTables(transMeta);
    TransLogTable originTransLogTable = transMeta.getTransLogTable();
    StepLogTable originStepLogTable = transMeta.getStepLogTable();
    PerformanceLogTable originPerformanceLogTable = transMeta.getPerformanceLogTable();
    ChannelLogTable originChannelLogTable = transMeta.getChannelLogTable();
    MetricsLogTable originMetricsLogTable = transMeta.getMetricsLogTable();
    when(spoon.getActiveTransformation()).thenReturn(transMeta);
    when(spoon.saveXMLFile(any(TransMeta.class), anyBoolean())).thenReturn(true);
    when(spoon.saveXMLFile(anyBoolean())).thenCallRealMethod();
    spoon.saveXMLFile(true);
    tablesCommonValuesEqual(originTransLogTable, transMeta.getTransLogTable());
    assertEquals(originTransLogTable.getLogInterval(), transMeta.getTransLogTable().getLogInterval());
    assertEquals(originTransLogTable.getLogSizeLimit(), transMeta.getTransLogTable().getLogSizeLimit());
    tablesCommonValuesEqual(originStepLogTable, transMeta.getStepLogTable());
    tablesCommonValuesEqual(originPerformanceLogTable, transMeta.getPerformanceLogTable());
    assertEquals(originPerformanceLogTable.getLogInterval(), transMeta.getPerformanceLogTable().getLogInterval());
    tablesCommonValuesEqual(originChannelLogTable, transMeta.getChannelLogTable());
    tablesCommonValuesEqual(originMetricsLogTable, transMeta.getMetricsLogTable());
}

Example 3 with PerformanceLogTable

use of org.pentaho.di.core.logging.PerformanceLogTable in project pentaho-kettle by pentaho.

the class Spoon method saveTransAsXmlFile.

private boolean saveTransAsXmlFile(TransMeta transMeta, boolean export) {
    TransLogTable origTransLogTable = transMeta.getTransLogTable();
    StepLogTable origStepLogTable = transMeta.getStepLogTable();
    PerformanceLogTable origPerformanceLogTable = transMeta.getPerformanceLogTable();
    ChannelLogTable origChannelLogTable = transMeta.getChannelLogTable();
    MetricsLogTable origMetricsLogTable = transMeta.getMetricsLogTable();
    try {
        XmlExportHelper.swapTables(transMeta);
        return saveXMLFile(transMeta, export);
    } finally {
        transMeta.setTransLogTable(origTransLogTable);
        transMeta.setStepLogTable(origStepLogTable);
        transMeta.setPerformanceLogTable(origPerformanceLogTable);
        transMeta.setChannelLogTable(origChannelLogTable);
        transMeta.setMetricsLogTable(origMetricsLogTable);
    }
}

Example 4 with PerformanceLogTable

use of org.pentaho.di.core.logging.PerformanceLogTable in project pentaho-kettle by pentaho.

the class RepositoryTestBase method createTransMeta.

protected TransMeta createTransMeta(final String dbName) throws Exception {
    RepositoryDirectoryInterface rootDir = loadStartDirectory();
    TransMeta transMeta = new TransMeta();
    transMeta.setName(EXP_TRANS_NAME.concat(dbName));
    transMeta.setDescription(EXP_TRANS_DESC);
    transMeta.setExtendedDescription(EXP_TRANS_EXTENDED_DESC);
    transMeta.setRepositoryDirectory(rootDir.findDirectory(DIR_TRANSFORMATIONS));
    transMeta.setTransversion(EXP_TRANS_VERSION);
    transMeta.setTransstatus(EXP_TRANS_STATUS);
    transMeta.setCreatedUser(EXP_TRANS_CREATED_USER);
    transMeta.setCreatedDate(EXP_TRANS_CREATED_DATE);
    transMeta.setModifiedUser(EXP_TRANS_MOD_USER);
    transMeta.setModifiedDate(EXP_TRANS_MOD_DATE);
    transMeta.addParameterDefinition(EXP_TRANS_PARAM_1_NAME, EXP_TRANS_PARAM_1_DEF, EXP_TRANS_PARAM_1_DESC);
    // TODO mlowery other transLogTable fields could be set for testing here
    TransLogTable transLogTable = TransLogTable.getDefault(transMeta, transMeta, new ArrayList<StepMeta>(0));
    transLogTable.setConnectionName(EXP_TRANS_LOG_TABLE_CONN_NAME);
    transLogTable.setLogInterval(EXP_TRANS_LOG_TABLE_INTERVAL);
    transLogTable.setSchemaName(EXP_TRANS_LOG_TABLE_SCHEMA_NAME);
    transLogTable.setLogSizeLimit(EXP_TRANS_LOG_TABLE_SIZE_LIMIT);
    transLogTable.setTableName(EXP_TRANS_LOG_TABLE_TABLE_NAME);
    transLogTable.setTimeoutInDays(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS);
    transMeta.setTransLogTable(transLogTable);
    // TODO mlowery other perfLogTable fields could be set for testing here
    PerformanceLogTable perfLogTable = PerformanceLogTable.getDefault(transMeta, transMeta);
    perfLogTable.setConnectionName(EXP_TRANS_LOG_TABLE_CONN_NAME);
    perfLogTable.setLogInterval(EXP_TRANS_LOG_TABLE_INTERVAL);
    perfLogTable.setSchemaName(EXP_TRANS_LOG_TABLE_SCHEMA_NAME);
    perfLogTable.setTableName(EXP_TRANS_LOG_TABLE_TABLE_NAME);
    perfLogTable.setTimeoutInDays(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS);
    transMeta.setPerformanceLogTable(perfLogTable);
    // TODO mlowery other channelLogTable fields could be set for testing here
    ChannelLogTable channelLogTable = ChannelLogTable.getDefault(transMeta, transMeta);
    channelLogTable.setConnectionName(EXP_TRANS_LOG_TABLE_CONN_NAME);
    channelLogTable.setSchemaName(EXP_TRANS_LOG_TABLE_SCHEMA_NAME);
    channelLogTable.setTableName(EXP_TRANS_LOG_TABLE_TABLE_NAME);
    channelLogTable.setTimeoutInDays(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS);
    transMeta.setChannelLogTable(channelLogTable);
    // TODO mlowery other stepLogTable fields could be set for testing here
    StepLogTable stepLogTable = StepLogTable.getDefault(transMeta, transMeta);
    stepLogTable.setConnectionName(EXP_TRANS_LOG_TABLE_CONN_NAME);
    stepLogTable.setSchemaName(EXP_TRANS_LOG_TABLE_SCHEMA_NAME);
    stepLogTable.setTableName(EXP_TRANS_LOG_TABLE_TABLE_NAME);
    stepLogTable.setTimeoutInDays(EXP_TRANS_LOG_TABLE_TIMEOUT_IN_DAYS);
    transMeta.setStepLogTable(stepLogTable);
    DatabaseMeta dbMeta = createDatabaseMeta(dbName);
    // dbMeta must be saved so that it gets an ID
    repository.save(dbMeta, VERSION_COMMENT_V1, null);
    deleteStack.push(dbMeta);
    transMeta.setMaxDateConnection(dbMeta);
    transMeta.setMaxDateTable(EXP_TRANS_MAX_DATE_TABLE);
    transMeta.setMaxDateField(EXP_TRANS_MAX_DATE_FIELD);
    transMeta.setMaxDateOffset(EXP_TRANS_MAX_DATE_OFFSET);
    transMeta.setMaxDateDifference(EXP_TRANS_MAX_DATE_DIFF);
    transMeta.setSizeRowset(EXP_TRANS_SIZE_ROWSET);
    transMeta.setSleepTimeEmpty(EXP_TRANS_SLEEP_TIME_EMPTY);
    transMeta.setSleepTimeFull(EXP_TRANS_SLEEP_TIME_FULL);
    transMeta.setUsingUniqueConnections(EXP_TRANS_USING_UNIQUE_CONN);
    transMeta.setFeedbackShown(EXP_TRANS_FEEDBACK_SHOWN);
    transMeta.setFeedbackSize(EXP_TRANS_FEEDBACK_SIZE);
    transMeta.setUsingThreadPriorityManagment(EXP_TRANS_USING_THREAD_PRIORITY_MGMT);
    transMeta.setSharedObjectsFile(EXP_TRANS_SHARED_OBJECTS_FILE);
    transMeta.setCapturingStepPerformanceSnapShots(EXP_TRANS_CAPTURE_STEP_PERF_SNAPSHOTS);
    transMeta.setStepPerformanceCapturingDelay(EXP_TRANS_STEP_PERF_CAP_DELAY);
    transMeta.addDependency(new TransDependency(dbMeta, EXP_TRANS_DEP_TABLE_NAME, EXP_TRANS_DEP_FIELD_NAME));
    DatabaseMeta stepDbMeta = createDatabaseMeta(EXP_DBMETA_NAME_STEP.concat(dbName));
    repository.save(stepDbMeta, VERSION_COMMENT_V1, null);
    deleteStack.push(stepDbMeta);
    Condition cond = new Condition();
    StepMeta step1 = createStepMeta1(transMeta, stepDbMeta, cond);
    transMeta.addStep(step1);
    StepMeta step2 = createStepMeta2(stepDbMeta, cond);
    transMeta.addStep(step2);
    transMeta.addTransHop(createTransHopMeta(step1, step2));
    SlaveServer slaveServer = createSlaveServer(dbName);
    PartitionSchema partSchema = createPartitionSchema(dbName);
    // slaveServer, partSchema must be saved so that they get IDs
    repository.save(slaveServer, VERSION_COMMENT_V1, null);
    deleteStack.push(slaveServer);
    repository.save(partSchema, VERSION_COMMENT_V1, null);
    deleteStack.push(partSchema);
    SlaveStepCopyPartitionDistribution slaveStepCopyPartitionDistribution = new SlaveStepCopyPartitionDistribution();
    slaveStepCopyPartitionDistribution.addPartition(EXP_SLAVE_NAME, EXP_PART_SCHEMA_NAME, 0);
    slaveStepCopyPartitionDistribution.setOriginalPartitionSchemas(Arrays.asList(new PartitionSchema[] { partSchema }));
    transMeta.setSlaveStepCopyPartitionDistribution(slaveStepCopyPartitionDistribution);
    transMeta.setSlaveTransformation(EXP_TRANS_SLAVE_TRANSFORMATION);
    return transMeta;
}

Example 5 with PerformanceLogTable

use of org.pentaho.di.core.logging.PerformanceLogTable in project pentaho-kettle by pentaho.

the class XmlExportHelper method swapTables.

/**
 * When exporting meta we should not export user global parameters.
 * Method makes clone for each table and deletes all global parameters.
 * We have to make clones of each table, because we don't want to change real tables content.
 *
 * @param transMeta
 *              meta, that contains log tables to be refactored before export
 */
public static void swapTables(TransMeta transMeta) {
    TransLogTable transLogTable = transMeta.getTransLogTable();
    if (transLogTable != null) {
        TransLogTable cloneTransLogTable = (TransLogTable) transLogTable.clone();
        cloneTransLogTable.setAllGlobalParametersToNull();
        transMeta.setTransLogTable(cloneTransLogTable);
    }
    StepLogTable stepLogTable = transMeta.getStepLogTable();
    if (stepLogTable != null) {
        StepLogTable cloneStepLogTable = (StepLogTable) stepLogTable.clone();
        cloneStepLogTable.setAllGlobalParametersToNull();
        transMeta.setStepLogTable(cloneStepLogTable);
    }
    PerformanceLogTable performanceLogTable = transMeta.getPerformanceLogTable();
    if (performanceLogTable != null) {
        PerformanceLogTable clonePerformanceLogTable = (PerformanceLogTable) performanceLogTable.clone();
        clonePerformanceLogTable.setAllGlobalParametersToNull();
        transMeta.setPerformanceLogTable(clonePerformanceLogTable);
    }
    ChannelLogTable channelLogTable = transMeta.getChannelLogTable();
    if (channelLogTable != null) {
        ChannelLogTable cloneChannelLogTable = (ChannelLogTable) channelLogTable.clone();
        cloneChannelLogTable.setAllGlobalParametersToNull();
        transMeta.setChannelLogTable(cloneChannelLogTable);
    }
    MetricsLogTable metricsLogTable = transMeta.getMetricsLogTable();
    if (metricsLogTable != null) {
        MetricsLogTable cloneMetricsLogTable = (MetricsLogTable) metricsLogTable.clone();
        cloneMetricsLogTable.setAllGlobalParametersToNull();
        transMeta.setMetricsLogTable(cloneMetricsLogTable);
    }
}