Examples with Repository org.pentaho.di.repository.Repository used on opensource projects

Example 96 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class PanCommandExecutor method execute.

public int execute(String repoName, String noRepo, String username, String trustUser, String password, String dirName, String filename, String jarFile, String transName, String listTrans, String listDirs, String exportRepo, String initialDir, String listRepos, String safemode, String metrics, String listParams, NamedParams params, String[] arguments) throws Throwable {
    getLog().logMinimal(BaseMessages.getString(getPkgClazz(), "Pan.Log.StartingToRun"));
    // capture execution start time
    Date start = Calendar.getInstance().getTime();
    logDebug("Pan.Log.AllocatteNewTrans");
    Trans trans = null;
    // In case we use a repository...
    Repository repository = null;
    try {
        if (getMetaStore() == null) {
            setMetaStore(createDefaultMetastore());
        }
        logDebug("Pan.Log.StartingToLookOptions");
        // Read kettle transformation specified
        if (!Utils.isEmpty(repoName) || !Utils.isEmpty(filename) || !Utils.isEmpty(jarFile)) {
            logDebug("Pan.Log.ParsingCommandline");
            if (!Utils.isEmpty(repoName) && !YES.equalsIgnoreCase(noRepo)) {
                /**
                 * if set, _trust_user_ needs to be considered. See pur-plugin's:
                 *
                 * @link https://github.com/pentaho/pentaho-kettle/blob/8.0.0.0-R/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/PurRepositoryConnector.java#L97-L101
                 * @link https://github.com/pentaho/pentaho-kettle/blob/8.0.0.0-R/plugins/pur/core/src/main/java/org/pentaho/di/repository/pur/WebServiceManager.java#L130-L133
                 */
                if (YES.equalsIgnoreCase(trustUser)) {
                    System.setProperty("pentaho.repository.client.attemptTrust", YES);
                }
                // In case we use a repository...
                // some commands are to load a Trans from the repo; others are merely to print some repo-related information
                RepositoryMeta repositoryMeta = loadRepositoryConnection(repoName, "Pan.Log.LoadingAvailableRep", "Pan.Error.NoRepsDefined", "Pan.Log.FindingRep");
                repository = establishRepositoryConnection(repositoryMeta, username, password, RepositoryOperation.EXECUTE_TRANSFORMATION);
                trans = executeRepositoryBasedCommand(repository, repositoryMeta, dirName, transName, listTrans, listDirs, exportRepo);
            }
            // You could implement some fail-over mechanism this way.
            if (trans == null) {
                trans = executeFilesystemBasedCommand(initialDir, filename, jarFile);
            }
        }
        if (YES.equalsIgnoreCase(listRepos)) {
            // list the repositories placed at repositories.xml
            printRepositories(loadRepositoryInfo("Pan.Log.LoadingAvailableRep", "Pan.Error.NoRepsDefined"));
        }
    } catch (Exception e) {
        trans = null;
        System.out.println(BaseMessages.getString(getPkgClazz(), "Pan.Error.ProcessStopError", e.getMessage()));
        e.printStackTrace();
        if (repository != null) {
            repository.disconnect();
        }
        return CommandExecutorCodes.Pan.ERRORS_DURING_PROCESSING.getCode();
    }
    if (trans == null) {
        if (!YES.equalsIgnoreCase(listTrans) && !YES.equalsIgnoreCase(listDirs) && !YES.equalsIgnoreCase(listRepos) && Utils.isEmpty(exportRepo)) {
            System.out.println(BaseMessages.getString(getPkgClazz(), "Pan.Error.CanNotLoadTrans"));
            return CommandExecutorCodes.Pan.COULD_NOT_LOAD_TRANS.getCode();
        } else {
            return CommandExecutorCodes.Pan.SUCCESS.getCode();
        }
    }
    try {
        trans.setLogLevel(getLog().getLogLevel());
        configureParameters(trans, params, trans.getTransMeta());
        // run in safe mode if requested
        trans.setSafeModeEnabled(YES.equalsIgnoreCase(safemode));
        // enable kettle metric gathering if requested
        trans.setGatheringMetrics(YES.equalsIgnoreCase(metrics));
        // List the parameters defined in this transformation, and then simply exit
        if (YES.equalsIgnoreCase(listParams)) {
            printTransformationParameters(trans);
            // same as the other list options
            return CommandExecutorCodes.Pan.COULD_NOT_LOAD_TRANS.getCode();
        }
        // allocate & run the required sub-threads
        try {
            trans.execute(arguments);
        } catch (KettleException ke) {
            logDebug(ke.getLocalizedMessage());
            System.out.println(BaseMessages.getString(getPkgClazz(), "Pan.Error.UnablePrepareInitTrans"));
            return CommandExecutorCodes.Pan.UNABLE_TO_PREP_INIT_TRANS.getCode();
        }
        // Give the transformation up to 10 seconds to finish execution
        waitUntilFinished(trans, 100);
        if (trans.isRunning()) {
            getLog().logError(BaseMessages.getString(getPkgClazz(), "Pan.Log.NotStopping"));
        }
        getLog().logMinimal(BaseMessages.getString(getPkgClazz(), "Pan.Log.Finished"));
        // capture execution stop time
        Date stop = Calendar.getInstance().getTime();
        int completionTimeSeconds = calculateAndPrintElapsedTime(start, stop, "Pan.Log.StartStop", "Pan.Log.ProcessingEndAfter", "Pan.Log.ProcessingEndAfterLong", "Pan.Log.ProcessingEndAfterLonger", "Pan.Log.ProcessingEndAfterLongest");
        if (trans.getResult().getNrErrors() == 0) {
            trans.printStats(completionTimeSeconds);
            return CommandExecutorCodes.Pan.SUCCESS.getCode();
        } else {
            String transJVMExitCode = trans.getVariable(Const.KETTLE_TRANS_PAN_JVM_EXIT_CODE);
            // If the trans has a return code to return to the OS, then we exit with that
            if (!Utils.isEmpty(transJVMExitCode)) {
                try {
                    return Integer.parseInt(transJVMExitCode);
                } catch (NumberFormatException nfe) {
                    getLog().logError(BaseMessages.getString(getPkgClazz(), "Pan.Error.TransJVMExitCodeInvalid", Const.KETTLE_TRANS_PAN_JVM_EXIT_CODE, transJVMExitCode));
                    getLog().logError(BaseMessages.getString(getPkgClazz(), "Pan.Log.JVMExitCode", "1"));
                    return CommandExecutorCodes.Pan.ERRORS_DURING_PROCESSING.getCode();
                }
            } else {
                // the trans does not have a return code.
                return CommandExecutorCodes.Pan.ERRORS_DURING_PROCESSING.getCode();
            }
        }
    } catch (KettleException ke) {
        System.out.println(BaseMessages.getString(getPkgClazz(), "Pan.Log.ErrorOccurred", "" + ke.getMessage()));
        getLog().logError(BaseMessages.getString(getPkgClazz(), "Pan.Log.UnexpectedErrorOccurred", "" + ke.getMessage()));
        return CommandExecutorCodes.Pan.UNEXPECTED_ERROR.getCode();
    } finally {
        if (repository != null) {
            repository.disconnect();
        }
        if (YES.equalsIgnoreCase(trustUser)) {
            // we set it, now we sanitize it
            System.clearProperty("pentaho.repository.client.attemptTrust");
        }
    }
}

Example 97 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class JobExecutionConfiguration method connectRepository.

public Repository connectRepository(RepositoriesMeta repositoriesMeta, String repositoryName, String username, String password) throws KettleException {
    RepositoryMeta repositoryMeta = repositoriesMeta.findRepository(repositoryName);
    if (repositoryMeta == null) {
        log.logBasic("I couldn't find the repository with name '" + repositoryName + "'");
        return null;
    }
    Repository rep = PluginRegistry.getInstance().loadClass(RepositoryPluginType.class, repositoryMeta, Repository.class);
    rep.init(repositoryMeta);
    try {
        rep.connect(username, password);
        log.logBasic("Connected to " + repositoryName + " as " + username);
        setRepository(rep);
        return rep;
    } catch (Exception e) {
        log.logBasic("Unable to connect to the repository with name '" + repositoryName + "'");
        return null;
    }
}

Example 98 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class StepWithMappingMetaTest method loadMappingMetaTest.

@Test
@PrepareForTest(StepWithMappingMeta.class)
public void loadMappingMetaTest() throws Exception {
    String childParam = "childParam";
    String childValue = "childValue";
    String paramOverwrite = "paramOverwrite";
    String parentParam = "parentParam";
    String parentValue = "parentValue";
    String variablePath = "Internal.Entry.Current.Directory";
    String virtualDir = "/testFolder/CDA-91";
    String fileName = "testTrans.ktr";
    VariableSpace variables = new Variables();
    variables.setVariable(parentParam, parentValue);
    variables.setVariable(paramOverwrite, parentValue);
    StepMeta stepMeta = new StepMeta();
    TransMeta parentTransMeta = new TransMeta();
    stepMeta.setParentTransMeta(parentTransMeta);
    RepositoryDirectoryInterface repositoryDirectory = Mockito.mock(RepositoryDirectoryInterface.class);
    when(repositoryDirectory.toString()).thenReturn(virtualDir);
    stepMeta.getParentTransMeta().setRepositoryDirectory(repositoryDirectory);
    StepWithMappingMeta mappingMetaMock = mock(StepWithMappingMeta.class);
    when(mappingMetaMock.getSpecificationMethod()).thenReturn(ObjectLocationSpecificationMethod.FILENAME);
    when(mappingMetaMock.getFileName()).thenReturn("${" + variablePath + "}/" + fileName);
    when(mappingMetaMock.getParentStepMeta()).thenReturn(stepMeta);
    Repository rep = mock(Repository.class);
    Mockito.doReturn(Mockito.mock(RepositoryDirectoryInterface.class)).when(rep).findDirectory(anyString());
    TransMeta child = new TransMeta();
    child.setVariable(childParam, childValue);
    child.setVariable(paramOverwrite, childValue);
    Mockito.doReturn(child).when(rep).loadTransformation(anyString(), any(), any(), anyBoolean(), any());
    TransMeta transMeta = StepWithMappingMeta.loadMappingMeta(mappingMetaMock, rep, null, variables, true);
    Assert.assertNotNull(transMeta);
    // When the child parameter does exist in the parent parameters, overwrite the child parameter by the parent parameter.
    Assert.assertEquals(parentValue, transMeta.getVariable(paramOverwrite));
    // When the child parameter does not exist in the parent parameters, keep it.
    Assert.assertEquals(childValue, transMeta.getVariable(childParam));
    // All other parent parameters need to get copied into the child parameters  (when the 'Inherit all
    // variables from the transformation?' option is checked)
    Assert.assertEquals(parentValue, transMeta.getVariable(parentParam));
}

Example 99 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class TransMetaTest method testTransWithOneStepIsConsideredUsed.

@Test
public void testTransWithOneStepIsConsideredUsed() throws Exception {
    TransMeta transMeta = new TransMeta(getClass().getResource("one-step-trans.ktr").getPath());
    assertEquals(1, transMeta.getUsedSteps().size());
    Repository rep = mock(Repository.class);
    ProgressMonitorListener monitor = mock(ProgressMonitorListener.class);
    List<CheckResultInterface> remarks = new ArrayList<>();
    IMetaStore metaStore = mock(IMetaStore.class);
    transMeta.checkSteps(remarks, false, monitor, new Variables(), rep, metaStore);
    assertEquals(4, remarks.size());
    for (CheckResultInterface remark : remarks) {
        assertEquals(CheckResultInterface.TYPE_RESULT_OK, remark.getType());
    }
}

Example 100 with Repository

use of org.pentaho.di.repository.Repository in project pentaho-kettle by pentaho.

the class TransMetaTest method testLoadXml.

@Test
public void testLoadXml() throws KettleException {
    String directory = "/home/admin";
    Node jobNode = Mockito.mock(Node.class);
    NodeList nodeList = new NodeList() {

        ArrayList<Node> nodes = new ArrayList<>();

        {
            Node nodeInfo = Mockito.mock(Node.class);
            Mockito.when(nodeInfo.getNodeName()).thenReturn(TransMeta.XML_TAG_INFO);
            Mockito.when(nodeInfo.getChildNodes()).thenReturn(this);
            Node nodeDirectory = Mockito.mock(Node.class);
            Mockito.when(nodeDirectory.getNodeName()).thenReturn("directory");
            Node child = Mockito.mock(Node.class);
            Mockito.when(nodeDirectory.getFirstChild()).thenReturn(child);
            Mockito.when(child.getNodeValue()).thenReturn(directory);
            nodes.add(nodeDirectory);
            nodes.add(nodeInfo);
        }

        @Override
        public Node item(int index) {
            return nodes.get(index);
        }

        @Override
        public int getLength() {
            return nodes.size();
        }
    };
    Mockito.when(jobNode.getChildNodes()).thenReturn(nodeList);
    Repository rep = Mockito.mock(Repository.class);
    RepositoryDirectory repDirectory = new RepositoryDirectory(new RepositoryDirectory(new RepositoryDirectory(), "home"), "admin");
    Mockito.when(rep.findDirectory(Mockito.eq(directory))).thenReturn(repDirectory);
    TransMeta meta = new TransMeta();
    VariableSpace variableSpace = Mockito.mock(VariableSpace.class);
    Mockito.when(variableSpace.listVariables()).thenReturn(new String[0]);
    meta.loadXML(jobNode, null, Mockito.mock(IMetaStore.class), rep, false, variableSpace, Mockito.mock(OverwritePrompter.class));
    meta.setInternalKettleVariables(null);
    assertEquals(repDirectory.getPath(), meta.getVariable(Const.INTERNAL_VARIABLE_TRANSFORMATION_REPOSITORY_DIRECTORY));
}