use of org.talend.core.service.ITransformService in project tdi-studio-se by Talend.
the class BuildJobHandlerTest method testBuildJob.
@Test
public void testBuildJob() throws Exception {
Map<ExportChoice, Object> exportChoiceMap = new HashMap<ExportChoice, Object>();
exportChoiceMap.put(ExportChoice.needLauncher, true);
exportChoiceMap.put(ExportChoice.needSystemRoutine, true);
exportChoiceMap.put(ExportChoice.needUserRoutine, true);
exportChoiceMap.put(ExportChoice.needTalendLibraries, true);
exportChoiceMap.put(ExportChoice.needJobItem, false);
exportChoiceMap.put(ExportChoice.needJobScript, true);
exportChoiceMap.put(ExportChoice.needSourceCode, true);
exportChoiceMap.put(ExportChoice.includeLibs, true);
exportChoiceMap.put(ExportChoice.includeTestSource, false);
exportChoiceMap.put(ExportChoice.executeTests, false);
exportChoiceMap.put(ExportChoice.binaries, true);
exportChoiceMap.put(ExportChoice.needContext, true);
exportChoiceMap.put(ExportChoice.contextName, "Default");
exportChoiceMap.put(ExportChoice.applyToChildren, false);
exportChoiceMap.put(ExportChoice.needLog4jLevel, false);
exportChoiceMap.put(ExportChoice.log4jLevel, null);
exportChoiceMap.put(ExportChoice.needDependencies, true);
exportChoiceMap.put(ExportChoice.needParameterValues, false);
destinationPath = ExportJobUtil.getTmpFolderPath() + "/testBuildJob.zip";
BuildJobManager.getInstance().buildJob(destinationPath, processItem, "0.1", "Default", exportChoiceMap, JobExportType.POJO, new NullProgressMonitor());
assertTrue(new File(destinationPath).exists());
ZipFile zip = null;
try {
zip = new ZipFile(destinationPath);
// jobInfo
ZipEntry jobInfoEntry = zip.getEntry("jobInfo.properties");
assertNotNull("Can't find the jobInfo.properties", jobInfoEntry);
final InputStream jobInfoStream = zip.getInputStream(jobInfoEntry);
Properties jobInfoProp = new Properties();
jobInfoProp.load(jobInfoStream);
assertEquals("testBuildJob", jobInfoProp.getProperty("job"));
assertEquals("0.1", jobInfoProp.getProperty("jobVersion"));
assertEquals("Default", jobInfoProp.getProperty("contextName"));
assertEquals("_bWyBUAYbEeapTZ0aKwL_YA", jobInfoProp.getProperty("jobId"));
assertEquals("Standard", jobInfoProp.getProperty("jobType"));
final String technicalLabel = ProjectManager.getInstance().getCurrentProject().getTechnicalLabel();
assertEquals(technicalLabel, jobInfoProp.getProperty("project"));
ZipEntry libEntry = zip.getEntry("lib");
assertNotNull("No lib folder", libEntry);
// log4j
ZipEntry log4jXmlEntry = zip.getEntry("testBuildJob/log4j.xml");
assertNotNull("No log4j.xml", log4jXmlEntry);
// shell+bat
ZipEntry batEntry = zip.getEntry("testBuildJob/testBuildJob_run.bat");
assertNotNull("No bat file", batEntry);
ZipEntry shEntry = zip.getEntry("testBuildJob/testBuildJob_run.sh");
assertNotNull("No shell file", shEntry);
ZipEntry jarEntry = zip.getEntry("testBuildJob/testbuildjob_0_1.jar");
assertNotNull("No shell file", jarEntry);
// src
ZipEntry javaEntry = zip.getEntry("testBuildJob/src/main/java/" + technicalLabel.toLowerCase() + "/testbuildjob_0_1/testBuildJob.java");
assertNotNull("No job source code file", javaEntry);
ZipEntry routinesEntry = zip.getEntry("testBuildJob/src/main/java/routines/");
assertNotNull("No routines source code files", routinesEntry);
assertTrue(routinesEntry.isDirectory());
ZipEntry contextEntry = zip.getEntry("testBuildJob/src/main/resources/" + technicalLabel.toLowerCase() + "/testbuildjob_0_1/contexts/Default.properties");
assertNotNull("No context file", contextEntry);
// dq
ZipEntry tdq = zip.getEntry("testBuildJob/items/reports/");
assertNotNull("Can't find the dq reports items", tdq);
assertTrue(tdq.isDirectory());
// if the tdm is load
if (GlobalServiceRegister.getDefault().isServiceRegistered(ITransformService.class)) {
ITransformService tdmService = (ITransformService) GlobalServiceRegister.getDefault().getService(ITransformService.class);
if (tdmService.isTransformItem(processItem)) {
ZipEntry tdmSettingEntry = zip.getEntry("testBuildJob/items/" + technicalLabel.toLowerCase() + "/.settings/com.oaklandsw.base.projectProps");
assertNotNull("Can't export tdm rightly", tdmSettingEntry);
/*
* the __tdm has been moved into job jar. so need test it in jar.
*/
// ZipEntry tdmEntry = zip.getEntry("testBuildJob/__tdm/");
// assertNotNull("Can't export tdm rightly", tdmEntry);
// assertTrue("build job with tdm failure", tdmEntry.isDirectory());
// testbuildjob_0_1.jar!/__tdm/TEST_NOLOGIN.zip
final JarInputStream jarStream = new JarInputStream(zip.getInputStream(jarEntry));
boolean found = false;
JarEntry entry;
while ((entry = jarStream.getNextJarEntry()) != null) {
final String name = entry.getName();
if (name.contains("__tdm") && name.endsWith(technicalLabel + ".zip")) {
found = true;
}
}
jarStream.close();
assertTrue("Can't find the __tdm in job jar after build", found);
}
}
} finally {
if (zip != null) {
zip.close();
}
}
}
use of org.talend.core.service.ITransformService in project tdi-studio-se by Talend.
the class ExportItemUtil method exportItems2.
private Map<File, IPath> exportItems2(Collection<Item> items, File destinationDirectory, boolean projectFolderStructure, IProgressMonitor progressMonitor) throws Exception {
Map<File, IPath> toExport = new HashMap<File, IPath>();
//$NON-NLS-1$
progressMonitor.beginTask("Export Items", items.size() + 1);
final boolean oldMeasureActived = TimeMeasure.measureActive;
if (!oldMeasureActived) {
// not active before.
TimeMeasure.display = TimeMeasure.displaySteps = TimeMeasure.measureActive = CommonsPlugin.isDebugMode();
}
//$NON-NLS-1$
final String idTimer = "exportItems";
TimeMeasure.begin(idTimer);
try {
// store item and its corresponding project
Map<Item, Project> itemProjectMap = new HashMap<Item, Project>();
Collection<Item> allItems = new ArrayList<Item>(items);
items.clear();
// ycbai added for TDI-21387
if (allItems.isEmpty()) {
addTalendProjectFile(toExport, destinationDirectory);
return toExport;
}
allItems = sortItemsByProject(allItems, itemProjectMap);
ITransformService tdmService = null;
if (GlobalServiceRegister.getDefault().isServiceRegistered(ITransformService.class)) {
tdmService = (ITransformService) GlobalServiceRegister.getDefault().getService(ITransformService.class);
}
itemProjectMap.clear();
Set<String> jarNameList = new HashSet<String>();
Iterator<Item> iterator = allItems.iterator();
Set<String> projectHasTdm = new HashSet<String>();
while (iterator.hasNext()) {
Item item = iterator.next();
project = pManager.getProject(item);
String label = item.getProperty().getLabel();
// project
addTalendProjectFile(toExport, destinationDirectory);
// tdm .settings/com.oaklandsw.base.projectProps
String technicalLabel = project.getTechnicalLabel();
if (tdmService != null && !projectHasTdm.contains(technicalLabel) && tdmService.isTransformItem(item)) {
projectHasTdm.add(technicalLabel);
IPath propsSourcePath = getProjectLocationPath(technicalLabel).append(FileConstants.TDM_PROPS_PATH);
IPath tdmPropsPath = getProjectOutputPath().append(FileConstants.TDM_PROPS_PATH);
IPath propsTargetPath = new Path(destinationDirectory.getAbsolutePath()).append(tdmPropsPath);
File source = new File(propsSourcePath.toPortableString());
if (source.exists()) {
copyAndAddResource(toExport, propsSourcePath, propsTargetPath, tdmPropsPath);
}
}
// tdm simple files
if (item.getProperty() instanceof FakePropertyImpl) {
FakePropertyImpl fakeProperty = (FakePropertyImpl) item.getProperty();
IPath itemResPath = fakeProperty.getItemPath().makeRelative();
IPath itemSourcePath = getProjectLocationPath(technicalLabel).removeLastSegments(1).append(itemResPath);
// replace the project segment
IPath outputRelativeItemPath = getProjectOutputPath().append(itemResPath.removeFirstSegments(1));
IPath itemTargetPath = new Path(destinationDirectory.getAbsolutePath()).append(outputRelativeItemPath);
copyAndAddResource(toExport, itemSourcePath, itemTargetPath, outputRelativeItemPath);
continue;
}
// property and related resources eg:item, reference files
XmiResourceManager localRepositoryManager = ProxyRepositoryFactory.getInstance().getRepositoryFactoryFromProvider().getResourceManager();
IPath propertyPath = null;
for (Resource curResource : localRepositoryManager.getAffectedResources(item.getProperty())) {
URI uri = curResource.getURI();
IPath relativeItemPath = URIHelper.convert(uri).makeRelative();
Project project = ProjectManager.getInstance().getProject(item);
IPath sourcePath = getProjectLocationPath(project.getTechnicalLabel()).removeLastSegments(1).append(relativeItemPath);
// replace the project segment
IPath outputRelativeItemPath = getProjectOutputPath().append(relativeItemPath.removeFirstSegments(1));
IPath targetPath = new Path(destinationDirectory.getAbsolutePath()).append(outputRelativeItemPath);
copyAndAddResource(toExport, sourcePath, targetPath, outputRelativeItemPath);
if (uri.lastSegment() != null && uri.lastSegment().endsWith(FileConstants.PROPERTIES_FILE_SUFFIX)) {
propertyPath = targetPath;
}
}
if (GlobalServiceRegister.getDefault().isServiceRegistered(ITestContainerProviderService.class)) {
ITestContainerProviderService testContainerService = (ITestContainerProviderService) GlobalServiceRegister.getDefault().getService(ITestContainerProviderService.class);
if (testContainerService != null) {
List<IResource> dataFileList = testContainerService.getDataFiles(item);
for (IResource dataFile : dataFileList) {
IPath relativeItemPath = dataFile.getFullPath();
IPath sourcePath = getProjectLocationPath(project.getTechnicalLabel()).removeLastSegments(1).append(relativeItemPath);
// replace the project segment
IPath outputRelativeItemPath = getProjectOutputPath().append(relativeItemPath.removeFirstSegments(1));
IPath targetPath = new Path(destinationDirectory.getAbsolutePath()).append(outputRelativeItemPath);
copyAndAddResource(toExport, sourcePath, targetPath, outputRelativeItemPath);
}
}
}
if (propertyPath == null) {
return toExport;
}
if (item instanceof RoutineItem) {
List list = ((RoutineItem) item).getImports();
for (int i = 0; i < list.size(); i++) {
String jarName = ((IMPORTTypeImpl) list.get(i)).getMODULE();
jarNameList.add(jarName.toString());
}
}
boolean needChangeItem = false;
needChangeItem = needChangeItem || item.getState().isLocked();
// keep the same as function fixItem()
needChangeItem = needChangeItem || !item.getProperty().getLabel().replace(' ', '_').equals(item.getProperty().getLabel());
if (needChangeItem) {
// load in memory, fix the item and save it
XmiResourceManager xmiMamanger = new XmiResourceManager();
// loadProject
IPath proRelativePath = getProjectOutputPath().append(FileConstants.LOCAL_PROJECT_FILENAME);
IPath proTargetPath = new Path(destinationDirectory.getAbsolutePath()).append(proRelativePath);
Resource loadProject = projectResourcMap.get(proTargetPath);
if (loadProject == null) {
URI projectUri = URI.createFileURI(proTargetPath.toPortableString());
loadProject = xmiMamanger.resourceSet.getResource(projectUri, true);
projectResourcMap.put(proTargetPath, loadProject);
}
URI propertyUri = URI.createFileURI(propertyPath.toPortableString());
Resource propertyResource = xmiMamanger.resourceSet.getResource(propertyUri, true);
Property loadProperty = (Property) EcoreUtil.getObjectByType(propertyResource.getContents(), PropertiesPackage.eINSTANCE.getProperty());
Item newItem = loadProperty.getItem();
fixItem(newItem);
fixItemLockState(newItem);
saveResources(xmiMamanger.resourceSet);
}
iterator.remove();
TimeMeasure.step(idTimer, "export item: " + label);
progressMonitor.worked(1);
}
ILibraryManagerService repositoryBundleService = CorePlugin.getDefault().getRepositoryBundleService();
// add the routines of the jars at the end, to add them only once in the export.
IPath libPath = getProjectOutputPath().append(JavaUtils.JAVA_LIB_DIRECTORY);
String libAbsPath = new Path(destinationDirectory.toString()).append(libPath.toString()).toPortableString();
for (String jarName : jarNameList) {
if (repositoryBundleService.contains(jarName)) {
repositoryBundleService.retrieve(jarName, libAbsPath, new NullProgressMonitor());
toExport.put(new File(libAbsPath, jarName), libPath.append(jarName));
}
}
} catch (Exception e) {
ExceptionHandler.process(e);
} finally {
TimeMeasure.end(idTimer);
// if active before, not disable and active still.
if (!oldMeasureActived) {
TimeMeasure.display = TimeMeasure.displaySteps = TimeMeasure.measureActive = false;
}
}
return toExport;
}
use of org.talend.core.service.ITransformService in project tdi-studio-se by Talend.
the class JobScriptsManager method addDependencies.
/**
* DOC qwei Comment method "addDepencies".
*/
protected void addDependencies(ExportFileResource[] allResources, ProcessItem processItem, Boolean needDependencies, ExportFileResource resource) {
if (!needDependencies) {
return;
}
// export current job's dependencies.
if (!exportCaculatedItems.contains(processItem)) {
BuildExportManager.getInstance().exportDependencies(resource, processItem);
exportCaculatedItems.add(processItem);
}
Collection<IRepositoryViewObject> allDependencies = ProcessUtils.getAllProcessDependencies(Arrays.asList(new Item[] { processItem }), false);
ITransformService tdmService = null;
if (GlobalServiceRegister.getDefault().isServiceRegistered(ITransformService.class)) {
tdmService = (ITransformService) GlobalServiceRegister.getDefault().getService(ITransformService.class);
}
try {
for (IRepositoryViewObject object : allDependencies) {
Item item = object.getProperty().getItem();
ERepositoryObjectType itemType = ERepositoryObjectType.getItemType(item);
IPath typeFolderPath = new Path(ERepositoryObjectType.getFolderName(itemType));
String itemName = item.getProperty().getLabel();
String itemVersion = item.getProperty().getVersion();
String itemPath = item.getState().getPath();
//$NON-NLS-1$ //$NON-NLS-2$
itemPath = (itemPath == null || itemPath.equals("")) ? "" : itemPath;
IPath projectRootPath = getCorrespondingProjectRootPath(item);
String projectName = getCorrespondingProjectName(item);
String relativePath = JOB_ITEMS_FOLDER_NAME + PATH_SEPARATOR + projectName;
// add .settings/com.oaklandsw.base.projectProps for tdm
if (tdmService != null && !exportCaculatedProject.contains(projectName) && tdmService.isTransformItem(item)) {
IPath propsPath = getCorrespondingProjectRootPath(item).append(FileConstants.TDM_PROPS_PATH);
String propsRelativePath = relativePath;
propsRelativePath = propsRelativePath + PATH_SEPARATOR + FileConstants.TDM_PROPS_FOLDER;
resource.addResource(propsRelativePath, propsPath.toFile().toURL());
}
// project file
IPath projectFilePath = getCorrespondingProjectRootPath(item).append(FileConstants.LOCAL_PROJECT_FILENAME);
checkAndAddProjectResource(allResources, resource, relativePath, FileLocator.toFileURL(projectFilePath.toFile().toURL()));
// export simple file
if (projectRootPath != null && item.getProperty() instanceof FakePropertyImpl) {
String basePath = relativePath + PATH_SEPARATOR + typeFolderPath.toString();
FakePropertyImpl fakeProperty = (FakePropertyImpl) item.getProperty();
IPath relativeItemPath = fakeProperty.getItemPath();
IPath absItemPath = projectRootPath.removeLastSegments(1).append(relativeItemPath.makeRelative());
resource.addResource(basePath, absItemPath.toFile().toURL());
continue;
}
IPath itemFilePath;
String itemVersionString = (itemVersion == null) ? "" : "_" + itemVersion;
if (itemPath.startsWith(typeFolderPath.toString())) {
itemPath = itemPath.substring(typeFolderPath.toString().length());
}
if (item.getFileExtension() == null || "".equals(item.getFileExtension())) {
//$NON-NLS-1$
itemFilePath = projectRootPath.append(typeFolderPath).append(itemPath).append(itemName + (item.isNeedVersion() ? itemVersionString : "") + "." + //$NON-NLS-1$
FileConstants.ITEM_EXTENSION);
} else {
itemFilePath = projectRootPath.append(typeFolderPath).append(itemPath).append(//$NON-NLS-1$
itemName + (item.isNeedVersion() ? itemVersionString : "") + "." + item.getFileExtension());
}
IPath propertiesFilePath = projectRootPath.append(typeFolderPath).append(itemPath).append(//$NON-NLS-1$
itemName + itemVersionString + "." + FileConstants.PROPERTIES_EXTENSION);
List<URL> metadataNameFileUrls = new ArrayList<URL>();
File itemFile = itemFilePath.toFile();
if (itemFile.exists()) {
metadataNameFileUrls.add(FileLocator.toFileURL(itemFile.toURI().toURL()));
} else {
ExceptionHandler.log(Messages.getString("JobScriptsManager.ResourceNotFoundForExport", itemFilePath));
}
File propertiesFile = propertiesFilePath.toFile();
if (propertiesFile.exists()) {
metadataNameFileUrls.add(FileLocator.toFileURL(propertiesFile.toURI().toURL()));
} else {
ExceptionHandler.log(Messages.getString("JobScriptsManager.ResourceNotFoundForExport", propertiesFilePath));
}
String basePath = relativePath + PATH_SEPARATOR + typeFolderPath.toString();
if (itemPath != null && !"".equals(itemPath)) {
//$NON-NLS-1$
basePath = basePath + PATH_SEPARATOR + itemPath;
}
resource.addResources(basePath, metadataNameFileUrls);
// children dependencies
if (!exportCaculatedItems.contains(item)) {
BuildExportManager.getInstance().exportDependencies(resource, item);
exportCaculatedItems.add(item);
}
}
if (GlobalServiceRegister.getDefault().isServiceRegistered(ITDQItemService.class)) {
ITDQItemService tdqItemService = (ITDQItemService) GlobalServiceRegister.getDefault().getService(ITDQItemService.class);
if (tdqItemService != null && tdqItemService.hasProcessItemDependencies(Arrays.asList(new Item[] { processItem }))) {
// add .Talend.definition file
//$NON-NLS-1$
String defIdxFolderName = "TDQ_Libraries";
//$NON-NLS-1$
String defIdxFileName = ".Talend.definition";
IProject project = ReponsitoryContextBridge.getRootProject();
IFile defIdxFile = project.getFile(defIdxFolderName + PATH_SEPARATOR + defIdxFileName);
if (defIdxFile.exists()) {
String defIdxBasePath = JOB_ITEMS_FOLDER_NAME + PATH_SEPARATOR + project.getName().toLowerCase() + PATH_SEPARATOR + defIdxFolderName;
List<URL> defIdxUrls = new ArrayList<URL>();
defIdxUrls.add(project.getLocation().makeAbsolute().append(defIdxFolderName).append(defIdxFileName).toFile().toURI().toURL());
resource.addResources(defIdxBasePath, defIdxUrls);
}
// add report header image & template files
//$NON-NLS-1$
String reportingBundlePath = PluginChecker.getBundlePath("org.talend.dataquality.reporting");
List<URL> reportResourceUrls = new ArrayList<URL>();
//$NON-NLS-1$
File imageFolder = new File(reportingBundlePath + PATH_SEPARATOR + "images");
if (imageFolder.exists()) {
reportResourceUrls.add(imageFolder.toURI().toURL());
}
//$NON-NLS-1$
File templateFolder = new File(reportingBundlePath + PATH_SEPARATOR + "reports");
if (templateFolder.exists() && templateFolder.isDirectory()) {
reportResourceUrls.add(templateFolder.toURI().toURL());
}
// include all added resources
resource.addResources(JOB_ITEMS_FOLDER_NAME + PATH_SEPARATOR, reportResourceUrls);
}
}
} catch (Exception e) {
ExceptionHandler.process(e);
}
}
use of org.talend.core.service.ITransformService in project tdi-studio-se by Talend.
the class BuildJobHandler method addTDMDependencies.
/**
* DOC nrousseau Comment method "addTDMDependencies".
*
* @param items
* @param itemsFolder
*/
private void addTDMDependencies(IFolder itemsFolder, List<Item> items) {
ITransformService tdmService = null;
if (GlobalServiceRegister.getDefault().isServiceRegistered(ITransformService.class)) {
tdmService = (ITransformService) GlobalServiceRegister.getDefault().getService(ITransformService.class);
}
if (tdmService == null) {
return;
}
try {
// add __tdm dependencies
ExportFileResource resouece = new ExportFileResource();
BuildExportManager.getInstance().exportDependencies(resouece, processItem);
if (!resouece.getAllResources().isEmpty()) {
final Iterator<String> relativepath = resouece.getRelativePathList().iterator();
while (relativepath.hasNext()) {
String relativePath = relativepath.next();
// TDQ-12852 do nothing if it is DQ resources.
if (relativePath == null || relativePath.contains("metadata/survivorship")) {
//$NON-NLS-1$
continue;
}
Set<URL> sources = resouece.getResourcesByRelativePath(relativePath);
for (URL sourceUrl : sources) {
File currentResource = new File(org.talend.commons.utils.io.FilesUtils.getFileRealPath(sourceUrl.getPath()));
if (currentResource.exists()) {
// the __tdm will be out of items folder, same level for items
IFolder targetFolder = ((IFolder) itemsFolder.getParent()).getFolder(relativePath);
if (!targetFolder.exists()) {
targetFolder.create(true, true, new NullProgressMonitor());
}
FilesUtils.copyFile(currentResource, new File(targetFolder.getLocation().toPortableString() + File.separator + currentResource.getName()));
}
}
}
}
itemsFolder.refreshLocal(IResource.DEPTH_INFINITE, null);
// make sure to export again.
for (Item item : items) {
if (tdmService.isTransformItem(item)) {
setNeedItemDependencies(true);
String itemProjectFolder = getProject(item).getTechnicalLabel();
if (isProjectNameLowerCase()) {
// should be same as ExportItemUtil.getProjectOutputPath
itemProjectFolder = itemProjectFolder.toLowerCase();
}
IPath targetSettingPath = new Path(itemProjectFolder).append(RepositoryConstants.SETTING_DIRECTORY);
IFolder targetSettingsFolder = talendProcessJavaProject.createSubFolder(null, itemsFolder, targetSettingPath.toString());
IProject sourceProject = getCorrespondingProjectRootFolder(item);
if (sourceProject.exists()) {
IFile targetTdmPropsFile = targetSettingsFolder.getFile(FileConstants.TDM_PROPS);
IFile sourceTdmPropsFile = sourceProject.getFolder(RepositoryConstants.SETTING_DIRECTORY).getFile(FileConstants.TDM_PROPS);
// if have existed, no need copy again.
if (sourceTdmPropsFile.exists() && !targetTdmPropsFile.exists()) {
sourceTdmPropsFile.copy(targetTdmPropsFile.getFullPath(), true, null);
}
}
// only deal with one time.
break;
}
}
} catch (Exception e) {
// don't block all the export if got exception here
ExceptionHandler.process(e);
}
}
Aggregations