Examples with StepExecutionEntity edu.harvard.iq.dataverse.batch.entities.StepExecutionEntity used on opensource projects

Example 6 with StepExecutionEntity

use of edu.harvard.iq.dataverse.batch.entities.StepExecutionEntity in project dataverse by IQSS.

the class FileRecordJobIT method testFilesWithChecksumManifest.

@Test
@Ignore
public /**
 * The success case: all files uploaded and present in checksum manifest
 */
void testFilesWithChecksumManifest() {
    try {
        // create test files and checksum manifest
        File file1 = createTestFile(dsDir, "testfile1.txt", 0.25);
        File file2 = createTestFile(dsDir, "testfile2.txt", 0.25);
        String checksum1 = "asfdasdfasdfasdf";
        String checksum2 = "sgsdgdsgfsdgsdgf";
        if (file1 != null && file2 != null) {
            PrintWriter pw = new PrintWriter(new FileWriter(dsDir + "/files.sha"));
            pw.write(checksum1 + " " + file1.getName());
            pw.write("\n");
            pw.write(checksum2 + " " + file2.getName());
            pw.write("\n");
            pw.close();
        } else {
            fail();
        }
        JobExecutionEntity job = getJob();
        assertEquals(job.getSteps().size(), 1);
        StepExecutionEntity step1 = job.getSteps().get(0);
        Map<String, Long> metrics1 = step1.getMetrics();
        // check job status
        assertEquals(BatchStatus.COMPLETED.name(), job.getExitStatus());
        assertEquals(BatchStatus.COMPLETED, job.getStatus());
        // check step 1 status and name
        assertEquals(step1.getExitStatus(), BatchStatus.COMPLETED.name());
        assertEquals(step1.getStatus(), BatchStatus.COMPLETED);
        assertEquals(step1.getName(), "import-files");
        // verify step 1 metrics
        assertEquals((long) metrics1.get("write_skip_count"), 0);
        assertEquals((long) metrics1.get("commit_count"), 1);
        assertEquals((long) metrics1.get("process_skip_count"), 0);
        assertEquals((long) metrics1.get("read_skip_count"), 0);
        assertEquals((long) metrics1.get("write_count"), 2);
        assertEquals((long) metrics1.get("rollback_count"), 0);
        assertEquals((long) metrics1.get("filter_count"), 0);
        assertEquals((long) metrics1.get("read_count"), 2);
        // should be no user data (error messages)
        assertEquals(step1.getPersistentUserData(), null);
        // confirm files were imported
        updateDatasetJsonPath();
        List<String> filenames = new ArrayList<>();
        filenames.add(dsPath.getString("data.latestVersion.files[0].dataFile.filename"));
        filenames.add(dsPath.getString("data.latestVersion.files[1].dataFile.filename"));
        assert (filenames.contains("testfile1.txt"));
        assert (filenames.contains("testfile2.txt"));
        // confirm checksums were imported
        List<String> checksums = new ArrayList<>();
        checksums.add(dsPath.getString("data.latestVersion.files[0].dataFile.checksum.value"));
        checksums.add(dsPath.getString("data.latestVersion.files[1].dataFile.checksum.value"));
        assert (checksums.contains(checksum1));
        assert (checksums.contains(checksum2));
    } catch (Exception e) {
        System.out.println("Error testChecksumImport: " + e.getMessage());
        e.printStackTrace();
        fail();
    }
}

Example 7 with StepExecutionEntity

use of edu.harvard.iq.dataverse.batch.entities.StepExecutionEntity in project dataverse by IQSS.

the class FileRecordJobIT method testFileMissingInChecksumManifest.

@Test
@Ignore
public /**
 * Checksum manifest is missing an uploaded file
 */
void testFileMissingInChecksumManifest() {
    try {
        // create test files and checksum manifest with just one of the files
        File file1 = createTestFile(dsDir, "testfile1.txt", 0.25);
        File file2 = createTestFile(dsDir, "testfile2.txt", 0.25);
        String checksum1 = "";
        if (file1 != null && file2 != null) {
            PrintWriter pw = new PrintWriter(new FileWriter(dsDir + "/files.sha"));
            checksum1 = "asasdlfkj880asfdasflj";
            pw.write(checksum1 + " " + file1.getName());
            pw.write("\n");
            pw.close();
        } else {
            fail();
        }
        JobExecutionEntity job = getJob();
        assertEquals(job.getSteps().size(), 1);
        StepExecutionEntity step1 = job.getSteps().get(0);
        Map<String, Long> metrics1 = step1.getMetrics();
        // check job status
        assertEquals(job.getExitStatus(), BatchStatus.COMPLETED.name());
        assertEquals(job.getStatus(), BatchStatus.COMPLETED);
        // check step 1 status and name
        assertEquals(step1.getExitStatus(), BatchStatus.COMPLETED.name());
        assertEquals(step1.getStatus(), BatchStatus.COMPLETED);
        assertEquals(step1.getName(), "import-files");
        // verify step 1 metrics
        assertEquals((long) metrics1.get("write_skip_count"), 0);
        assertEquals((long) metrics1.get("commit_count"), 1);
        assertEquals((long) metrics1.get("process_skip_count"), 0);
        assertEquals((long) metrics1.get("read_skip_count"), 0);
        assertEquals((long) metrics1.get("write_count"), 2);
        assertEquals((long) metrics1.get("rollback_count"), 0);
        assertEquals((long) metrics1.get("filter_count"), 0);
        assertEquals((long) metrics1.get("read_count"), 2);
        // should be no user data (error messages)
        assertEquals(step1.getPersistentUserData(), null);
        // confirm files were imported
        updateDatasetJsonPath();
        List<String> filenames = new ArrayList<>();
        filenames.add(dsPath.getString("data.latestVersion.files[0].dataFile.filename"));
        filenames.add(dsPath.getString("data.latestVersion.files[1].dataFile.filename"));
        assert (filenames.contains("testfile1.txt"));
        assert (filenames.contains("testfile2.txt"));
        // confirm one checksums was imported, one not
        List<String> checksums = new ArrayList<>();
        checksums.add(dsPath.getString("data.latestVersion.files[0].dataFile.checksum.value"));
        checksums.add(dsPath.getString("data.latestVersion.files[1].dataFile.checksum.value"));
        assert (checksums.contains(checksum1));
        assert (checksums.contains("Unknown"));
    } catch (Exception e) {
        System.out.println("Error testChecksumImport: " + e.getMessage());
        e.printStackTrace();
        fail();
    }
}