The special configuration for Mutations testing allows for testing of inserted mutations. For best performance the mutations should be able to be enabled without rebuilding anything. In this configuration it is possible to use a high bandwidth since no code checkout needs to be duplicated.

Test Setup

This type of test setup works as follows:

  1. After configuring which mutations to test PinDown is launched once

  2. PinDown selects a subset of the mutations and launches the test suite (a script written by the user). All test jobs run on the computer farm.

  3. When the first failure occurs PinDown stops the test suite.

  4. PinDown debugs the active mutations in parallel, and marks failing mutations as detected.

  5. If no failures are found during the test phase all active mutations are marked as non-detect.

  6. PinDown goes back to step 2 selecting a new subset among the mutations still not classified as detect/non-detect.

Optimal Settings

The optimal PinDown settings for this test setup is as follows:

Setup for mutation testing

Use 100 slots at a time when testing, and test 100 unique mutations at the same time.

set_debug_bandwidth 100;
set_diagnosis_optimization -target "mutation";
set_mutations "1-100,500-1000" -bandwidth "100";
Checkout special mutation control repository instead of customer code.

The variable %NUM_MUTATIONS% is always updated to the current number of active mutations being tested.

During checkout a special file mutations.txt is created in the checkout area. It contains the current set of enabled mutations for this checkout, one per line if multiple are enabled.

set_repository -option "TAGFILTER=mutation_.*";
set_repository -type "git" -location "/1234/git_mutation" -username "" -password "" -checkoutarea "mutation" -initialrevision "mutation_%NUM_MUTATIONS%";
Correctly Group Failure Messages Into Buckets

PinDown automatically groups failure messages into buckets. The precision of this grouping can be improved by filtering out parts of the line that doesn’t contain the actaul error messages, such as file names, timestamps etc. The better the grouping the better for performance. If there are say 4 different problems the goal is to end up with exactly 4 distinct buckets.

set_group_filter -type "all" -replace ".*test.log" -with "";
set_group_filter -type "all" -replace ".*FAILED" -with "FAILED";

Example

Here is an example of PinDown setup for mutation. It also contains the options needed to automatically switch to debugging hard to detect failures that only appears in combination with other failures.

pindown_config.txt
/*--------------------------------------------------------

 PinDown Setup - Mutation Testing

---------------------------------------------------------*/

// Server paths
license -flexlm "8055@flexlm.srv.local";
set_webinterface -path "http://vm.verifyter.com:8080/PinDown/";
set_results_database -location "vm:5432" -database "pindown" -username "postgres" -password "ENC:A3faSFgErdD35F";

// Debug 100 mutations at a time
set_debug_bandwidth 100;
if "%MUTATION_MODE% == 'HIGHER_ORDER'" {
set_diagnosis_optimization -target "mutation";
  set_diagnosis_optimization -target "time" -limit "10" -minopenbug "10";
  set_max_tests "1";
} else {
  set_diagnosis_optimization -target "mutation";
}

// Select mutations 1-100 and 500-1000 and use First and Higher order modes.
set_mutations "1-100,500-1000" -bandwidth "100" -option "MODE=FIRST_ORDER,HIGHER_ORDER";

// Group according to the failure messages, ignore path to the test.log and any numbers/timestamps preceeding FAILED:
set_group_filter -type "all" -replace ".*test.log" -with "";
set_group_filter -type "all" -replace ".*FAILED" -with "FAILED";

// Clear bugs for fresh analysis each time and do not check the setup
clear_results_database; // Database must be cleared each time since the active mutations change.
check_setup "false";

// Kill all test jobs
set_kill_tests "kill_farm_jobs.sh";

// Kill PinDown itself
set_kill_pindown "kill %PINDOWN_PID%";

// Repositories
set_repository -option "CHECKOUTINTERVAL=1";
set_repository -option "CONTROLCHECKOUT=3";
set_repository -option "TAGFILTER=mutation_.*";
set_repository -type "git" -location "/1234/git_mutation" -username "" -password "" -checkoutarea "mutation" -initialrevision "mutation_%NUM_MUTATIONS%";

// Test Results
group -name "test" -commands {
  extract -type "configlabel" -label "configLabel" -source "filename" -path "project_cat/.*/reg/mrl_.*" -keywords "";
  extract -type "buildpass" -path "%configLabel%/vcs/compile.log" -keywords "simv up to date";
  extract -type "buildfail" -path "%configLabel%/vcs/compile.log" -keywords "Error|syntax error|Compilation aborted";

  extract -type "testname" -path "%configLabel%/vcs/stat/log.sanity.*" -keywords "PASS|FAIL" -filter "%WAIVER%";
  extract -type "testpass" -path "%configLabel%/vcs/stat/log.sanity.*" -keywords "PASS" -filter "PROJECT_HOME is not currently set";
  extract -type "testfail" -path "%configLabel%/vcs/stat/log.sanity.*" -keywords "FAIL";
  extract -type "replace" -label "testname" -text ".*:" -with "";
  extract -type "replace" -label "testname" -text " .*" -with "";
  extract -type "replace" -label "configLabel" -text ".*\/reg\/" -with "";

  set_completion -source "file" -path "done.txt" -keywords "";
  set_completion -source "newfailure" -path "" -keywords "ELAPSED<20m:1" -filter "";
};

// Checkout
checkout_repositories -latest;

// Launch tests and wait for test results
shell "qsub -cwd -V -P bnormal -l os_bit=64,qsc='h|i|j|l|k',os_version='WS7.0' -N runtests_dev -o runtests_log.txt -e runtests_log.txt /fileserver1/pindown/dev/runtests.csh" -test;
wait -completion "test" -sleep "5m" -timeout "2h";

// Extract Test Results
extract -group "test" -file "test_result.xml";
read_results "test_result.xml";

set_variable -key "DEBUG_TIME" -value "Math.min(60,Math.max(5,%PINDOWN_TESTPHASE_TIME%*1.4))" -option "eval,no-publish";

// Debug failures
diagnose -runtests {
  checkout_repositories -requested;
  shell "cp $CHECKOUTAREA/mutations.txt mutations.txt" -diagnosis;
  shell "qsub -cwd -V -P bnormal -l os_bit=64,qsc='h|i|j|l|k',os_version='WS7.0' -N runtests_dev -o rundebug_log.txt -e rundebug_log.txt /fileserver1/pindown/dev/rundebug.csh" -diagnosis;
  if "%MUTATION_MODE% == 'HIGHER_ORDER'" {
    wait -completion "diagnosis" -sleep "15s" -timeout "60m";
  } else {
    wait -completion "diagnosis" -sleep "15s" -timeout "COMPLETED_FAIL==0:60m;COMPLETED<50%:60m;%DEBUG_TIME%m";
  }
  extract -group "diagnosis" -file "debug_result.xml";
  read_results "debug_result.xml";
};