Improved test_run to fail when a test generates an error on create_ref or no_check_ref
[scilab.git] / scilab / modules / development_tools / help / en_US / test_run.xml
index 2e48c1a..1638de4 100644 (file)
@@ -15,7 +15,9 @@
 <refentry xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svg="http://www.w3.org/2000/svg" xmlns:ns5="http://www.w3.org/1999/xhtml" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:db="http://docbook.org/ns/docbook" xmlns:scilab="http://www.scilab.org" xml:id="test_run" xml:lang="en">
     <refnamediv>
         <refname>test_run</refname>
-        <refpurpose>Launch tests</refpurpose>
+        <refpurpose>
+            Launches unit tests and non-regression test in a module or directory. The test run checks first that test scripts does not generate an error, then that the test diary (<literal>dia</literal> files) is the same as the reference diary (<literal>dia.ref</literal> files).
+        </refpurpose>
     </refnamediv>
     <refsynopsisdiv>
         <title>Calling Sequence</title>
             <varlistentry>
                 <term>module</term>
                 <listitem>
-                    <para>A String array. This input argument must be</para>
+                    <para>
+                        A String array or <literal>[]</literal>. Name of the modules or directory for the tests, all internal modules if <literal>[]</literal>
+                    </para>
                     <itemizedlist>
                         <listitem>
                             <para>
-                                the name of an internal Scilab module ("core", "time", ...) or a sub-module (e.g. "optimization|neldermead").
+                                the name of an internal Scilab module ("core", "time", ...), a sub-module (e.g. "optimization|neldermead").
                             </para>
                         </listitem>
                         <listitem>
@@ -46,7 +50,7 @@
                         </listitem>
                         <listitem>
                             <para>
-                                the absolute directory path of a module.
+                                the absolute directory path of a module containing test/unit_tests or test/nonreg_tests.
                             </para>
                         </listitem>
                     </itemizedlist>
             <varlistentry>
                 <term>test_name</term>
                 <listitem>
-                    <para>A string array</para>
+                    <para>
+                        A string array or []. The names of the tests to execute during this run. If <literal>[]</literal> executes all tests found.
+                    </para>
                 </listitem>
             </varlistentry>
             <varlistentry>
                 <term>options</term>
                 <listitem>
-                    <para>A string array</para>
-                    <variablelist>
+                    <para>
+                        A string array or <literal>[]</literal>. The options for the tests during this run, default options if <literal>[]</literal>.
+                    </para>
+                    <variablelist> or
                         <varlistentry>
-                            <term>no_check_ref</term>
+                            <term>"no_check_ref"</term>
                             <listitem>
                                 <para>does not check if the .dia and .dia.ref are equal</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>no_check_error_output</term>
+                            <term>"no_check_error_output"</term>
                             <listitem>
                                 <para>The error output stream is not checked. This option can be used when Scilab complains about the localization being not available.</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>create_ref</term>
+                            <term>"create_ref"</term>
                             <listitem>
                                 <para>create the .dia.ref file and does not check if the .dia and .dia.ref are equal</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>show_error</term>
+                            <term>"show_error"</term>
                             <listitem>
-                                <para>If an error occurs, show the last 10 lines of the execution</para>
+                                <para>If an error occurs, shows the last 10 lines of the execution</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>show_diff</term>
+                            <term>"show_diff"</term>
                             <listitem>
                                 <para>
-                                    If a difference is found, show the result of the command <literal>diff -u</literal>
+                                    If a difference is found, shows the result of the command <literal>diff -u</literal>
                                 </para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>list</term>
+                            <term>"list"</term>
                             <listitem>
                                 <para>Does not perform the tests but displays a list of available tests</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>help</term>
+                            <term>"help"</term>
                             <listitem>
                                 <para>display some examples about how to use this command</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>mode_nw</term>
+                            <term>"mode_nw"</term>
                             <listitem>
                                 <para>Add the "-nw" option to the launch</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>mode_nwni</term>
+                            <term>"mode_nwni"</term>
                             <listitem>
                                 <para>Add the "-nwni" option to the launch</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>mode_nwni_profiling</term>
+                            <term>"mode_nwni_profiling"</term>
                             <listitem>
                                 <para>Add the "-nwni -profiling" option to the launch for detect valgrind error (Linux only)</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>nonreg_tests</term>
+                            <term>"nonreg_tests"</term>
                             <listitem>
                                 <para>runs only the non-regression tests, skipping unit tests</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>unit_tests</term>
+                            <term>"unit_tests"</term>
                             <listitem>
                                 <para>Runs only the unit tests, skipping non-regression tests</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>skip_tests</term>
+                            <term>"skip_tests"</term>
                             <listitem>
-                                <para>Skip the tests</para>
+                                <para>
+                                    Skip the tests specified in <term>test_name</term>.
+                                </para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>enable_lt</term>
+                            <term>"enable_lt"</term>
                             <listitem>
                                 <para>Enable long-time execution tests</para>
                             </listitem>
                         </varlistentry>
                         <varlistentry>
-                            <term>short_summary</term>
+                            <term>"short_summary"</term>
                             <listitem>
                                 <para>Does not display statistics nor execution time after execution (only number of executed, passed, failed and skipped tests will be displayed on a single line).</para>
                             </listitem>
             execute them, and display a report about success of failures.
             The .tst files are searched in directories SCI+"/modules/*/tests/unit_tests"
             and SCI+"/modules/*/tests/nonreg_tests".
-            Whenever a test is executed, a .dia file is generated which contains
-            the full list of commands executed along with message which appears in the
-            console. When the script is done, the .dia file is compared with
-            the .dia.ref file which is expected to be in the same directory
+        </para>
+        <para>
+            First <literal>test_run</literal> checks that a test does not produce an error.
+        </para>
+        <para>
+            Then <literal>test_run</literal> checks that the output and commands of a script are identical to the reference file. Whenever a test is executed, a <literal>.dia</literal> file is generated which contains
+            the full list of commands executed along with messages that appear in the
+            console output. When the script is done, the <literal>.dia</literal> file is compared with
+            the <literal>.dia.ref</literal> file which is expected to be in the same directory
             as the .tst file. If the two file are different, the test fails.
         </para>
         <para>
         <para>
             The algorithm is the following.
             First, the <literal>.ref</literal> is considered. If this file does not exist,
-            the platform-specific <literal>.ref</literal> file is examined depending on the current platform.
+            the platform-specific <literal>platform.ref</literal> file is examined depending on the current platform.
         </para>
         <itemizedlist>
             <listitem>
@@ -453,8 +468,6 @@ test_run('time','datenum',[],TMPDIR+"/time_datenum_test_run.xml");
         <para>
             The tests are performed in the temporary directory, not in the directory
             which originally contain the tests files.
-            The .tst file is copied into the temporary directory, the test is performed
-            and the .dia.ref is copied back into the original location.
         </para>
         <para>
             The .tst script is not run as is. Instead, a header and a footer are
@@ -491,8 +504,12 @@ test_run('time','datenum',[],TMPDIR+"/time_datenum_test_run.xml");
                         Fourth argument added to export to a XML file
                     </para>
                 </revdescription>
+            </revision>
+            <revision>
                 <revnumber>5.5.0</revnumber>
                 <revdescription>32/64bits separation available</revdescription>
+            </revision>
+            <revision>
                 <revnumber>6.0.0</revnumber>
                 <revdescription>profiling mode added to profile execution with valgrind (Linux only)</revdescription>
             </revision>