Improved test_run to fail when a test generates an error on create_ref or no_check_ref
[scilab.git] / scilab / modules / development_tools / help / en_US / test_run.xml
1 <?xml version="1.0" encoding="UTF-8"?>
2 <!--
3  * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  * Copyright (C) INRIA
5  * Copyright (C) 2009-2011 - DIGITEO - Michael Baudin
6  * Copyright (C) 2013 - Scilab Enterprises - Paul Bignier: added 32/64bits separation
7  *
8  * This file must be used under the terms of the CeCILL.
9  * This source file is licensed as described in the file COPYING, which
10  * you should have received as part of this distribution.  The terms
11  * are also available at
12  * http://www.cecill.info/licences/Licence_CeCILL_V2.1-en.txt
13  *
14  -->
15 <refentry xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:svg="http://www.w3.org/2000/svg" xmlns:ns5="http://www.w3.org/1999/xhtml" xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:db="http://docbook.org/ns/docbook" xmlns:scilab="http://www.scilab.org" xml:id="test_run" xml:lang="en">
16     <refnamediv>
17         <refname>test_run</refname>
18         <refpurpose>
19             Launches unit tests and non-regression test in a module or directory. The test run checks first that test scripts does not generate an error, then that the test diary (<literal>dia</literal> files) is the same as the reference diary (<literal>dia.ref</literal> files).
20         </refpurpose>
21     </refnamediv>
22     <refsynopsisdiv>
23         <title>Calling Sequence</title>
24         <synopsis>
25             status = test_run()
26             status = test_run(module)
27             status = test_run(module, test_name)
28             status = test_run(module, test_name, options, exportToFile)
29         </synopsis>
30     </refsynopsisdiv>
31     <refsection>
32         <title>Arguments</title>
33         <variablelist>
34             <varlistentry>
35                 <term>module</term>
36                 <listitem>
37                     <para>
38                         A String array or <literal>[]</literal>. Name of the modules or directory for the tests, all internal modules if <literal>[]</literal>
39                     </para>
40                     <itemizedlist>
41                         <listitem>
42                             <para>
43                                 the name of an internal Scilab module ("core", "time", ...), a sub-module (e.g. "optimization|neldermead").
44                             </para>
45                         </listitem>
46                         <listitem>
47                             <para>
48                                 the name of an ATOMS module ("module_lycee", "nisp", ...). To be taken into account, the module must be loaded when test_run() is called.
49                             </para>
50                         </listitem>
51                         <listitem>
52                             <para>
53                                 the absolute directory path of a module containing test/unit_tests or test/nonreg_tests.
54                             </para>
55                         </listitem>
56                     </itemizedlist>
57                 </listitem>
58             </varlistentry>
59             <varlistentry>
60                 <term>test_name</term>
61                 <listitem>
62                     <para>
63                         A string array or []. The names of the tests to execute during this run. If <literal>[]</literal> executes all tests found.
64                     </para>
65                 </listitem>
66             </varlistentry>
67             <varlistentry>
68                 <term>options</term>
69                 <listitem>
70                     <para>
71                         A string array or <literal>[]</literal>. The options for the tests during this run, default options if <literal>[]</literal>.
72                     </para>
73                     <variablelist> or
74                         <varlistentry>
75                             <term>"no_check_ref"</term>
76                             <listitem>
77                                 <para>does not check if the .dia and .dia.ref are equal</para>
78                             </listitem>
79                         </varlistentry>
80                         <varlistentry>
81                             <term>"no_check_error_output"</term>
82                             <listitem>
83                                 <para>The error output stream is not checked. This option can be used when Scilab complains about the localization being not available.</para>
84                             </listitem>
85                         </varlistentry>
86                         <varlistentry>
87                             <term>"create_ref"</term>
88                             <listitem>
89                                 <para>create the .dia.ref file and does not check if the .dia and .dia.ref are equal</para>
90                             </listitem>
91                         </varlistentry>
92                         <varlistentry>
93                             <term>"show_error"</term>
94                             <listitem>
95                                 <para>If an error occurs, shows the last 10 lines of the execution</para>
96                             </listitem>
97                         </varlistentry>
98                         <varlistentry>
99                             <term>"show_diff"</term>
100                             <listitem>
101                                 <para>
102                                     If a difference is found, shows the result of the command <literal>diff -u</literal>
103                                 </para>
104                             </listitem>
105                         </varlistentry>
106                         <varlistentry>
107                             <term>"list"</term>
108                             <listitem>
109                                 <para>Does not perform the tests but displays a list of available tests</para>
110                             </listitem>
111                         </varlistentry>
112                         <varlistentry>
113                             <term>"help"</term>
114                             <listitem>
115                                 <para>display some examples about how to use this command</para>
116                             </listitem>
117                         </varlistentry>
118                         <varlistentry>
119                             <term>"mode_nw"</term>
120                             <listitem>
121                                 <para>Add the "-nw" option to the launch</para>
122                             </listitem>
123                         </varlistentry>
124                         <varlistentry>
125                             <term>"mode_nwni"</term>
126                             <listitem>
127                                 <para>Add the "-nwni" option to the launch</para>
128                             </listitem>
129                         </varlistentry>
130                         <varlistentry>
131                             <term>"mode_nwni_profiling"</term>
132                             <listitem>
133                                 <para>Add the "-nwni -profiling" option to the launch for detect valgrind error (Linux only)</para>
134                             </listitem>
135                         </varlistentry>
136                         <varlistentry>
137                             <term>"nonreg_tests"</term>
138                             <listitem>
139                                 <para>runs only the non-regression tests, skipping unit tests</para>
140                             </listitem>
141                         </varlistentry>
142                         <varlistentry>
143                             <term>"unit_tests"</term>
144                             <listitem>
145                                 <para>Runs only the unit tests, skipping non-regression tests</para>
146                             </listitem>
147                         </varlistentry>
148                         <varlistentry>
149                             <term>"skip_tests"</term>
150                             <listitem>
151                                 <para>
152                                     Skip the tests specified in <term>test_name</term>.
153                                 </para>
154                             </listitem>
155                         </varlistentry>
156                         <varlistentry>
157                             <term>"enable_lt"</term>
158                             <listitem>
159                                 <para>Enable long-time execution tests</para>
160                             </listitem>
161                         </varlistentry>
162                         <varlistentry>
163                             <term>"short_summary"</term>
164                             <listitem>
165                                 <para>Does not display statistics nor execution time after execution (only number of executed, passed, failed and skipped tests will be displayed on a single line).</para>
166                             </listitem>
167                         </varlistentry>
168                     </variablelist>
169                 </listitem>
170             </varlistentry>
171             <varlistentry>
172                 <term>exportToFile</term>
173                 <listitem>
174                     <para>
175                         Export to a XML file the result of the test. This file follows the specification of the XUnit format.
176                         Note that the usage of this option enables <literal>show_diff</literal> and <literal>show_error</literal>.
177                     </para>
178                     <para>
179                         If the file pointed by <varname>exportToFile</varname> already exists, the new result will be added to the existing file.
180                     </para>
181                 </listitem>
182             </varlistentry>
183             <varlistentry>
184                 <term>status</term>
185                 <listitem>
186                     <para>
187                         Boolean value
188                         Returns %t if no error has been detected
189                         Returns %f if any error has been detected
190                     </para>
191                 </listitem>
192             </varlistentry>
193         </variablelist>
194     </refsection>
195     <refsection>
196         <title>Description</title>
197         <para>
198             Search for .tst files in the unit test and non-regression test library
199             execute them, and display a report about success of failures.
200             The .tst files are searched in directories SCI+"/modules/*/tests/unit_tests"
201             and SCI+"/modules/*/tests/nonreg_tests".
202         </para>
203         <para>
204             First <literal>test_run</literal> checks that a test does not produce an error.
205         </para>
206         <para>
207             Then <literal>test_run</literal> checks that the output and commands of a script are identical to the reference file. Whenever a test is executed, a <literal>.dia</literal> file is generated which contains
208             the full list of commands executed along with messages that appear in the
209             console output. When the script is done, the <literal>.dia</literal> file is compared with
210             the <literal>.dia.ref</literal> file which is expected to be in the same directory
211             as the .tst file. If the two file are different, the test fails.
212         </para>
213         <para>
214             Special tags may be inserted in the .tst file, which help to
215             control the processing of the corresponding test. These tags
216             are expected to be found in Scilab comments.
217         </para>
218         <para>These are the available tags:</para>
219         <itemizedlist>
220             <listitem>
221                 <para>
222                     &lt;-- INTERACTIVE TEST --&gt;
223                     This test will be skipped because it is interactive.
224                 </para>
225             </listitem>
226             <listitem>
227                 <para>
228                     &lt;-- LONG TIME EXECUTION --&gt;
229                     This test will be skipped because it needs long-time duration. To enable
230                     the test, call test_run with the following option: "enable_lt"
231                 </para>
232             </listitem>
233             <listitem>
234                 <para>
235                     &lt;-- NOT FIXED --&gt;
236                     This test will be skipped because it is a known, but unfixed bug.
237                 </para>
238             </listitem>
239             <listitem>
240                 <para>
241                     &lt;-- TEST WITH GRAPHIC --&gt;
242                     This test will be executed with scilab -nw. (default mode)
243                 </para>
244             </listitem>
245             <listitem>
246                 <para>
247                     &lt;-- NO TRY CATCH --&gt;
248                 </para>
249             </listitem>
250             <listitem>
251                 <para>
252                     &lt;-- NO CHECK ERROR OUTPUT --&gt;
253                     The error output file is not checked
254                 </para>
255             </listitem>
256             <listitem>
257                 <para>
258                     &lt;-- NO CHECK REF --&gt;
259                     The .dia and the .dia.ref files are not compared.
260                 </para>
261             </listitem>
262             <listitem>
263                 <para>
264                     &lt;-- ENGLISH IMPOSED --&gt;
265                     This test will be executed with the -l en_US option.
266                 </para>
267             </listitem>
268             <listitem>
269                 <para>
270                     &lt;-- FRENCH IMPOSED --&gt;
271                     This test will be executed with the -l fr_FR option.
272                 </para>
273             </listitem>
274             <listitem>
275                 <para>
276                     &lt;-- CLI SHELL MODE --&gt;
277                     (was &lt;-- JVM NOT MANDATORY --&gt; and is obsolete)
278                     This test will be executed with scilab -nwni.
279                     All tests without this, will not be executed in "mode_mwni".
280                 </para>
281             </listitem>
282             <listitem>
283                 <para>
284                     &lt;-- WINDOWS ONLY --&gt;
285                     If the operating system isn't Windows, the test is skipped.
286                 </para>
287             </listitem>
288             <listitem>
289                 <para>
290                     &lt;-- UNIX ONLY --&gt;
291                     If the operating system isn't an Unix OS, the test is skipped.
292                 </para>
293             </listitem>
294             <listitem>
295                 <para>
296                     &lt;-- LINUX ONLY --&gt;
297                     If the operating system isn't GNU/Linux, the test is skipped.
298                 </para>
299             </listitem>
300             <listitem>
301                 <para>
302                     &lt;-- MACOSX ONLY --&gt;
303                     If the operating system isn't Mac OS X, the test is skipped.
304                 </para>
305             </listitem>
306             <listitem>
307                 <para>
308                     &lt;-- XCOS TEST --&gt;
309                     This test will launch all the necessary Xcos libs. This test
310                     will be launched in nw mode.
311                 </para>
312             </listitem>
313         </itemizedlist>
314         <para>
315             Each test is executed in a separated process, created with the "host" command.
316             That enables the current command to continue, even if the test as
317             created an unstable environment. It also enables the tests to be
318             independent from one another.
319         </para>
320     </refsection>
321     <refsection>
322         <title>Platform-specific tests</title>
323         <para>
324             It may happen that the output of a test depends on the platform on which it is
325             executed. In this case, the <literal>.ref</literal> file cannot be correct for
326             all platforms and unit tests may fail for some platform. In this case, we can
327             create a default <literal>.ref</literal> and create additional <literal>.ref</literal>
328             file for each platform.
329         </para>
330         <para>
331             The various platform-specific <literal>.ref</literal> files must have one of the following extensions.
332         </para>
333         <itemizedlist>
334             <listitem>
335                 <para>
336                     <literal>.unix.dia.ref</literal> for Unix platform,
337                 </para>
338             </listitem>
339             <listitem>
340                 <para>
341                     <literal>.linux.dia.ref</literal> for GNU/Linux platform,
342                 </para>
343             </listitem>
344             <listitem>
345                 <para>
346                     <literal>.linux32.dia.ref</literal> for GNU/Linux platform with 32bits processors,
347                 </para>
348             </listitem>
349             <listitem>
350                 <para>
351                     <literal>.win.dia.ref</literal> for Windows platform,
352                 </para>
353             </listitem>
354             <listitem>
355                 <para>
356                     <literal>.win32.dia.ref</literal> for Windows platform with 32bits processors,
357                 </para>
358             </listitem>
359             <listitem>
360                 <para>
361                     <literal>.macosx.dia.ref</literal> for Mac OS X platform.
362                 </para>
363             </listitem>
364         </itemizedlist>
365         <para>
366             The algorithm is the following.
367             First, the <literal>.ref</literal> is considered. If this file does not exist,
368             the platform-specific <literal>platform.ref</literal> file is examined depending on the current platform.
369         </para>
370         <itemizedlist>
371             <listitem>
372                 <para>
373                     on Windows platforms: <literal>.win.dia.ref</literal>, <literal>.win32.dia.ref</literal>
374                 </para>
375             </listitem>
376             <listitem>
377                 <para>
378                     on Max OS X platforms: <literal>.unix.dia.ref</literal>, <literal>.macosx.dia.ref</literal>,
379                 </para>
380             </listitem>
381             <listitem>
382                 <para>
383                     on GNU/Linux platforms: <literal>.unix.dia.ref</literal>, <literal>.linux.dia.ref</literal>, <literal>.linux32.dia.ref</literal>.
384                 </para>
385             </listitem>
386         </itemizedlist>
387     </refsection>
388     <refsection>
389         <title>Examples</title>
390         <programlisting role="example"><![CDATA[
391 // Launch all tests
392 // =============================================
393
394 test_run();
395 test_run([]);
396 test_run([],[]);
397
398 // Test one or several module
399 // =============================================
400
401 // Test one module
402 test_run('time');
403
404 // Test several modules
405 test_run(['time','string']);
406
407 // Test a submodule
408 test_run('optimization|neldermead');
409
410 // Refer to a module by its path
411 test_run(SCI+'/modules/core');
412
413 // Launch a specific test
414 // =============================================
415
416 // One specific test
417 test_run('time','datenum');
418
419 // Several tests
420 test_run('time',['datenum';'calendar']);
421
422 // Skip some tests
423 // =============================================
424
425 test_run('time',['datenum';'calendar'],'skip_tests');
426
427 // Options
428 // =============================================
429
430 // does not check if the .dia and .dia.ref are equal
431 test_run('time','datenum','no_check_ref');
432
433 // Create the .dia.ref file and does not check if the .dia and .dia.ref are equal
434 test_run([],[],'create_ref');
435
436 // Does not perform the tests but displays a list of available tests
437 test_run([],[],'list');
438
439 // Display some examples about how to use this command
440 test_run([],[],'help');
441
442 // Runs only the non-regression tests, skipping unit tests
443 test_run([],[],'nonreg_test');
444
445 // Runs only the unit tests, skipping non-regression tests
446 test_run([],[],'unit_test');
447
448 // Do not check the error output (std err)
449 test_run('boolean','bug_2799','no_check_error_output');
450
451 // Combine several options
452 test_run([],[],['no_check_ref','mode_nw']);
453  ]]></programlisting>
454         
455         <programlisting role="example"><![CDATA[
456 // Run unitary tests of an external module (with his path)
457 test_run('SCI/contrib/toolbox_skeleton')
458  ]]></programlisting>
459         
460         <programlisting role="example"><![CDATA[
461 // Export to a XML Xunit file
462 test_run('boolean',[],[],TMPDIR+"/boolean_test_run.xml");
463 test_run('time','datenum',[],TMPDIR+"/time_datenum_test_run.xml");
464  ]]></programlisting>
465     </refsection>
466     <refsection>
467         <title>Internal Design</title>
468         <para>
469             The tests are performed in the temporary directory, not in the directory
470             which originally contain the tests files.
471         </para>
472         <para>
473             The .tst script is not run as is. Instead, a header and a footer are
474             inserted at the beginning and at the end of the .tst at the time
475             the script is copied into the temporary directory.
476             The role of this modification is to redirect the output messages
477             into the .dia file, so that the user can have a log file once the test
478             is performed.
479         </para>
480     </refsection>
481     <refsection>
482         <title>History</title>
483         <revhistory>
484             <revision>
485                 <revnumber>5.4.0</revnumber>
486                 <revdescription>test_run returns a status:
487                     <itemizedlist><listitem>
488                             Returns %t if no error has been detected
489                         </listitem>
490                         <listitem>
491                             Returns %f if any error has been detected
492                         </listitem>
493                     </itemizedlist>
494                     <para>
495                         <literal>show_diff</literal> and <literal>show_error</literal> added as new options
496                     </para>
497                     <para>
498                         <literal>CLI SHELL MODE</literal> tag is added. Replaces <literal>JVM NOT MANDATORY</literal> (still supported)
499                     </para>
500                     <para>
501                         <literal>test_run</literal> can work on an external module.
502                     </para>
503                     <para>
504                         Fourth argument added to export to a XML file
505                     </para>
506                 </revdescription>
507             </revision>
508             <revision>
509                 <revnumber>5.5.0</revnumber>
510                 <revdescription>32/64bits separation available</revdescription>
511             </revision>
512             <revision>
513                 <revnumber>6.0.0</revnumber>
514                 <revdescription>profiling mode added to profile execution with valgrind (Linux only)</revdescription>
515             </revision>
516         </revhistory>
517     </refsection>
518 </refentry>