3963400cdbdd441c8f8b93976019e080e357a9f8
[scilab.git] / scilab / modules / development_tools / help / en_US / test_run.xml
1 <?xml version="1.0" encoding="UTF-8"?>
2 <!--
3  * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  * Copyright (C) INRIA
5  * Copyright (C) 2009-2011 - DIGITEO - Michael Baudin
6  * Copyright (C) 2013 - Scilab Enterprises - Paul Bignier: added 32/64bits separation
7  * Copyright (C) 2012 - 2016 - Scilab Enterprises
8  * Copyright (C) 2018 - Samuel GOUGEON
9  *
10  * This file is hereby licensed under the terms of the GNU GPL v2.0,
11  * pursuant to article 5.3.4 of the CeCILL v.2.1.
12  * This file was originally licensed under the terms of the CeCILL v2.1,
13  * and continues to be available under such terms.
14  * For more information, see the COPYING file which you should have received
15  * along with this program.
16  *
17  -->
18 <refentry xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink"
19           xmlns:svg="http://www.w3.org/2000/svg" xmlns:ns5="http://www.w3.org/1999/xhtml"
20           xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:db="http://docbook.org/ns/docbook"
21           xmlns:scilab="http://www.scilab.org" xml:id="test_run" xml:lang="en">
22     <refnamediv>
23         <refname>test_run</refname>
24         <refpurpose>
25             runs unit tests and non-regression tests of a module or in a directory
26         </refpurpose>
27     </refnamediv>
28     <refsynopsisdiv>
29         <title>Syntax</title>
30         <synopsis>
31             status = test_run()
32             status = test_run(module)
33             status = test_run(module, test_name)
34             status = test_run(module, test_name, options, exportToFile)
35         </synopsis>
36     </refsynopsisdiv>
37     <refsection>
38         <title>Arguments</title>
39         <variablelist>
40             <varlistentry>
41                 <term>module</term>
42                 <listitem>
43                     <para>
44                         A String array or <literal>[]</literal> (or equivalently <literal>"[]"</literal>). Name of the modules or directory for the tests, all internal modules if <literal>[]</literal>
45                     </para>
46                     <itemizedlist>
47                         <listitem>
48                             <para>
49                                 the name of an internal Scilab module ("core", "time", ...), a sub-module (e.g. "optimization|neldermead").
50                             </para>
51                         </listitem>
52                         <listitem>
53                             <para>
54                                 the name of an ATOMS module ("module_lycee", "nisp", ...). To be taken into account, the module must be loaded when test_run() is called.
55                             </para>
56                         </listitem>
57                         <listitem>
58                             <para>
59                                 the absolute directory path of a module containing test/unit_tests or test/nonreg_tests.
60                             </para>
61                         </listitem>
62                     </itemizedlist>
63                 </listitem>
64             </varlistentry>
65             <varlistentry>
66                 <term>test_name</term>
67                 <listitem>
68                     <para>
69                         A string array or <literal>[]</literal> or <literal>"[]"</literal>:
70                         The names of the tests to execute during this run. If <varname>test_name</varname>
71                         is <literal>[]</literal>, all tests found in the module or in the directory are executed.
72                     </para>
73                     <para>
74                         The wildcard * can be used, like in <literal>*sin</literal>,
75                         <literal>*sin</literal>, or <literal>*sin*</literal>.
76                     </para>
77                 </listitem>
78             </varlistentry>
79             <varlistentry>
80                 <term>options</term>
81                 <listitem>
82                     <para>
83                         A string array or <literal>[]</literal> or <literal>"[]"</literal>. The options for the tests during this run, default options if <literal>[]</literal> or <literal>"[]"</literal>.
84                     </para>
85                     <variablelist> or
86                         <varlistentry>
87                             <term>"no_check_ref"</term>
88                             <listitem>
89                                 <para>does not check if the .dia and .dia.ref are equal</para>
90                             </listitem>
91                         </varlistentry>
92                         <varlistentry>
93                             <term>"no_check_error_output"</term>
94                             <listitem>
95                                 <para>The error output stream is not checked. This option can be used when Scilab complains about the localization being not available.</para>
96                             </listitem>
97                         </varlistentry>
98                         <varlistentry>
99                             <term>"create_ref"</term>
100                             <listitem>
101                                 <para>creates the .dia.ref file (for tests not having the
102                                   <literal>&lt;-- NO CHECK REF --></literal> flag)
103                                   and does not check if the .dia and .dia.ref are equal.
104                                   </para>
105                             </listitem>
106                         </varlistentry>
107                         <varlistentry>
108                             <term>"show_error"</term>
109                             <listitem>
110                                 <para>If an error occurs, shows the last 10 lines of the execution</para>
111                             </listitem>
112                         </varlistentry>
113                         <varlistentry>
114                             <term>"show_diff"</term>
115                             <listitem>
116                                 <para>
117                                     If a difference is found, shows the result of the command <literal>diff -u</literal>
118                                 </para>
119                             </listitem>
120                         </varlistentry>
121                         <varlistentry>
122                             <term>"list"</term>
123                             <listitem>
124                                 <para>Does not perform the tests but displays a list of available tests</para>
125                             </listitem>
126                         </varlistentry>
127                         <varlistentry>
128                             <term>"help"</term>
129                             <listitem>
130                                 <para>display some examples about how to use this command</para>
131                             </listitem>
132                         </varlistentry>
133                         <varlistentry>
134                             <term>"mode_nw"</term>
135                             <listitem>
136                                 <para>Add the "-nw" option to the launch</para>
137                             </listitem>
138                         </varlistentry>
139                         <varlistentry>
140                             <term>"mode_nwni"</term>
141                             <listitem>
142                                 <para>Add the "-nwni" option to the launch</para>
143                             </listitem>
144                         </varlistentry>
145                         <varlistentry>
146                             <term>"mode_nwni_profiling"</term>
147                             <listitem>
148                                 <para>Add the "-nwni -profiling" option to the launch for detect valgrind error (Linux only)</para>
149                             </listitem>
150                         </varlistentry>
151                         <varlistentry>
152                             <term>"nonreg_tests"</term>
153                             <listitem>
154                                 <para>runs only the non-regression tests, skipping unit tests</para>
155                             </listitem>
156                         </varlistentry>
157                         <varlistentry>
158                             <term>"unit_tests"</term>
159                             <listitem>
160                                 <para>Runs only the unit tests, skipping non-regression tests</para>
161                             </listitem>
162                         </varlistentry>
163                         <varlistentry>
164                             <term>"skip_tests"</term>
165                             <listitem>
166                                 <para>
167                                     Skip the tests specified in <term>test_name</term>.
168                                 </para>
169                             </listitem>
170                         </varlistentry>
171                         <varlistentry>
172                             <term>"enable_lt"</term>
173                             <listitem>
174                                 <para>Enable long-time execution tests</para>
175                             </listitem>
176                         </varlistentry>
177                         <varlistentry>
178                             <term>"short_summary"</term>
179                             <listitem>
180                                 <para>Does not display statistics nor execution time after execution (only number of executed, passed, failed and skipped tests will be displayed on a single line).</para>
181                             </listitem>
182                         </varlistentry>
183                     </variablelist>
184                 </listitem>
185             </varlistentry>
186             <varlistentry>
187                 <term>exportToFile</term>
188                 <listitem>
189                     <para>
190                         Export to a XML file the result of the test. This file follows the specification of the XUnit format.
191                         Note that the usage of this option enables <literal>show_diff</literal> and <literal>show_error</literal>.
192                     </para>
193                     <para>
194                         If the file pointed by <varname>exportToFile</varname> already exists,
195                         the new results are appended to the existing file.
196                     </para>
197                 </listitem>
198             </varlistentry>
199             <varlistentry>
200                 <term>status</term>
201                 <listitem>
202                     <para>
203                         Boolean value
204                         Returns %t if no error has been detected
205                         Returns %f if any error has been detected
206                     </para>
207                 </listitem>
208             </varlistentry>
209         </variablelist>
210     </refsection>
211     <refsection>
212         <title>Description</title>
213         <para>
214             Search for .tst files in the unit test and non-regression test library,
215             execute them, and display a report about success or failures.
216             The .tst files are searched in directories ~/tests/unit_tests"
217             and ~/tests/nonreg_tests" where "~" is the root directory of targeted modules.
218         </para>
219         <para>
220             First <literal>test_run</literal> checks that a test does not produce an error.
221         </para>
222         <para>
223             Then <literal>test_run</literal> checks that the output and commands of a script are
224             identical to the reference file. Whenever a test is executed, a <literal>.dia</literal>
225             file is generated which contains
226             the full list of commands executed along with messages that appear in the
227             console output. When the script is done, the <literal>.dia</literal> file is compared with
228             the <literal>.dia.ref</literal> file which is expected to be in the same directory
229             as the .tst file. If the two file are different, the test fails.
230         </para>
231         <para>
232             Special tags may be inserted in the .tst file, which help to
233             control the processing of the corresponding test. These tags
234             are expected to be found in Scilab comments.
235         </para>
236         <para>These are the available tags:</para>
237         <itemizedlist>
238             <listitem>
239                 <para>
240                     &lt;-- INTERACTIVE TEST --&gt;
241                     This test will be skipped because it is interactive.
242                 </para>
243             </listitem>
244             <listitem>
245                 <para>
246                     &lt;-- LONG TIME EXECUTION --&gt;
247                     This test will be skipped because it needs long-time duration. To enable
248                     the test, call test_run with the following option: "enable_lt"
249                 </para>
250             </listitem>
251             <listitem>
252                 <para>
253                     &lt;-- NOT FIXED --&gt;
254                     This test will be skipped because it is a known, but unfixed bug.
255                 </para>
256             </listitem>
257             <listitem>
258                 <para>
259                     &lt;-- TEST WITH GRAPHIC --&gt;
260                     This test will be executed with scilab -nw. (default mode)
261                 </para>
262             </listitem>
263             <listitem>
264                 <para>
265                     &lt;-- NO TRY CATCH --&gt;
266                 </para>
267             </listitem>
268             <listitem>
269                 <para>
270                     &lt;-- NO CHECK ERROR OUTPUT --&gt;
271                     The error output file is not checked
272                 </para>
273             </listitem>
274             <listitem>
275                 <para>
276                     &lt;-- NO CHECK REF --&gt;
277                     The .dia and the .dia.ref files are not compared.
278                 </para>
279             </listitem>
280             <listitem>
281                 <para>
282                     &lt;-- ENGLISH IMPOSED --&gt;
283                     This test will be executed with the -l en_US option.
284                 </para>
285             </listitem>
286             <listitem>
287                 <para>
288                     &lt;-- FRENCH IMPOSED --&gt;
289                     This test will be executed with the -l fr_FR option.
290                 </para>
291             </listitem>
292             <listitem>
293                 <para>
294                     &lt;-- CLI SHELL MODE --&gt;
295                     (was &lt;-- JVM NOT MANDATORY --&gt; and is obsolete)
296                     This test will be executed with scilab -nwni.
297                     All tests without this, will not be executed in "mode_mwni".
298                 </para>
299             </listitem>
300             <listitem>
301                 <para>
302                     &lt;-- WINDOWS ONLY --&gt;
303                     If the operating system isn't Windows, the test is skipped.
304                 </para>
305             </listitem>
306             <listitem>
307                 <para>
308                     &lt;-- UNIX ONLY --&gt;
309                     If the operating system isn't an Unix OS, the test is skipped.
310                 </para>
311             </listitem>
312             <listitem>
313                 <para>
314                     &lt;-- LINUX ONLY --&gt;
315                     If the operating system isn't GNU/Linux, the test is skipped.
316                 </para>
317             </listitem>
318             <listitem>
319                 <para>
320                     &lt;-- MACOSX ONLY --&gt;
321                     If the operating system isn't Mac OS X, the test is skipped.
322                 </para>
323             </listitem>
324             <listitem>
325                 <para>
326                     &lt;-- XCOS TEST --&gt;
327                     This test will launch all the necessary Xcos libs. This test
328                     will be launched in nw mode.
329                 </para>
330             </listitem>
331         </itemizedlist>
332         <para>
333             Each test is executed in a separated process, created with the "host" command.
334             That enables the current command to continue, even if the test as
335             created an unstable environment. It also enables the tests to be
336             independent from one another.
337         </para>
338     </refsection>
339     <refsection>
340         <title>Platform-specific tests</title>
341         <para>
342             It may happen that the output of a test depends on the platform on which it is
343             executed. In this case, the <literal>.ref</literal> file cannot be correct for
344             all platforms and unit tests may fail for some platform. In this case, we can
345             create a default <literal>.ref</literal> and create additional <literal>.ref</literal>
346             file for each platform.
347         </para>
348         <para>
349             The various platform-specific <literal>.ref</literal> files must have one of the following extensions.
350         </para>
351         <itemizedlist>
352             <listitem>
353                 <para>
354                     <literal>.unix.dia.ref</literal> for Unix platform,
355                 </para>
356             </listitem>
357             <listitem>
358                 <para>
359                     <literal>.linux.dia.ref</literal> for GNU/Linux platform,
360                 </para>
361             </listitem>
362             <listitem>
363                 <para>
364                     <literal>.linux32.dia.ref</literal> for GNU/Linux platform with 32bits processors,
365                 </para>
366             </listitem>
367             <listitem>
368                 <para>
369                     <literal>.win.dia.ref</literal> for Windows platform,
370                 </para>
371             </listitem>
372             <listitem>
373                 <para>
374                     <literal>.win32.dia.ref</literal> for Windows platform with 32bits processors,
375                 </para>
376             </listitem>
377             <listitem>
378                 <para>
379                     <literal>.macosx.dia.ref</literal> for Mac OS X platform.
380                 </para>
381             </listitem>
382         </itemizedlist>
383         <para>
384             The algorithm is the following.
385             First, the <literal>.ref</literal> is considered. If this file does not exist,
386             the platform-specific <literal>platform.ref</literal> file is examined depending on the current platform.
387         </para>
388         <itemizedlist>
389             <listitem>
390                 <para>
391                     on Windows platforms: <literal>.win.dia.ref</literal>, <literal>.win32.dia.ref</literal>
392                 </para>
393             </listitem>
394             <listitem>
395                 <para>
396                     on Max OS X platforms: <literal>.unix.dia.ref</literal>, <literal>.macosx.dia.ref</literal>,
397                 </para>
398             </listitem>
399             <listitem>
400                 <para>
401                     on GNU/Linux platforms: <literal>.unix.dia.ref</literal>, <literal>.linux.dia.ref</literal>, <literal>.linux32.dia.ref</literal>.
402                 </para>
403             </listitem>
404         </itemizedlist>
405     </refsection>
406     <refsection>
407         <title>Examples</title>
408         <programlisting role="example"><![CDATA[
409 // Launch all tests
410 // This may take some time
411 // =============================================
412
413 // test_run();
414 // test_run([]);
415 // test_run([],[]);
416 // test_run("[]","[]");
417 // test_run [] [];
418
419 // Test one or several module
420 // =============================================
421
422 // Test one module
423 test_run('time');
424
425 // Test several modules
426 test_run(['time','string']);
427
428 // Test a submodule
429 test_run('optimization|neldermead');
430
431 // Refer to a module by its path
432 test_run(SCI+'/modules/core');
433
434 // Launch a specific test
435 // =============================================
436
437 // One specific test
438 test_run('time','datenum');
439
440 // Several tests
441 test_run('time',['datenum';'calendar']);
442
443 // Skip some tests
444 // =============================================
445
446 test_run('time',['datenum';'calendar'],'skip_tests');
447
448 // Options
449 // =============================================
450
451 // does not check if the .dia and .dia.ref are equal
452 test_run('time','datenum','no_check_ref');
453
454 // Create the .dia.ref file and does not check if the .dia and .dia.ref are equal
455 test_run([],[],'create_ref');
456
457 // Does not perform the tests but displays a list of available tests
458 test_run([],[],'list');
459
460 // Display some examples about how to use this command
461 test_run([],[],'help');
462
463 // Runs only the non-regression tests, skipping unit tests
464 test_run([],[],'nonreg_test');
465
466 // Runs only the unit tests, skipping non-regression tests
467 test_run([],[],'unit_test');
468
469 // Do not check the error output (std err)
470 test_run('boolean','bug_2799','no_check_error_output');
471
472 // Combine several options
473 test_run([],[],['no_check_ref','mode_nw']);
474
475 // Console mode
476 test_run time [] no_check_ref //tests time module with no_check_ref option
477  ]]></programlisting>
478
479         <programlisting role="example"><![CDATA[
480 // Run unitary tests of an external module (with his path)
481 test_run('SCI/contrib/toolbox_skeleton')
482  ]]></programlisting>
483
484         <programlisting role="example"><![CDATA[
485 // Export to a XML Xunit file
486 test_run('boolean',[],[],TMPDIR+"/boolean_test_run.xml");
487 test_run('time','datenum',[],TMPDIR+"/time_datenum_test_run.xml");
488  ]]></programlisting>
489         <para>
490             <emphasis role="bold">Selections with wildcard *:</emphasis>
491         </para>
492         <programlisting role="example"><![CDATA[
493 test_run elementary_functions *space
494 test_run elementary_functions dec2*
495 test_run string *ascii*
496  ]]></programlisting>
497     <screen><![CDATA[
498 --> test_run elementary_functions *space
499    TMPDIR = C:\MyPath\AppData\Local\Temp\SCI_TMP_3668_1147
500
501    001/002 - [elementary_functions] logspace....................passed
502    002/002 - [elementary_functions] linspace....................passed
503    --------------------------------------------------------------------------
504    Summary
505 ../..
506
507 --> test_run elementary_functions dec2*
508    TMPDIR = C:\MyPath\AppData\Local\Temp\SCI_TMP_3668_1147
509
510    001/004 - [elementary_functions] dec2oct.....................passed
511    002/004 - [elementary_functions] dec2hex.....................passed
512    003/004 - [elementary_functions] dec2bin.....................passed
513    004/004 - [elementary_functions] dec2base....................passed
514    --------------------------------------------------------------------------
515    Summary
516 ../..
517
518 --> test_run string *ascii*
519    TMPDIR = C:\MyPath\AppData\Local\Temp\SCI_TMP_3668_1147
520
521    001/003 - [string] isascii...................................passed
522    002/003 - [string] asciimat..................................passed
523    003/003 - [string] ascii.....................................passed
524    --------------------------------------------------------------------------
525    Summary
526 ../..
527 ]]></screen>
528     </refsection>
529     <refsection>
530         <title>Internal Design</title>
531         <para>
532             The tests are performed in the temporary directory, not in the directory
533             which originally contain the tests files.
534         </para>
535         <para>
536             The .tst script is not run as is. Instead, a header and a footer are
537             inserted at the beginning and at the end of the .tst at the time
538             the script is copied into the temporary directory.
539             The role of this modification is to redirect the output messages
540             into the .dia file, so that the user can have a log file once the test
541             is performed.
542         </para>
543         <para>
544             An execution timeout delay (watchdog timer) is setup to 5 minutes
545             for each regular test. To ignore this timeout use the long-time
546             execution (<literal>LONG TIME EXECUTION</literal>) flag.
547         </para>
548     </refsection>
549     <refsection>
550         <title>History</title>
551         <revhistory>
552             <revision>
553                 <revnumber>5.4.0</revnumber>
554                 <revdescription>test_run returns a status:
555                     <itemizedlist><listitem>
556                             Returns %t if no error has been detected
557                         </listitem>
558                         <listitem>
559                             Returns %f if any error has been detected
560                         </listitem>
561                     </itemizedlist>
562                     <para>
563                         <literal>show_diff</literal> and <literal>show_error</literal> added as new options
564                     </para>
565                     <para>
566                         <literal>CLI SHELL MODE</literal> tag is added. Replaces <literal>JVM NOT MANDATORY</literal> (still supported)
567                     </para>
568                     <para>
569                         <literal>test_run</literal> can work on an external module.
570                     </para>
571                     <para>
572                         Fourth argument added to export to a XML file
573                     </para>
574                 </revdescription>
575             </revision>
576             <revision>
577                 <revnumber>5.5.0</revnumber>
578                 <revdescription>32/64bits separation available</revdescription>
579             </revision>
580             <revision>
581                 <revnumber>6.0.0</revnumber>
582                 <revdescription>
583                     <para>profiling mode added to profile execution with valgrind (Linux only)</para>
584                     <para>
585                         timeout delay (watchdog timer) set to 5 minutes for single tests without <literal>LONG TIME EXECUTION</literal>
586                     </para>
587                 </revdescription>
588             </revision>
589             <revision>
590                 <revnumber>6.0.2</revnumber>
591                 <revdescription>
592                     <para>Tests names with the * wildcard like sin*, *sin, or *sin* are now allowed</para>
593                 </revdescription>
594             </revision>
595         </revhistory>
596     </refsection>
597 </refentry>