scilab/modules/development_tools/help/en_US/test_run.xml

   1 <?xml version="1.0" encoding="UTF-8"?>
   2 <!--
   3  * Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
   4  * Copyright (C) INRIA
   5  * Copyright (C) 2009-2011 - DIGITEO - Michael Baudin
   6  * Copyright (C) 2013 - Scilab Enterprises - Paul Bignier: added 32/64bits separation
   7  * Copyright (C) 2012 - 2016 - Scilab Enterprises
   8  * Copyright (C) 2018 - Samuel GOUGEON
   9  *
  10  * This file is hereby licensed under the terms of the GNU GPL v2.0,
  11  * pursuant to article 5.3.4 of the CeCILL v.2.1.
  12  * This file was originally licensed under the terms of the CeCILL v2.1,
  13  * and continues to be available under such terms.
  14  * For more information, see the COPYING file which you should have received
  15  * along with this program.
  16  *
  17  -->
  18 <refentry xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink"
  19           xmlns:svg="http://www.w3.org/2000/svg" xmlns:ns5="http://www.w3.org/1999/xhtml"
  20           xmlns:mml="http://www.w3.org/1998/Math/MathML" xmlns:db="http://docbook.org/ns/docbook"
  21           xmlns:scilab="http://www.scilab.org" xml:id="test_run" xml:lang="en">
  22     <refnamediv>
  23         <refname>test_run</refname>
  24         <refpurpose>
  25             runs unit tests and non-regression tests of a module or in a directory
  26         </refpurpose>
  27     </refnamediv>
  28     <refsynopsisdiv>
  29         <title>Syntax</title>
  30         <synopsis>
  31             status = test_run()
  32             status = test_run(module)
  33             status = test_run(module, test_name)
  34             status = test_run(module, test_name, options, exportToFile)
  35         </synopsis>
  36     </refsynopsisdiv>
  37     <refsection>
  38         <title>Arguments</title>
  39         <variablelist>
  40             <varlistentry>
  41                 <term>module</term>
  42                 <listitem>
  43                     <para>
  44                         A String array or <literal>[]</literal> (or equivalently <literal>"[]"</literal>). Name of the modules or directory for the tests, all internal modules if <literal>[]</literal>
  45                     </para>
  46                     <itemizedlist>
  47                         <listitem>
  48                             <para>
  49                                 the name of an internal Scilab module ("core", "time", ...), a sub-module (e.g. "optimization|neldermead").
  50                             </para>
  51                         </listitem>
  52                         <listitem>
  53                             <para>
  54                                 the name of an ATOMS module ("module_lycee", "nisp", ...). To be taken into account, the module must be loaded when test_run() is called.
  55                             </para>
  56                         </listitem>
  57                         <listitem>
  58                             <para>
  59                                 the absolute directory path of a module containing test/unit_tests or test/nonreg_tests.
  60                             </para>
  61                         </listitem>
  62                     </itemizedlist>
  63                 </listitem>
  64             </varlistentry>
  65             <varlistentry>
  66                 <term>test_name</term>
  67                 <listitem>
  68                     <para>
  69                         A string array or <literal>[]</literal> or <literal>"[]"</literal>:
  70                         The names of the tests to execute during this run. If <varname>test_name</varname>
  71                         is <literal>[]</literal>, all tests found in the module or in the directory are executed.
  72                     </para>
  73                     <para>
  74                         The wildcard * can be used, like in <literal>*sin</literal>,
  75                         <literal>*sin</literal>, or <literal>*sin*</literal>.
  76                     </para>
  77                 </listitem>
  78             </varlistentry>
  79             <varlistentry>
  80                 <term>options</term>
  81                 <listitem>
  82                     <para>
  83                         A string array or <literal>[]</literal> or <literal>"[]"</literal>. The options for the tests during this run, default options if <literal>[]</literal> or <literal>"[]"</literal>.
  84                     </para>
  85                     <variablelist> or
  86                         <varlistentry>
  87                             <term>"no_check_ref"</term>
  88                             <listitem>
  89                                 <para>does not check if the .dia and .dia.ref are equal</para>
  90                             </listitem>
  91                         </varlistentry>
  92                         <varlistentry>
  93                             <term>"no_check_error_output"</term>
  94                             <listitem>
  95                                 <para>The error output stream is not checked. This option can be used when Scilab complains about the localization being not available.</para>
  96                             </listitem>
  97                         </varlistentry>
  98                         <varlistentry>
  99                             <term>"create_ref"</term>
 100                             <listitem>
 101                                 <para>creates the .dia.ref file (for tests not having the
 102                                   <literal>&lt;-- NO CHECK REF --></literal> flag)
 103                                   and does not check if the .dia and .dia.ref are equal.
 104                                   </para>
 105                             </listitem>
 106                         </varlistentry>
 107                         <varlistentry>
 108                             <term>"show_error"</term>
 109                             <listitem>
 110                                 <para>If an error occurs, shows the last 10 lines of the execution</para>
 111                             </listitem>
 112                         </varlistentry>
 113                         <varlistentry>
 114                             <term>"show_diff"</term>
 115                             <listitem>
 116                                 <para>
 117                                     If a difference is found, shows the result of the command <literal>diff -u</literal>
 118                                 </para>
 119                             </listitem>
 120                         </varlistentry>
 121                         <varlistentry>
 122                             <term>"list"</term>
 123                             <listitem>
 124                                 <para>Does not perform the tests but displays a list of available tests</para>
 125                             </listitem>
 126                         </varlistentry>
 127                         <varlistentry>
 128                             <term>"help"</term>
 129                             <listitem>
 130                                 <para>display some examples about how to use this command</para>
 131                             </listitem>
 132                         </varlistentry>
 133                         <varlistentry>
 134                             <term>"mode_nw"</term>
 135                             <listitem>
 136                                 <para>Add the "-nw" option to the launch</para>
 137                             </listitem>
 138                         </varlistentry>
 139                         <varlistentry>
 140                             <term>"mode_nwni"</term>
 141                             <listitem>
 142                                 <para>Add the "-nwni" option to the launch</para>
 143                             </listitem>
 144                         </varlistentry>
 145                         <varlistentry>
 146                             <term>"mode_nwni_profiling"</term>
 147                             <listitem>
 148                                 <para>Add the "-nwni -profiling" option to the launch for detect valgrind error (Linux only)</para>
 149                             </listitem>
 150                         </varlistentry>
 151                         <varlistentry>
 152                             <term>"nonreg_tests"</term>
 153                             <listitem>
 154                                 <para>runs only the non-regression tests, skipping unit tests</para>
 155                             </listitem>
 156                         </varlistentry>
 157                         <varlistentry>
 158                             <term>"unit_tests"</term>
 159                             <listitem>
 160                                 <para>Runs only the unit tests, skipping non-regression tests</para>
 161                             </listitem>
 162                         </varlistentry>
 163                         <varlistentry>
 164                             <term>"skip_tests"</term>
 165                             <listitem>
 166                                 <para>
 167                                     Skip the tests specified in <term>test_name</term>.
 168                                 </para>
 169                             </listitem>
 170                         </varlistentry>
 171                         <varlistentry>
 172                             <term>"enable_lt"</term>
 173                             <listitem>
 174                                 <para>Enable long-time execution tests</para>
 175                             </listitem>
 176                         </varlistentry>
 177                         <varlistentry>
 178                             <term>"short_summary"</term>
 179                             <listitem>
 180                                 <para>Does not display statistics nor execution time after execution (only number of executed, passed, failed and skipped tests will be displayed on a single line).</para>
 181                             </listitem>
 182                         </varlistentry>
 183                     </variablelist>
 184                 </listitem>
 185             </varlistentry>
 186             <varlistentry>
 187                 <term>exportToFile</term>
 188                 <listitem>
 189                     <para>
 190                         Export to a XML file the result of the test. This file follows the specification of the XUnit format.
 191                         Note that the usage of this option enables <literal>show_diff</literal> and <literal>show_error</literal>.
 192                     </para>
 193                     <para>
 194                         If the file pointed by <varname>exportToFile</varname> already exists,
 195                         the new results are appended to the existing file.
 196                     </para>
 197                 </listitem>
 198             </varlistentry>
 199             <varlistentry>
 200                 <term>status</term>
 201                 <listitem>
 202                     <para>
 203                         Boolean value
 204                         Returns %t if no error has been detected
 205                         Returns %f if any error has been detected
 206                     </para>
 207                 </listitem>
 208             </varlistentry>
 209         </variablelist>
 210     </refsection>
 211     <refsection>
 212         <title>Description</title>
 213         <para>
 214             Search for .tst files in the unit test and non-regression test library,
 215             execute them, and display a report about success or failures.
 216             The .tst files are searched in directories ~/tests/unit_tests"
 217             and ~/tests/nonreg_tests" where "~" is the root directory of targeted modules.
 218         </para>
 219         <para>
 220             First <literal>test_run</literal> checks that a test does not produce an error.
 221         </para>
 222         <para>
 223             Then <literal>test_run</literal> checks that the output and commands of a script are
 224             identical to the reference file. Whenever a test is executed, a <literal>.dia</literal>
 225             file is generated which contains
 226             the full list of commands executed along with messages that appear in the
 227             console output. When the script is done, the <literal>.dia</literal> file is compared with
 228             the <literal>.dia.ref</literal> file which is expected to be in the same directory
 229             as the .tst file. If the two file are different, the test fails.
 230         </para>
 231         <para>
 232             Special tags may be inserted in the .tst file, which help to
 233             control the processing of the corresponding test. These tags
 234             are expected to be found in Scilab comments.
 235         </para>
 236         <para>These are the available tags:</para>
 237         <itemizedlist>
 238             <listitem>
 239                 <para>
 240                     &lt;-- INTERACTIVE TEST --&gt;
 241                     This test will be skipped because it is interactive.
 242                 </para>
 243             </listitem>
 244             <listitem>
 245                 <para>
 246                     &lt;-- LONG TIME EXECUTION --&gt;
 247                     This test will be skipped because it needs long-time duration. To enable
 248                     the test, call test_run with the following option: "enable_lt"
 249                 </para>
 250             </listitem>
 251             <listitem>
 252                 <para>
 253                     &lt;-- NOT FIXED --&gt;
 254                     This test will be skipped because it is a known, but unfixed bug.
 255                 </para>
 256             </listitem>
 257             <listitem>
 258                 <para>
 259                     &lt;-- TEST WITH GRAPHIC --&gt;
 260                     This test will be executed with scilab -nw. (default mode)
 261                 </para>
 262             </listitem>
 263             <listitem>
 264                 <para>
 265                     &lt;-- NO TRY CATCH --&gt;
 266                 </para>
 267             </listitem>
 268             <listitem>
 269                 <para>
 270                     &lt;-- NO CHECK ERROR OUTPUT --&gt;
 271                     The error output file is not checked
 272                 </para>
 273             </listitem>
 274             <listitem>
 275                 <para>
 276                     &lt;-- NO CHECK REF --&gt;
 277                     The .dia and the .dia.ref files are not compared.
 278                 </para>
 279             </listitem>
 280             <listitem>
 281                 <para>
 282                     &lt;-- ENGLISH IMPOSED --&gt;
 283                     This test will be executed with the -l en_US option.
 284                 </para>
 285             </listitem>
 286             <listitem>
 287                 <para>
 288                     &lt;-- FRENCH IMPOSED --&gt;
 289                     This test will be executed with the -l fr_FR option.
 290                 </para>
 291             </listitem>
 292             <listitem>
 293                 <para>
 294                     &lt;-- CLI SHELL MODE --&gt;
 295                     (was &lt;-- JVM NOT MANDATORY --&gt; and is obsolete)
 296                     This test will be executed with scilab -nwni.
 297                     All tests without this, will not be executed in "mode_mwni".
 298                 </para>
 299             </listitem>
 300             <listitem>
 301                 <para>
 302                     &lt;-- WINDOWS ONLY --&gt;
 303                     If the operating system isn't Windows, the test is skipped.
 304                 </para>
 305             </listitem>
 306             <listitem>
 307                 <para>
 308                     &lt;-- UNIX ONLY --&gt;
 309                     If the operating system isn't an Unix OS, the test is skipped.
 310                 </para>
 311             </listitem>
 312             <listitem>
 313                 <para>
 314                     &lt;-- LINUX ONLY --&gt;
 315                     If the operating system isn't GNU/Linux, the test is skipped.
 316                 </para>
 317             </listitem>
 318             <listitem>
 319                 <para>
 320                     &lt;-- MACOSX ONLY --&gt;
 321                     If the operating system isn't Mac OS X, the test is skipped.
 322                 </para>
 323             </listitem>
 324             <listitem>
 325                 <para>
 326                     &lt;-- XCOS TEST --&gt;
 327                     This test will launch all the necessary Xcos libs. This test
 328                     will be launched in nw mode.
 329                 </para>
 330             </listitem>
 331         </itemizedlist>
 332         <para>
 333             Each test is executed in a separated process, created with the "host" command.
 334             That enables the current command to continue, even if the test as
 335             created an unstable environment. It also enables the tests to be
 336             independent from one another.
 337         </para>
 338     </refsection>
 339     <refsection>
 340         <title>Platform-specific tests</title>
 341         <para>
 342             It may happen that the output of a test depends on the platform on which it is
 343             executed. In this case, the <literal>.ref</literal> file cannot be correct for
 344             all platforms and unit tests may fail for some platform. In this case, we can
 345             create a default <literal>.ref</literal> and create additional <literal>.ref</literal>
 346             file for each platform.
 347         </para>
 348         <para>
 349             The various platform-specific <literal>.ref</literal> files must have one of the following extensions.
 350         </para>
 351         <itemizedlist>
 352             <listitem>
 353                 <para>
 354                     <literal>.unix.dia.ref</literal> for Unix platform,
 355                 </para>
 356             </listitem>
 357             <listitem>
 358                 <para>
 359                     <literal>.linux.dia.ref</literal> for GNU/Linux platform,
 360                 </para>
 361             </listitem>
 362             <listitem>
 363                 <para>
 364                     <literal>.linux32.dia.ref</literal> for GNU/Linux platform with 32bits processors,
 365                 </para>
 366             </listitem>
 367             <listitem>
 368                 <para>
 369                     <literal>.win.dia.ref</literal> for Windows platform,
 370                 </para>
 371             </listitem>
 372             <listitem>
 373                 <para>
 374                     <literal>.win32.dia.ref</literal> for Windows platform with 32bits processors,
 375                 </para>
 376             </listitem>
 377             <listitem>
 378                 <para>
 379                     <literal>.macosx.dia.ref</literal> for Mac OS X platform.
 380                 </para>
 381             </listitem>
 382         </itemizedlist>
 383         <para>
 384             The algorithm is the following.
 385             First, the <literal>.ref</literal> is considered. If this file does not exist,
 386             the platform-specific <literal>platform.ref</literal> file is examined depending on the current platform.
 387         </para>
 388         <itemizedlist>
 389             <listitem>
 390                 <para>
 391                     on Windows platforms: <literal>.win.dia.ref</literal>, <literal>.win32.dia.ref</literal>
 392                 </para>
 393             </listitem>
 394             <listitem>
 395                 <para>
 396                     on Max OS X platforms: <literal>.unix.dia.ref</literal>, <literal>.macosx.dia.ref</literal>,
 397                 </para>
 398             </listitem>
 399             <listitem>
 400                 <para>
 401                     on GNU/Linux platforms: <literal>.unix.dia.ref</literal>, <literal>.linux.dia.ref</literal>, <literal>.linux32.dia.ref</literal>.
 402                 </para>
 403             </listitem>
 404         </itemizedlist>
 405     </refsection>
 406     <refsection>
 407         <title>Examples</title>
 408         <programlisting role="example"><![CDATA[
 409 // Launch all tests
 410 // This may take some time
 411 // =============================================
 412
 413 // test_run();
 414 // test_run([]);
 415 // test_run([],[]);
 416 // test_run("[]","[]");
 417 // test_run [] [];
 418
 419 // Test one or several module
 420 // =============================================
 421
 422 // Test one module
 423 test_run('time');
 424
 425 // Test several modules
 426 test_run(['time','string']);
 427
 428 // Test a submodule
 429 test_run('optimization|neldermead');
 430
 431 // Refer to a module by its path
 432 test_run(SCI+'/modules/core');
 433
 434 // Launch a specific test
 435 // =============================================
 436
 437 // One specific test
 438 test_run('time','datenum');
 439
 440 // Several tests
 441 test_run('time',['datenum';'calendar']);
 442
 443 // Skip some tests
 444 // =============================================
 445
 446 test_run('time',['datenum';'calendar'],'skip_tests');
 447
 448 // Options
 449 // =============================================
 450
 451 // does not check if the .dia and .dia.ref are equal
 452 test_run('time','datenum','no_check_ref');
 453
 454 // Create the .dia.ref file and does not check if the .dia and .dia.ref are equal
 455 test_run([],[],'create_ref');
 456
 457 // Does not perform the tests but displays a list of available tests
 458 test_run([],[],'list');
 459
 460 // Display some examples about how to use this command
 461 test_run([],[],'help');
 462
 463 // Runs only the non-regression tests, skipping unit tests
 464 test_run([],[],'nonreg_test');
 465
 466 // Runs only the unit tests, skipping non-regression tests
 467 test_run([],[],'unit_test');
 468
 469 // Do not check the error output (std err)
 470 test_run('boolean','bug_2799','no_check_error_output');
 471
 472 // Combine several options
 473 test_run([],[],['no_check_ref','mode_nw']);
 474
 475 // Console mode
 476 test_run time [] no_check_ref //tests time module with no_check_ref option
 477  ]]></programlisting>
 478
 479         <programlisting role="example"><![CDATA[
 480 // Run unitary tests of an external module (with his path)
 481 test_run('SCI/contrib/toolbox_skeleton')
 482  ]]></programlisting>
 483
 484         <programlisting role="example"><![CDATA[
 485 // Export to a XML Xunit file
 486 test_run('boolean',[],[],TMPDIR+"/boolean_test_run.xml");
 487 test_run('time','datenum',[],TMPDIR+"/time_datenum_test_run.xml");
 488  ]]></programlisting>
 489         <para>
 490             <emphasis role="bold">Selections with wildcard *:</emphasis>
 491         </para>
 492         <programlisting role="example"><![CDATA[
 493 test_run elementary_functions *space
 494 test_run elementary_functions dec2*
 495 test_run string *ascii*
 496  ]]></programlisting>
 497     <screen><![CDATA[
 498 --> test_run elementary_functions *space
 499    TMPDIR = C:\MyPath\AppData\Local\Temp\SCI_TMP_3668_1147
 500
 501    001/002 - [elementary_functions] logspace....................passed
 502    002/002 - [elementary_functions] linspace....................passed
 503    --------------------------------------------------------------------------
 504    Summary
 505 ../..
 506
 507 --> test_run elementary_functions dec2*
 508    TMPDIR = C:\MyPath\AppData\Local\Temp\SCI_TMP_3668_1147
 509
 510    001/004 - [elementary_functions] dec2oct.....................passed
 511    002/004 - [elementary_functions] dec2hex.....................passed
 512    003/004 - [elementary_functions] dec2bin.....................passed
 513    004/004 - [elementary_functions] dec2base....................passed
 514    --------------------------------------------------------------------------
 515    Summary
 516 ../..
 517
 518 --> test_run string *ascii*
 519    TMPDIR = C:\MyPath\AppData\Local\Temp\SCI_TMP_3668_1147
 520
 521    001/003 - [string] isascii...................................passed
 522    002/003 - [string] asciimat..................................passed
 523    003/003 - [string] ascii.....................................passed
 524    --------------------------------------------------------------------------
 525    Summary
 526 ../..
 527 ]]></screen>
 528     </refsection>
 529     <refsection>
 530         <title>Internal Design</title>
 531         <para>
 532             The tests are performed in the temporary directory, not in the directory
 533             which originally contain the tests files.
 534         </para>
 535         <para>
 536             The .tst script is not run as is. Instead, a header and a footer are
 537             inserted at the beginning and at the end of the .tst at the time
 538             the script is copied into the temporary directory.
 539             The role of this modification is to redirect the output messages
 540             into the .dia file, so that the user can have a log file once the test
 541             is performed.
 542         </para>
 543         <para>
 544             An execution timeout delay (watchdog timer) is setup to 5 minutes
 545             for each regular test. To ignore this timeout use the long-time
 546             execution (<literal>LONG TIME EXECUTION</literal>) flag.
 547         </para>
 548     </refsection>
 549     <refsection>
 550         <title>History</title>
 551         <revhistory>
 552             <revision>
 553                 <revnumber>5.4.0</revnumber>
 554                 <revdescription>test_run returns a status:
 555                     <itemizedlist><listitem>
 556                             Returns %t if no error has been detected
 557                         </listitem>
 558                         <listitem>
 559                             Returns %f if any error has been detected
 560                         </listitem>
 561                     </itemizedlist>
 562                     <para>
 563                         <literal>show_diff</literal> and <literal>show_error</literal> added as new options
 564                     </para>
 565                     <para>
 566                         <literal>CLI SHELL MODE</literal> tag is added. Replaces <literal>JVM NOT MANDATORY</literal> (still supported)
 567                     </para>
 568                     <para>
 569                         <literal>test_run</literal> can work on an external module.
 570                     </para>
 571                     <para>
 572                         Fourth argument added to export to a XML file
 573                     </para>
 574                 </revdescription>
 575             </revision>
 576             <revision>
 577                 <revnumber>5.5.0</revnumber>
 578                 <revdescription>32/64bits separation available</revdescription>
 579             </revision>
 580             <revision>
 581                 <revnumber>6.0.0</revnumber>
 582                 <revdescription>
 583                     <para>profiling mode added to profile execution with valgrind (Linux only)</para>
 584                     <para>
 585                         timeout delay (watchdog timer) set to 5 minutes for single tests without <literal>LONG TIME EXECUTION</literal>
 586                     </para>
 587                 </revdescription>
 588             </revision>
 589             <revision>
 590                 <revnumber>6.0.2</revnumber>
 591                 <revdescription>
 592                     <para>Tests names with the * wildcard like sin*, *sin, or *sin* are now allowed</para>
 593                 </revdescription>
 594             </revision>
 595         </revhistory>
 596     </refsection>
 597 </refentry>