Sort the test list with a increasing lexical order (to have the same log under differ...
[scilab.git] / scilab / modules / development_tools / macros / unit_test_run.sci
1 //-----------------------------------------------------------------------------
2 // Pierre MARECHAL
3 // Scilab team
4 // Copyright INRIA
5 // Date : 25/10/2007
6 //
7 // Launch unitary tests
8 //-----------------------------------------------------------------------------
9
10 function unit_test_run(varargin)
11         
12         lhs = argn(1);
13         rhs = argn(2);
14         
15         global test_list;
16         global test_count;
17         global displayed_txt;
18         
19         // options
20         
21         global check_ref;
22         global check_error_output;
23         global create_ref;
24         global launch_mode;
25
26         
27         check_ref          = %T;
28         check_error_output = %T;
29         create_ref         = %F;
30         launch_mode        = "-nw";
31         just_list_tests    = %F;
32         print_help         = %F;
33         
34         test_count         = 0;
35         test_passed_count  = 0;
36         test_failed_count  = 0;
37         test_skipped_count = 0;
38         
39         displayed_txt      = '';
40         details_failed     = '';
41         
42         // =======================================================
43         // Gestion des tests à lancer
44         // =======================================================
45         
46         if (rhs == 0) ..
47                                 | ((rhs == 1) & (varargin(1)==[])) ..
48                                 | (((rhs == 2)|(rhs == 3)) & (varargin(1)==[]) & (varargin(2)==[])) then
49                 
50                 // No input argument
51                 // unit_test_run()
52                 // unit_test_run([])
53                 // => Launch each test of each module
54                 
55                 module_list = getmodules();
56                 module_list = gsort(module_list,"lr","i");
57                 for k=1:size(module_list,'*')
58                         unit_test_add_module(module_list(k));
59                 end
60         
61         elseif (rhs == 1) ..
62                                 | ((rhs == 2) & (varargin(2)==[])) ..
63                                 | ((rhs == 3) & (varargin(2)==[])) then
64                 
65                 // One input argument
66                 // unit_test_run(<module_name>)
67                 // unit_test_run([<module_name_1>,<module_name_2>])
68                 
69                 // varargin(1) = [<module_name_1>,<module_name_2>]
70                 
71                 module_mat = varargin(1);
72                 
73                 [nl,nc] = size(module_mat);
74                 
75                 // unit_test_run([<module_name_1>,<module_name_2>])
76                 
77                 for i=1:nl
78                         for j=1:nc
79                                 if( with_module(module_mat(i,j)) ) then
80                                         unit_test_add_module(module_mat(i,j));
81                                 else
82                                         error(sprintf(gettext("%s is not an installed module"),module_mat(i,j)));
83                                 end
84                         end
85                 end
86                 
87         elseif (rhs == 2) | (rhs == 3) then
88                 
89                 // Two input arguments
90                 // unit_test_run(<module_name>,<test_name>)
91                 // unit_test_run(<module_name>,[<test_name_1>,<test_name_2>] )
92                 
93                 // varargin(1) = <module_name> ==> string 1x1
94                 // varargin(2) = <test_name_1> ==> mat nl x nc
95                 
96                 module   = varargin(1);
97                 test_mat = varargin(2);
98                 
99                 if ((or(size(module) <> [1,1])) & (test_mat <> [])) then
100                         example = unit_test_examples();
101                         err     = ["" ; gettext("error : Input argument sizes are not valid") ; "" ; example ];
102                         printf("%s\n",err);
103                         return;
104                 end
105                 
106                 [nl,nc] = size(test_mat);
107                 
108                 for i=1:nl
109                         for j=1:nc
110                                 if( fileinfo(SCI+"/modules/"+module+"/unit_tests/"+test_mat(i,j)+".tst") <> [] ) then
111                                         unit_test_add_onetest(module,test_mat(i,j));
112                                 else
113                                         error(sprintf(gettext("The test ""%s"" is not available from the ""%s"" module"),test_mat(i,j),module));
114                                 end
115                         end
116                 end
117         else
118                 error(gettext('Number of parameters incorrect.'));
119         end
120         
121         // =======================================================
122         // Gestion des options
123         // =======================================================
124         
125         if rhs == 3 then
126                 
127                 option_mat =  varargin(3);
128                 
129                 if grep(option_mat,"no_check_ref") <> [] then
130                         check_ref  = %F;
131                 end
132                 
133                 if grep(option_mat,"no_check_error_output") <> [] then
134                         check_error_output  = %F;
135                 end
136                 
137                 if grep(option_mat,"create_ref") <> [] then
138                         create_ref = %T;
139                         check_ref  = %F;
140                 end
141                 
142                 if grep(option_mat,"mode_nw") <> [] then
143                         launch_mode = "-nw";
144                 end
145                 
146                 if grep(option_mat,"mode_nwni") <> [] then
147                         launch_mode = "-nwni";
148                 end
149                 
150                 if grep(option_mat,"list") <> [] then
151                         just_list_tests    = %T;
152                 end
153                 
154                 if grep(option_mat,"help") <> [] then
155                         print_help         = %T;
156                 end
157         end
158
159         if print_help then
160                 
161                 example = unit_test_examples();
162                 printf("%s\n",example);
163                 return;
164         
165         elseif just_list_tests then
166                 
167                 // =======================================================
168                 // Just list tests
169                 // =======================================================
170                 
171                 for i=1:test_count
172                         printf("   %02d - ",i);
173                         printf("[%s] %s\n",test_list(i,1),test_list(i,2));
174                 end
175                 
176         else
177                 
178                 // =======================================================
179                 // Test launch
180                 // =======================================================
181                 
182                 printf("   TMPDIR = %s\n",TMPDIR);
183                 printf("\n");
184                 
185                 for i=1:test_count
186                         
187                         printf("   %02d/%02d - ",i,test_count);
188                         printf("[%s] %s",test_list(i,1),test_list(i,2));
189                         for j = length(test_list(i,2) + test_list(i,1)):50
190                                 printf(".");
191                         end
192                         
193                         [status_id,status_msg,status_details] = unit_test_run_onetest(test_list(i,1),test_list(i,2));
194                         printf("%s \n",status_msg);
195                         
196                         // Recencement des tests
197                         
198                         if status_id == 0 then
199                                 // passed
200                                 test_passed_count = test_passed_count + 1;
201                         
202                         elseif (status_id > 0) & (status_id < 10) then
203                                 // failed
204                                 test_failed_count = test_failed_count + 1;
205                                 details_failed = [ details_failed ; sprintf("   TEST : [%s] %s",test_list(i,1),test_list(i,2))];
206                                 details_failed = [ details_failed ; sprintf("     %s",status_msg) ];
207                                 details_failed = [ details_failed ; status_details ];
208                                 details_failed = [ details_failed ; "" ];
209                         
210                         elseif (status_id > 10) & (status_id < 20) then
211                                 // skipped
212                                 test_skipped_count = test_skipped_count + 1;
213                         end
214                 end
215                 
216                 // Summary
217                 
218                 test_passed_percent  = test_passed_count  / test_count * 100;
219                 test_skipped_percent = test_skipped_count / test_count * 100;
220                 test_failed_percent  = test_failed_count  / test_count * 100;
221                 
222                 printf("\n");
223                 printf("   ---------------------------------------------------------------------------------------\n");
224                 printf("   Summary\n\n");
225                 printf("   tests                     %4d - 100 %% \n",test_count);
226                 printf("   passed                    %4d - %3d %% \n",test_passed_count ,test_passed_percent);
227                 printf("   failed                    %4d - %3d %% \n",test_failed_count ,test_failed_percent);
228                 printf("   skipped                   %4d - %3d %% \n",test_skipped_count,test_skipped_percent);
229                 printf("   ---------------------------------------------------------------------------------------\n");
230                 
231                 if test_failed_count > 0 then
232                         printf("   Details\n\n");
233                         printf("%s\n",details_failed);
234                         printf("\n");
235                         printf("   ---------------------------------------------------------------------------------------\n");
236                 end
237                 
238         end
239         
240 endfunction
241
242 //-----------------------------------------------------------------------------
243 // Pierre MARECHAL
244 // Scilab team
245 // Copyright INRIA
246 // Date : 28 oct. 2007
247 //
248 // => List all test files in the module <module_mat>
249 // => Add them to the test_mat matrix
250 //-----------------------------------------------------------------------------
251
252 function unit_test_add_module(module_mat)
253         
254         module_test_dir = SCI+"/modules/"+module_mat+"/unit_tests";
255         test_mat        = gsort(basename(listfiles(module_test_dir+"/*.tst")),"lr","i");
256         
257         nl = size(test_mat,"*");
258         for i=1:nl
259                 unit_test_add_onetest(module_mat,test_mat(i));
260         end
261         
262 endfunction
263
264 //-----------------------------------------------------------------------------
265 // Pierre MARECHAL
266 // Scilab team
267 // Copyright INRIA
268 // Date : 28 oct. 2007
269 //
270 // => Add the test <test> to the test_mat matrix
271 //-----------------------------------------------------------------------------
272
273 function unit_test_add_onetest(module,test)
274         
275         global test_list;
276         global test_count;
277         
278         test_count = test_count + 1;
279         test_list( test_count , 1 ) = module;
280         test_list( test_count , 2 ) = test;
281         
282 endfunction
283
284 //-----------------------------------------------------------------------------
285 // Pierre MARECHAL
286 // Scilab team
287 // Copyright INRIA
288 // Date : 28 oct. 2007
289 //
290 // => Run one test
291 //-----------------------------------------------------------------------------
292
293 function [status_id,status_msg,status_details] = unit_test_run_onetest(module,test)
294         
295         status_id      = 0 ;
296         status_msg     = "passed" ;
297         status_details = "";
298
299         global check_ref;
300         global create_ref;
301         global launch_mode;
302         global check_error_output;
303         
304         // Some definitions
305         
306         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
307         diafile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia",%f,%f);
308         reffile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia.ref",%f,%f);
309         
310         tmp_tstfile = pathconvert(TMPDIR+"/"+test+".tst",%f,%f);
311         tmp_diafile = pathconvert(TMPDIR+"/"+test+".dia",%f,%f);
312         tmp_resfile = pathconvert(TMPDIR+"/"+test+".res",%f,%f);
313         tmp_errfile = pathconvert(TMPDIR+"/"+test+".err",%f,%f);
314         
315         // Remove the previous tmp files
316         if fileinfo(tmp_tstfile) <> [] then
317                 deletefile(tmp_tstfile)
318         end
319         
320         if fileinfo(tmp_diafile) <> [] then
321                 deletefile(tmp_diafile)
322         end
323         
324         if fileinfo(tmp_resfile) <> [] then
325                 deletefile(tmp_resfile)
326         end
327         
328         if fileinfo(tmp_errfile) <> [] then
329                 deletefile(tmp_errfile)
330         end
331         
332         //Reset standard globals
333         rand('seed',0);
334         rand('uniform');
335         
336         // Get the tst file
337         txt = mgetl(tstfile);
338         
339         // Check if it's an interactive test
340         if grep(txt,"<-- INTERACTIVE TEST -->") <> [] then
341                 status_msg = "skipped : interactive test";
342                 status_id  = 10;
343                 return;
344         end
345         
346         if (grep(txt,"<-- TEST WITH GRAPHIC -->") <> []) & (launch_mode=="-nwni") then
347                 status_msg = "skipped : test with graphic";
348                 status_id  = 11;
349                 return;
350         end
351         
352         // Do some modification in tst file
353         txt = strsubst(txt,'pause,end','bugmes();quit;end');
354         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
355         txt = strsubst(txt,'halt();','');
356         
357         head = [        "// <-- HEADER START -->";
358                                 "mode(3);" ;
359                                 "clear;" ;
360                                 "lines(28,72);";
361                                 "lines(0);" ;
362                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
363                                 "predef(''all'');" ;
364                                 "tmpdirToPrint = msprintf(''TMPDIR1=''''%s''''\n'',TMPDIR);" ;
365                                 "try";
366                                 "diary(''"+tmp_diafile+"'');";
367                                 "write(%io(2),tmpdirToPrint);";
368                                 "// <-- HEADER END -->"];
369                                 
370         tail = [        "// <-- FOOTER START -->";
371                                 "catch";
372                                 "       errmsg = ""<--""+""Error on the test script file""+""-->""";
373                                 "       printf(""%s"",errmsg)";
374                                 "end";
375                                 "diary(0);";
376                                 "exit;";
377                                 "// <-- FOOTER END -->"]
378         
379         txt = [head;
380                 txt;
381                 tail];
382         
383         // and save it in a temporary file
384         mputl(txt,tmp_tstfile);
385         
386         // Gestion de l'emplacement de bin/scilab
387         if (~MSDOS) & (fileinfo(SCI+"/bin/scilab")==[]) then
388                 SCI_BIN = strsubst(SCI,'share/scilab','');
389         else
390                 SCI_BIN = SCI;
391         end
392         
393         // Build the command to launch
394         if MSDOS then
395                 unit_test_cmd = "( """+SCI_BIN+"\bin\scilex.exe"+""""+" "+launch_mode+" -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
396         else
397                 unit_test_cmd = "( "+SCI_BIN+"/bin/scilab "+launch_mode+" -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
398         end
399         
400         // Launch the test exec
401         
402         host(unit_test_cmd);
403         
404         // First Check : error output
405         if check_error_output then
406                 tmp_errfile_info = fileinfo(tmp_errfile);
407                 
408                 if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
409                         status_msg = "failed  : error_output not empty"
410                         status_details = sprintf("     Check the following file : \n     - %s",tmp_errfile);
411                         status_id  = 5;
412                         return;
413                 end
414         end
415         
416         //  Get the dia file
417         dia = mgetl(tmp_diafile);
418         
419         // To get TMPDIR value
420         tmpdir1_line = grep(dia,"TMPDIR1");
421         execstr(dia(tmpdir1_line));
422         
423         //Check for execution errors
424         if grep(dia,"<--Error on the test script file-->")<>[] then
425                 status_msg = "failed  : premature end of the test script";
426                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
427                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
428                 status_id = 3;
429                 return;
430         end
431         
432         // Remove Header and Footer
433         dia = remove_headers(dia);
434         
435         //Check for execution errors
436         
437         if grep(dia,"!--error")<>[] then
438                 status_msg     = "failed  : the string (!--error) has been detected";
439                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
440                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
441                 status_id  = 1;
442                 return;
443         end
444         
445         if grep(dia,"error on test")<>[] then
446                 status_msg     = "failed  : one or several unit tests failed";
447                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
448                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
449                 status_id      = 2;
450                 return;
451         end
452
453         
454         // On test l'existence de la ref si besoin
455         
456         if check_ref then
457                 if fileinfo(reffile) == [] then
458                         status_msg     = "failed  : the ref file doesn''t exist";
459                         status_details = "     Add or create the following file"+reffile+" file";
460                         status_details = sprintf("     Add or create the following file : \n     - %s",reffile);
461                         status_id      = 5;
462                         return;
463                 end
464         end
465         
466         // Comparaison ref <--> dia
467         
468         if check_ref | create_ref then
469                 
470                 //  Do some modification in  dia file
471                 dia(grep(dia,"exec("))                     = [];
472                 dia(grep(dia,"write(%io(2),tmpdirToPrint"))= [];
473                 dia(grep(dia,"TMPDIR1"))                   = [];
474                 dia(grep(dia,"diary(0)"))                  = [];
475                 
476                 dia = strsubst(dia,TMPDIR,'TMPDIR');
477                 dia = strsubst(dia,TMPDIR1,'TMPDIR');
478                 dia = strsubst(dia,TMPDIR1,'TMPDIR');
479                 dia = strsubst(dia,SCI,'SCI');
480                 
481                 //suppress the prompts
482                 dia = strsubst(dia,'-->','');
483                 dia = strsubst(dia,'@#>','-->');
484                 dia = strsubst(dia,'-1->','');
485
486
487                 
488                 //standardise  number display
489                 dia = strsubst(strsubst(strsubst(strsubst(dia,' .','0.'),'E+','D+'),'E-','D-'),'-.','-0.');
490                 
491                 //not to change the ref files
492                 dia=strsubst(dia,'bugmes();return','bugmes();quit');
493                 
494                 if create_ref then
495                         
496                         // Delete previous .dia.ref file
497                         if fileinfo(reffile) <> [] then
498                                 deletefile(reffile)
499                         end
500                         
501                         mputl(dia,reffile);
502                         
503                         status_msg = "passed : ref created";
504                         status_id  = 20;
505                         
506                         return;
507                         
508                 else
509                         
510                         // write down the resulting dia file
511                         mputl(dia,diafile)
512                         
513                         //Check for diff with the .ref file
514                         
515                         [u,ierr] = mopen(reffile,'r');
516                         if ierr== 0 then //ref file exists
517                                 
518                                 ref=mgetl(u);
519                                 mclose(u)
520                                 
521                                 // suppress blank (diff -nw)
522                                 
523                                 dia = strsubst(dia,' ','')
524                                 ref = strsubst(ref,' ','')
525                                 
526                                 dia(find(dia=='')) = [];
527                                 ref(find(ref=='')) = [];
528                                 
529                                 dia(find(dia=='')) = [];
530                                 ref(find(ref=='')) = [];
531                                 
532                                 if or(ref<>dia) then
533                                         if MSDOS then
534                                                 status_msg     = "failed  : dia and ref are not equal";
535                                                 status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
536                                                 status_id      = 4;
537                                         else
538                                                 status_msg     = "failed  : dia and ref are not equal";
539                                                 status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
540                                                 status_id      = 4;
541                                         end
542                                 end
543                         else
544                                 error(sprintf(gettext("The ref file (%s) doesn''t exist"),reffile));
545                         end
546                 end
547                 
548         end
549         
550         return;
551         
552 endfunction
553
554 //-----------------------------------------------------------------------------
555 // Pierre MARECHAL
556 // Scilab team
557 // Copyright INRIA
558 // Date : 8 novembre 2007
559 //
560 // => remove header from the diary txt
561 //
562 //-----------------------------------------------------------------------------
563
564 function dia_out = remove_headers(dia_in)
565         
566         dia_out = dia_in;
567         
568         body_start = grep(dia_out,"// <-- HEADER END -->");
569         
570         if body_start<>[] then
571                 dia_out(1:body_start(1)) = [];
572         end
573         
574         body_end   = grep(dia_out,"// <-- FOOTER START -->");
575         
576         if body_end<>[] then
577                 [dia_nl,dia_nc] = size(dia);
578                 dia_out(body_end(1):dia_nl) = [];
579         end
580         
581         return;
582         
583 endfunction
584
585 //-----------------------------------------------------------------------------
586 // Pierre MARECHAL
587 // Scilab team
588 // Copyright INRIA
589 // Date : 28 oct. 2007
590 //
591 // => Check ref or generate ref
592 //-----------------------------------------------------------------------------
593
594 function example = unit_test_examples()
595         
596         example = [ sprintf("Examples :\n\n") ];
597
598         
599         example = [ example ; sprintf("// Launch all tests\n") ];
600         example = [ example ; sprintf("unit_test_run();\n") ];
601         example = [ example ; sprintf("unit_test_run([]);\n") ];
602         example = [ example ; sprintf("unit_test_run([],[]);\n") ];
603         example = [ example ; "" ];
604         example = [ example ; sprintf("// Test one or several module\n") ];
605         example = [ example ; sprintf("unit_test_run(''core'');\n") ];
606         example = [ example ; sprintf("unit_test_run(''core'',[]);\n") ];
607         example = [ example ; sprintf("unit_test_run([''core'',''string'']);\n") ];
608         example = [ example ; "" ];
609         example = [ example ; sprintf("// Launch one or several test in a specified module\n") ];
610         example = [ example ; sprintf("unit_test_run(''core'',[''trycatch'',''opcode'']);\n") ];
611         example = [ example ; "" ];
612         example = [ example ; sprintf("// With options\n") ];
613         example = [ example ; sprintf("unit_test_run([],[],''no_check_ref'');\n") ];
614         example = [ example ; sprintf("unit_test_run([],[],''no_check_error_output'');\n") ];
615         example = [ example ; sprintf("unit_test_run([],[],''create_ref'');\n") ];
616         example = [ example ; sprintf("unit_test_run([],[],''list'');\n") ];
617         example = [ example ; sprintf("unit_test_run([],[],''help'');\n") ];
618         example = [ example ; sprintf("unit_test_run([],[],[''no_check_ref'',''mode_nw'']);\n") ];
619         example = [ example ; "" ];
620         
621 endfunction