Launch test just one time
[scilab.git] / scilab / modules / development_tools / macros / unit_test_run.sci
1 //-----------------------------------------------------------------------------
2 // Pierre MARECHAL
3 // Scilab team
4 // Copyright INRIA
5 // Date : 25/10/2007
6 //
7 // Launch unitary tests
8 //-----------------------------------------------------------------------------
9
10 function unit_test_run(varargin)
11         
12         lhs = argn(1);
13         rhs = argn(2);
14         
15         global test_list;
16         global test_count;
17         global displayed_txt;
18         
19         // options
20         
21         global check_ref;
22         global check_error_output;
23         global create_ref;
24         global launch_mode;
25
26         
27         check_ref          = %T;
28         check_error_output = %T;
29         create_ref         = %F;
30         launch_mode        = "-nw";
31         just_list_tests    = %F;
32         print_help         = %F;
33         
34         test_count         = 0;
35         test_passed_count  = 0;
36         test_failed_count  = 0;
37         test_skipped_count = 0;
38         
39         displayed_txt      = '';
40         details_failed     = '';
41         
42         // =======================================================
43         // Gestion des tests à lancer
44         // =======================================================
45         
46         if (rhs == 0) ..
47                                 | ((rhs == 1) & (varargin(1)==[])) ..
48                                 | (((rhs == 2)|(rhs == 3)) & (varargin(1)==[]) & (varargin(2)==[])) then
49                 
50                 // No input argument
51                 // unit_test_run()
52                 // unit_test_run([])
53                 // => Launch each test of each module
54                 
55                 module_list = getmodules();
56                 for k=1:size(module_list,'*')
57                         unit_test_add_module(module_list(k));
58                 end
59         
60         elseif (rhs == 1) ..
61                                 | ((rhs == 2) & (varargin(2)==[])) ..
62                                 | ((rhs == 3) & (varargin(2)==[])) then
63                 
64                 // One input argument
65                 // unit_test_run(<module_name>)
66                 // unit_test_run([<module_name_1>,<module_name_2>])
67                 
68                 // varargin(1) = [<module_name_1>,<module_name_2>]
69                 
70                 module_mat = varargin(1);
71                 
72                 [nl,nc] = size(module_mat);
73                 
74                 // unit_test_run([<module_name_1>,<module_name_2>])
75                 
76                 for i=1:nl
77                         for j=1:nc
78                                 if( with_module(module_mat(i,j)) ) then
79                                         unit_test_add_module(module_mat(i,j));
80                                 else
81                                         error(sprintf(gettext("%s is not an installed module"),module_mat(i,j)));
82                                 end
83                         end
84                 end
85                 
86         elseif (rhs == 2) | (rhs == 3) then
87                 
88                 // Two input arguments
89                 // unit_test_run(<module_name>,<test_name>)
90                 // unit_test_run(<module_name>,[<test_name_1>,<test_name_2>] )
91                 
92                 // varargin(1) = <module_name> ==> string 1x1
93                 // varargin(2) = <test_name_1> ==> mat nl x nc
94                 
95                 module   = varargin(1);
96                 test_mat = varargin(2);
97                 
98                 if ((or(size(module) <> [1,1])) & (test_mat <> [])) then
99                         example = unit_test_examples();
100                         err     = ["" ; gettext("error : Input argument sizes are not valid") ; "" ; example ];
101                         printf("%s\n",err);
102                         return;
103                 end
104                 
105                 [nl,nc] = size(test_mat);
106                 
107                 for i=1:nl
108                         for j=1:nc
109                                 if( fileinfo(SCI+"/modules/"+module+"/unit_tests/"+test_mat(i,j)+".tst") <> [] ) then
110                                         unit_test_add_onetest(module,test_mat(i,j));
111                                 else
112                                         error(sprintf(gettext("The test ""%s"" is not available from the ""%s"" module"),test_mat(i,j),module));
113                                 end
114                         end
115                 end
116         else
117                 error(gettext('Number of parameters incorrect.'));
118         end
119         
120         // =======================================================
121         // Gestion des options
122         // =======================================================
123         
124         if rhs == 3 then
125                 
126                 option_mat =  varargin(3);
127                 
128                 if grep(option_mat,"no_check_ref") <> [] then
129                         check_ref  = %F;
130                 end
131                 
132                 if grep(option_mat,"no_check_error_output") <> [] then
133                         check_error_output  = %F;
134                 end
135                 
136                 if grep(option_mat,"create_ref") <> [] then
137                         create_ref = %T;
138                         check_ref  = %F;
139                 end
140                 
141                 if grep(option_mat,"mode_nw") <> [] then
142                         launch_mode = "-nw";
143                 end
144                 
145                 if grep(option_mat,"mode_nwni") <> [] then
146                         launch_mode = "-nwni";
147                 end
148                 
149                 if grep(option_mat,"list") <> [] then
150                         just_list_tests    = %T;
151                 end
152                 
153                 if grep(option_mat,"help") <> [] then
154                         print_help         = %T;
155                 end
156         end
157
158         if print_help then
159                 
160                 example = unit_test_examples();
161                 printf("%s\n",example);
162                 return;
163         
164         elseif just_list_tests then
165                 
166                 // =======================================================
167                 // Just list tests
168                 // =======================================================
169                 
170                 for i=1:test_count
171                         printf("   %02d - ",i);
172                         printf("[%s] %s\n",test_list(i,1),test_list(i,2));
173                 end
174                 
175         else
176                 
177                 // =======================================================
178                 // Test launch
179                 // =======================================================
180                 
181                 printf("   TMPDIR = %s\n",TMPDIR);
182                 printf("\n");
183                 
184                 for i=1:test_count
185                         
186                         printf("   %02d/%02d - ",i,test_count);
187                         printf("[%s] %s",test_list(i,1),test_list(i,2));
188                         for j = length(test_list(i,2) + test_list(i,1)):50
189                                 printf(".");
190                         end
191                         
192                         [status_id,status_msg,status_details] = unit_test_run_onetest(test_list(i,1),test_list(i,2));
193                         printf("%s \n",status_msg);
194                         
195                         // Recencement des tests
196                         
197                         if status_id == 0 then
198                                 // passed
199                                 test_passed_count = test_passed_count + 1;
200                         
201                         elseif (status_id > 0) & (status_id < 10) then
202                                 // failed
203                                 test_failed_count = test_failed_count + 1;
204                                 details_failed = [ details_failed ; sprintf("   TEST : [%s] %s",test_list(i,1),test_list(i,2))];
205                                 details_failed = [ details_failed ; sprintf("     %s",status_msg) ];
206                                 details_failed = [ details_failed ; status_details ];
207                                 details_failed = [ details_failed ; "" ];
208                         
209                         elseif (status_id > 10) & (status_id < 20) then
210                                 // skipped
211                                 test_skipped_count = test_skipped_count + 1;
212                         end
213                 end
214                 
215                 // Summary
216                 
217                 test_passed_percent  = test_passed_count  / test_count * 100;
218                 test_skipped_percent = test_skipped_count / test_count * 100;
219                 test_failed_percent  = test_failed_count  / test_count * 100;
220                 
221                 printf("\n");
222                 printf("   ---------------------------------------------------------------------------------------\n");
223                 printf("   Summary\n\n");
224                 printf("   tests                     %4d - 100 %% \n",test_count);
225                 printf("   passed                    %4d - %3d %% \n",test_passed_count ,test_passed_percent);
226                 printf("   failed                    %4d - %3d %% \n",test_failed_count ,test_failed_percent);
227                 printf("   skipped                   %4d - %3d %% \n",test_skipped_count,test_skipped_percent);
228                 printf("   ---------------------------------------------------------------------------------------\n");
229                 
230                 if test_failed_count > 0 then
231                         printf("   Details\n\n");
232                         printf("%s\n",details_failed);
233                         printf("\n");
234                         printf("   ---------------------------------------------------------------------------------------\n");
235                 end
236                 
237         end
238         
239 endfunction
240
241 //-----------------------------------------------------------------------------
242 // Pierre MARECHAL
243 // Scilab team
244 // Copyright INRIA
245 // Date : 28 oct. 2007
246 //
247 // => List all test files in the module <module_mat>
248 // => Add them to the test_mat matrix
249 //-----------------------------------------------------------------------------
250
251 function unit_test_add_module(module_mat)
252         
253         module_test_dir = SCI+"/modules/"+module_mat+"/unit_tests";
254         test_mat        = basename(listfiles(module_test_dir+"/*.tst"));
255         
256         nl = size(test_mat,"*");
257         for i=1:nl
258                 unit_test_add_onetest(module_mat,test_mat(i));
259         end
260         
261 endfunction
262
263 //-----------------------------------------------------------------------------
264 // Pierre MARECHAL
265 // Scilab team
266 // Copyright INRIA
267 // Date : 28 oct. 2007
268 //
269 // => Add the test <test> to the test_mat matrix
270 //-----------------------------------------------------------------------------
271
272 function unit_test_add_onetest(module,test)
273         
274         global test_list;
275         global test_count;
276         
277         test_count = test_count + 1;
278         test_list( test_count , 1 ) = module;
279         test_list( test_count , 2 ) = test;
280         
281 endfunction
282
283 //-----------------------------------------------------------------------------
284 // Pierre MARECHAL
285 // Scilab team
286 // Copyright INRIA
287 // Date : 28 oct. 2007
288 //
289 // => Run one test
290 //-----------------------------------------------------------------------------
291
292 function [status_id,status_msg,status_details] = unit_test_run_onetest(module,test)
293         
294         status_id      = 0 ;
295         status_msg     = "passed" ;
296         status_details = "";
297
298         global check_ref;
299         global create_ref;
300         global launch_mode;
301         global check_error_output;
302         
303         // Some definitions
304         
305         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
306         diafile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia",%f,%f);
307         reffile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia.ref",%f,%f);
308         
309         tmp_tstfile = pathconvert(TMPDIR+"/"+test+".tst",%f,%f);
310         tmp_diafile = pathconvert(TMPDIR+"/"+test+".dia",%f,%f);
311         tmp_resfile = pathconvert(TMPDIR+"/"+test+".res",%f,%f);
312         tmp_errfile = pathconvert(TMPDIR+"/"+test+".err",%f,%f);
313         
314         // Remove the previous tmp files
315         if fileinfo(tmp_tstfile) <> [] then
316                 deletefile(tmp_tstfile)
317         end
318         
319         if fileinfo(tmp_diafile) <> [] then
320                 deletefile(tmp_diafile)
321         end
322         
323         if fileinfo(tmp_resfile) <> [] then
324                 deletefile(tmp_resfile)
325         end
326         
327         if fileinfo(tmp_errfile) <> [] then
328                 deletefile(tmp_errfile)
329         end
330         
331         //Reset standard globals
332         rand('seed',0);
333         rand('uniform');
334         
335         // Get the tst file
336         txt = mgetl(tstfile);
337         
338         // Check if it's an interactive test
339         if grep(txt,"<-- INTERACTIVE TEST -->") <> [] then
340                 status_msg = "skipped : interactive test";
341                 status_id  = 10;
342                 return;
343         end
344         
345         if (grep(txt,"<-- TEST WITH GRAPHIC -->") <> []) & (launch_mode=="-nwni") then
346                 status_msg = "skipped : test with graphic";
347                 status_id  = 11;
348                 return;
349         end
350         
351         // Do some modification in tst file
352         txt = strsubst(txt,'pause,end','bugmes();quit;end');
353         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
354         txt = strsubst(txt,'halt();','');
355         
356         head = [        "// <-- HEADER START -->";
357                                 "mode(3);" ;
358                                 "clear;" ;
359                                 "lines(28,72);";
360                                 "lines(0);" ;
361                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
362                                 "predef(''all'');" ;
363                                 "tmpdirToPrint = msprintf(''TMPDIR1=''''%s''''\n'',TMPDIR);" ;
364                                 "try";
365                                 "diary(''"+tmp_diafile+"'');";
366                                 "write(%io(2),tmpdirToPrint);";
367                                 "// <-- HEADER END -->"];
368                                 
369         tail = [        "// <-- FOOTER START -->";
370                                 "catch";
371                                 "       errmsg = ""<--""+""Error on the test script file""+""-->""";
372                                 "       printf(""%s"",errmsg)";
373                                 "end";
374                                 "diary(0);";
375                                 "exit;";
376                                 "// <-- FOOTER END -->"]
377         
378         txt = [head;
379                 txt;
380                 tail];
381         
382         // and save it in a temporary file
383         mputl(txt,tmp_tstfile);
384         
385         // Gestion de l'emplacement de bin/scilab
386         if (~MSDOS) & (fileinfo(SCI+"/bin/scilab")==[]) then
387                 SCI_BIN = strsubst(SCI,'share/scilab','');
388         else
389                 SCI_BIN = SCI;
390         end
391         
392         // Build the command to launch
393         if MSDOS then
394                 unit_test_cmd = "( """+SCI_BIN+"\bin\scilex.exe"+""""+" "+launch_mode+" -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
395         else
396                 unit_test_cmd = "( "+SCI_BIN+"/bin/scilab "+launch_mode+" -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
397         end
398         
399         // Launch the test exec
400         
401         host(unit_test_cmd);
402         
403         // First Check : error output
404         if check_error_output then
405                 tmp_errfile_info = fileinfo(tmp_errfile);
406                 
407                 if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
408                         status_msg = "failed  : error_output not empty"
409                         status_details = sprintf("     Check the following file : \n     - %s",tmp_errfile);
410                         status_id  = 5;
411                         return;
412                 end
413         end
414         
415         //  Get the dia file
416         dia = mgetl(tmp_diafile);
417         
418         // To get TMPDIR value
419         tmpdir1_line = grep(dia,"TMPDIR1");
420         execstr(dia(tmpdir1_line));
421         
422         //Check for execution errors
423         if grep(dia,"<--Error on the test script file-->")<>[] then
424                 status_msg = "failed  : premature end of the test script";
425                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
426                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
427                 status_id = 3;
428                 return;
429         end
430         
431         // Remove Header and Footer
432         dia = remove_headers(dia);
433         
434         //Check for execution errors
435         
436         if grep(dia,"!--error")<>[] then
437                 status_msg     = "failed  : the string (!--error) has been detected";
438                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
439                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
440                 status_id  = 1;
441                 return;
442         end
443         
444         if grep(dia,"error on test")<>[] then
445                 status_msg     = "failed  : one or several unit tests failed";
446                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
447                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
448                 status_id      = 2;
449                 return;
450         end
451
452         
453         // On test l'existence de la ref si besoin
454         
455         if check_ref then
456                 if fileinfo(reffile) == [] then
457                         status_msg     = "failed  : the ref file doesn''t exist";
458                         status_details = "     Add or create the following file"+reffile+" file";
459                         status_details = sprintf("     Add or create the following file : \n     - %s",reffile);
460                         status_id      = 5;
461                         return;
462                 end
463         end
464         
465         // Comparaison ref <--> dia
466         
467         if check_ref | create_ref then
468                 
469                 //  Do some modification in  dia file
470                 dia(grep(dia,"exec("))                     = [];
471                 dia(grep(dia,"write(%io(2),tmpdirToPrint"))= [];
472                 dia(grep(dia,"TMPDIR1"))                   = [];
473                 dia(grep(dia,"diary(0)"))                  = [];
474                 
475                 dia = strsubst(dia,TMPDIR,'TMPDIR');
476                 dia = strsubst(dia,TMPDIR1,'TMPDIR');
477                 dia = strsubst(dia,TMPDIR1,'TMPDIR');
478                 dia = strsubst(dia,SCI,'SCI');
479                 
480                 //suppress the prompts
481                 dia = strsubst(dia,'-->','');
482                 dia = strsubst(dia,'@#>','-->');
483                 dia = strsubst(dia,'-1->','');
484
485
486                 
487                 //standardise  number display
488                 dia = strsubst(strsubst(strsubst(strsubst(dia,' .','0.'),'E+','D+'),'E-','D-'),'-.','-0.');
489                 
490                 //not to change the ref files
491                 dia=strsubst(dia,'bugmes();return','bugmes();quit');
492                 
493                 if create_ref then
494                         
495                         // Delete previous .dia.ref file
496                         if fileinfo(reffile) <> [] then
497                                 deletefile(reffile)
498                         end
499                         
500                         mputl(dia,reffile);
501                         
502                         status_msg = "passed : ref created";
503                         status_id  = 20;
504                         
505                         return;
506                         
507                 else
508                         
509                         // write down the resulting dia file
510                         mputl(dia,diafile)
511                         
512                         //Check for diff with the .ref file
513                         
514                         [u,ierr] = mopen(reffile,'r');
515                         if ierr== 0 then //ref file exists
516                                 
517                                 ref=mgetl(u);
518                                 mclose(u)
519                                 
520                                 // suppress blank (diff -nw)
521                                 
522                                 dia = strsubst(dia,' ','')
523                                 ref = strsubst(ref,' ','')
524                                 
525                                 dia(find(dia=='')) = [];
526                                 ref(find(ref=='')) = [];
527                                 
528                                 dia(find(dia=='')) = [];
529                                 ref(find(ref=='')) = [];
530                                 
531                                 if or(ref<>dia) then
532                                         if MSDOS then
533                                                 status_msg     = "failed  : dia and ref are not equal";
534                                                 status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
535                                                 status_id      = 4;
536                                         else
537                                                 status_msg     = "failed  : dia and ref are not equal";
538                                                 status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
539                                                 status_id      = 4;
540                                         end
541                                 end
542                         else
543                                 error(sprintf(gettext("The ref file (%s) doesn''t exist"),reffile));
544                         end
545                 end
546                 
547         end
548         
549         return;
550         
551 endfunction
552
553 //-----------------------------------------------------------------------------
554 // Pierre MARECHAL
555 // Scilab team
556 // Copyright INRIA
557 // Date : 8 novembre 2007
558 //
559 // => remove header from the diary txt
560 //
561 //-----------------------------------------------------------------------------
562
563 function dia_out = remove_headers(dia_in)
564         
565         dia_out = dia_in;
566         
567         body_start = grep(dia_out,"// <-- HEADER END -->");
568         
569         if body_start<>[] then
570                 dia_out(1:body_start(1)) = [];
571         end
572         
573         body_end   = grep(dia_out,"// <-- FOOTER START -->");
574         
575         if body_end<>[] then
576                 [dia_nl,dia_nc] = size(dia);
577                 dia_out(body_end(1):dia_nl) = [];
578         end
579         
580         return;
581         
582 endfunction
583
584 //-----------------------------------------------------------------------------
585 // Pierre MARECHAL
586 // Scilab team
587 // Copyright INRIA
588 // Date : 28 oct. 2007
589 //
590 // => Check ref or generate ref
591 //-----------------------------------------------------------------------------
592
593 function example = unit_test_examples()
594         
595         example = [ sprintf("Examples :\n\n") ];
596
597         
598         example = [ example ; sprintf("// Launch all tests\n") ];
599         example = [ example ; sprintf("unit_test_run();\n") ];
600         example = [ example ; sprintf("unit_test_run([]);\n") ];
601         example = [ example ; sprintf("unit_test_run([],[]);\n") ];
602         example = [ example ; "" ];
603         example = [ example ; sprintf("// Test one or several module\n") ];
604         example = [ example ; sprintf("unit_test_run(''core'');\n") ];
605         example = [ example ; sprintf("unit_test_run(''core'',[]);\n") ];
606         example = [ example ; sprintf("unit_test_run([''core'',''string'']);\n") ];
607         example = [ example ; "" ];
608         example = [ example ; sprintf("// Launch one or several test in a specified module\n") ];
609         example = [ example ; sprintf("unit_test_run(''core'',[''trycatch'',''opcode'']);\n") ];
610         example = [ example ; "" ];
611         example = [ example ; sprintf("// With options\n") ];
612         example = [ example ; sprintf("unit_test_run([],[],''no_check_ref'');\n") ];
613         example = [ example ; sprintf("unit_test_run([],[],''no_check_error_output'');\n") ];
614         example = [ example ; sprintf("unit_test_run([],[],''create_ref'');\n") ];
615         example = [ example ; sprintf("unit_test_run([],[],''list'');\n") ];
616         example = [ example ; sprintf("unit_test_run([],[],''help'');\n") ];
617         example = [ example ; sprintf("unit_test_run([],[],[''no_check_ref'',''mode_nw'']);\n") ];
618         example = [ example ; "" ];
619         
620 endfunction