fix tests
[scilab.git] / scilab / modules / development_tools / macros / unit_test_run.sci
1 //-----------------------------------------------------------------------------
2 // Pierre MARECHAL
3 // Scilab team
4 // Copyright INRIA
5 // Date : 25/10/2007
6 //
7 // Launch unitary tests
8 //-----------------------------------------------------------------------------
9
10 function unit_test_run(varargin)
11         
12         lhs = argn(1);
13         rhs = argn(2);
14         
15         global test_list;
16         global test_count;
17         global displayed_txt;
18         
19         global check_ref;
20         global create_ref;
21         
22         check_ref          = %T;
23         create_ref         = %F;
24         
25         test_count         = 0;
26         test_passed_count  = 0;
27         test_failed_count  = 0;
28         test_skipped_count = 0;
29         
30         displayed_txt      = '';
31         details_failed     = '';
32         
33         // =======================================================
34         // Gestion des tests à lancer
35         // =======================================================
36         
37         if (rhs == 0) ..
38                                 | ((rhs == 1) & (varargin(1)==[])) ..
39                                 | ((rhs == 2) & (varargin(1)==[]) & (varargin(1)==[])) then
40                 
41                 // No input argument
42                 // unit_test_run()
43                 // unit_test_run([])
44                 // => Launch each test of each module
45                 
46                 module_list = getmodules();
47                 for k=1:size(module_list,'*')
48                         unit_test_add_module(module_list(k));
49                 end
50         
51         elseif (rhs == 1) ..
52                                 | ((rhs == 2) & (varargin(2)==[])) ..
53                                 | ((rhs == 3) & (varargin(2)==[])) then
54                 
55                 // One input argument
56                 // unit_test_run(<module_name>)
57                 // unit_test_run([<module_name_1>,<module_name_2>])
58                 
59                 // varargin(1) = [<module_name_1>,<module_name_2>]
60                 
61                 module_mat = varargin(1);
62                 
63                 [nl,nc] = size(module_mat);
64                 
65                 // unit_test_run([<module_name_1>,<module_name_2>])
66                 
67                 for i=1:nl
68                         for j=1:nc
69                                 if( with_module(module_mat(i,j)) ) then
70                                         unit_test_add_module(module_mat(i,j));
71                                 else
72                                         error(sprintf(gettext("%s is not an installed module"),module_mat(i,j)));
73                                 end
74                         end
75                 end
76                 
77         elseif (rhs == 2) | (rhs == 3) then
78                 
79                 // Two input arguments
80                 // unit_test_run(<module_name>,<test_name>)
81                 // unit_test_run(<module_name>,[<test_name_1>,<test_name_2>] )
82                 
83                 // varargin(1) = <module_name> ==> string 1x1
84                 // varargin(2) = <test_name_1> ==> mat nl x nc
85                 
86                 module   = varargin(1);
87                 test_mat = varargin(2);
88                 
89                 if (size(module) <> [1,1]) & (size(module) <> [1,1]) then
90                         error(gettext("Input argument sizes are not valid"));
91                 end
92                 
93                 [nl,nc] = size(test_mat);
94                 
95                 for i=1:nl
96                         for j=1:nc
97                                 if( fileinfo(SCI+"/modules/"+module+"/unit_tests/"+test_mat(i,j)+".tst") <> [] ) then
98                                         unit_test_add_onetest(module,test_mat(i,j));
99                                 else
100                                         error(sprintf(gettext("The test ""%s"" is not available from the ""%s"" module"),test_mat(i,j),module));
101                                 end
102                         end
103                 end
104         else
105                 error(gettext('Number of parameters incorrect.'));
106         end
107         
108         // =======================================================
109         // Gestion des options
110         // =======================================================
111         
112         if rhs == 3 then
113                 
114                 option_mat =  varargin(3);
115                 
116                 if grep(option_mat,"no_check_ref") <> [] then
117                         check_ref  = %F;
118                 end
119                 
120                 if grep(option_mat,"create_ref") <> [] then
121                         create_ref = %T;
122                         check_ref  = %F;
123                 end
124                 
125         end
126         
127         // Test launch
128         printf("   TMPDIR = %s\n",TMPDIR);
129         printf("\n");
130         
131         for i=1:test_count
132                 
133                 printf("   %02d/%02d - ",i,test_count);
134                 printf("[%s] %s",test_list(i,1),test_list(i,2));
135                 for j = length(test_list(i,2) + test_list(i,1)):50
136                         printf(".");
137                 end
138                 
139                 [status_id,status_msg,status_details] = unit_test_run_onetest(test_list(i,1),test_list(i,2));
140                 printf("%s \n",status_msg);
141
142                 // Recencement des tests
143                 
144                 if status_id == 0 then
145                         test_passed_count = test_passed_count + 1;
146                         
147                 elseif status_id == 10 then
148                         test_skipped_count = test_skipped_count + 1;
149                         
150                 elseif status_id > 0 then
151                         test_failed_count = test_failed_count + 1;
152                         details_failed = [ details_failed ; sprintf("   TEST : [%s] %s",test_list(i,1),test_list(i,2))];
153                         details_failed = [ details_failed ; sprintf("     %s",status_msg) ];
154                         details_failed = [ details_failed ; status_details ];
155                         details_failed = [ details_failed ; "" ];
156                 end
157                 
158         end
159         
160         // Summary
161         
162         test_passed_percent  = test_passed_count  / test_count * 100;
163         test_skipped_percent = test_skipped_count / test_count * 100;
164         test_failed_percent  = test_failed_count  / test_count * 100;
165         
166         printf("\n");
167         printf("   ---------------------------------------------------------------------------------------\n");
168         printf("   Summary\n\n");
169         printf("   tests                     %4d - 100 percent \n",test_count);
170         printf("   passed                    %4d - %3d percent \n",test_passed_count ,test_passed_percent);
171         printf("   failed                    %4d - %3d percent \n",test_failed_count ,test_failed_percent);
172         printf("   skipped                   %4d - %3d percent \n",test_skipped_count,test_skipped_percent);
173         printf("   ---------------------------------------------------------------------------------------\n\n");
174         printf("   Details\n\n");
175         printf("%s\n",details_failed);
176         printf("\n");
177         printf("   ---------------------------------------------------------------------------------------\n");
178
179 endfunction
180
181 //-----------------------------------------------------------------------------
182 // Pierre MARECHAL
183 // Scilab team
184 // Copyright INRIA
185 // Date : 28 oct. 2007
186 //
187 // => List all test files in the module <module_mat>
188 // => Add them to the test_mat matrix
189 //-----------------------------------------------------------------------------
190
191 function unit_test_add_module(module_mat)
192         
193         module_test_dir = SCI+"/modules/"+module_mat+"/unit_tests";
194         test_mat        = basename(listfiles(module_test_dir+"/*.tst"));
195         
196         nl = size(test_mat,"*");
197         for i=1:nl
198                 unit_test_add_onetest(module_mat,test_mat(i));
199         end
200         
201 endfunction
202
203 //-----------------------------------------------------------------------------
204 // Pierre MARECHAL
205 // Scilab team
206 // Copyright INRIA
207 // Date : 28 oct. 2007
208 //
209 // => Add the test <test> to the test_mat matrix
210 //-----------------------------------------------------------------------------
211
212 function unit_test_add_onetest(module,test)
213         
214         global test_list;
215         global test_count;
216         
217         test_count = test_count + 1;
218         test_list( test_count , 1 ) = module;
219         test_list( test_count , 2 ) = test;
220         
221 endfunction
222
223 //-----------------------------------------------------------------------------
224 // Pierre MARECHAL
225 // Scilab team
226 // Copyright INRIA
227 // Date : 28 oct. 2007
228 //
229 // => Run one test
230 //-----------------------------------------------------------------------------
231
232 function [status_id,status_msg,status_details] = unit_test_run_onetest(module,test)
233         
234         global check_ref;
235         global create_ref;
236         
237         status_id      = 0 ;
238         status_msg     = "passed" ;
239         status_details = "";
240         
241         [status_id,status_msg,status_details] = unit_test_run_checkerror(module,test);
242         
243         if check_ref | create_ref then
244                 // Check ref or create ref
245                 if status_id == 0 then
246                         [status_id,status_msg,status_details] = unit_test_run_proc_ref(module,test);
247                 end
248         end
249         
250         return;
251         
252 endfunction
253
254 //-----------------------------------------------------------------------------
255 // Pierre MARECHAL
256 // Scilab team
257 // Copyright INRIA
258 // Date : 8 novembre 2007
259 //
260 // => Check the test
261 //-----------------------------------------------------------------------------
262
263 function [status_id,status_msg,status_details] = unit_test_run_checkerror(module,test)
264         
265         status_id      = 0 ;
266         status_msg     = "passed" ;
267         status_details = "";
268         
269         // Some definitions
270         
271         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
272         
273         tmp_tstfile = pathconvert(TMPDIR+"/"+test+"_1.tst",%f,%f);
274         tmp_diafile = pathconvert(TMPDIR+"/"+test+"_1.dia",%f,%f);
275         tmp_resfile = pathconvert(TMPDIR+"/"+test+"_1.res",%f,%f);
276         tmp_errfile = pathconvert(TMPDIR+"/"+test+"_1.err",%f,%f);
277         
278         // Remove the previous tmp files
279         if fileinfo(tmp_tstfile) <> [] then
280                 deletefile(tmp_tstfile)
281         end
282         
283         if fileinfo(tmp_diafile) <> [] then
284                 deletefile(tmp_diafile)
285         end
286         
287         if fileinfo(tmp_resfile) <> [] then
288                 deletefile(tmp_resfile)
289         end
290         
291         if fileinfo(tmp_errfile) <> [] then
292                 deletefile(tmp_errfile)
293         end
294         
295         //Reset standard globals
296         rand('seed',0);
297         rand('uniform');
298         
299         // Get the tst file
300         txt = mgetl(tstfile);
301         
302         // Check if it's an interactive test
303         if grep(txt,"<-- INTERACTIVE TEST -->")<>[] then
304                 status_msg = "skipped : interactive test";
305                 status_id  = 10;
306                 return;
307         end
308         
309         // Do some modification in tst file
310         txt = strsubst(txt,'pause,end','bugmes();quit;end');
311         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
312         txt = strsubst(txt,'halt();','');
313         
314         // Header : the same for each test
315         
316         head = [        "// <-- HEADER START -->";
317                                 "mode(3);" ;
318                                 "clear;" ;
319                                 "lines(28,72);";
320                                 "lines(0);" ;
321                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
322                                 "predef(''all'');" ;
323                                 "try";
324                                 "diary(''"+tmp_diafile+"'');";
325                                 "// <-- HEADER END -->"];
326                                 
327         tail = [        "// <-- FOOTER START -->";
328                                 "catch";
329                                 "       errmsg = ""<--""+""Error on the test script file""+""-->""";
330                                 "       printf(""%s"",errmsg)";
331                                 "end";
332                                 "diary(0);";
333                                 "exit;";
334                                 "// <-- FOOTER END -->"]
335         
336         txt = [head;
337                 txt;
338                 tail];
339         
340         // and save it in a temporary file
341         mputl(txt,tmp_tstfile);
342         
343         // Build the command to launch
344         if MSDOS then
345                 unit_test_cmd = "( """+SCI+"\bin\scilex.exe"+""""+" -nw -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
346         else
347                 unit_test_cmd = "( "+SCI+"/bin/scilab -nw -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
348         end
349         
350         // Launch the test exec
351         
352         host(unit_test_cmd);
353         
354         // First Check
355         tmp_errfile_info = fileinfo(tmp_errfile);
356         
357         if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
358                 status_msg = "failed  : error_output not empty"
359                 status_details = sprintf("     Check the following file : \n     - %s",tmp_errfile);
360                 status_id  = 5;
361                 return;
362         end
363         
364         //  Do some modification in  dia file
365         dia = mgetl(tmp_diafile);
366         
367         //Check for execution errors
368         if grep(dia,"<--Error on the test script file-->")<>[] then
369                 status_msg = "failed  : premature end of the test script";
370                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
371                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
372                 status_id = 3;
373                 return;
374         end
375         
376         // Remove Header and Footer
377         dia = remove_headers(dia);
378         
379         //Check for execution errors
380         
381         if grep(dia,"!--error")<>[] then
382                 status_msg     = "failed  : the string (!--error) has been detected";
383                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
384                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
385                 status_id  = 1;
386                 return;
387         end
388         
389         if grep(dia,"error on test")<>[] then
390                 status_msg     = "failed  : one or several unit tests failed";
391                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
392                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
393                 status_id      = 2;
394                 return;
395         end
396         
397         return;
398         
399 endfunction
400
401 //-----------------------------------------------------------------------------
402 // Pierre MARECHAL
403 // Scilab team
404 // Copyright INRIA
405 // Date : 28 oct. 2007
406 //
407 // => Check ref or generate ref
408 //-----------------------------------------------------------------------------
409
410 function [status_id,status_msg,status_details] = unit_test_run_proc_ref(module,test)
411         
412         global create_ref;
413         global check_ref;
414         
415         status_id      = 0 ;
416         status_msg     = "passed" ;
417         status_details = "";
418         
419         // Some definitions
420         
421         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
422         diafile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia",%f,%f);
423         reffile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia.ref",%f,%f);
424         
425         tmp_tstfile = pathconvert(TMPDIR+"/"+test+"_2.tst",%f,%f);
426         tmp_diafile = pathconvert(TMPDIR+"/"+test+"_2.dia",%f,%f);
427         tmp_resfile = pathconvert(TMPDIR+"/"+test+"_2.res",%f,%f);
428         tmp_errfile = pathconvert(TMPDIR+"/"+test+"_2.err",%f,%f);
429         
430         // On test l'existence de la ref si besoin
431         
432         if check_ref then
433                 if fileinfo(reffile) == [] then
434                         status_msg     = "failed  : the ref file doesn''t exist";
435                         status_details = "     Add or create the following file"+reffile+" file";
436                         status_details = sprintf("     Add or create the following file : \n     - %s",reffile);
437                         status_id      = 5;
438                         return;
439                 end
440         end
441         
442         // Remove the previous tmp files
443         if fileinfo(tmp_tstfile) <> [] then
444                 deletefile(tmp_tstfile)
445         end
446         
447         if fileinfo(tmp_diafile) <> [] then
448                 deletefile(tmp_diafile)
449         end
450         
451         if fileinfo(tmp_resfile) <> [] then
452                 deletefile(tmp_resfile)
453         end
454         
455         if fileinfo(tmp_errfile) <> [] then
456                 deletefile(tmp_errfile)
457         end
458         
459         //Reset standard globals
460         rand('seed',0);
461         rand('uniform');
462
463         // Do some modification in  tst file
464         
465         txt = mgetl(tstfile);
466         txt = strsubst(txt,'pause,end','bugmes();quit;end');
467         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
468         txt = strsubst(txt,'halt()','');
469         
470         // Header : the same for each test
471         
472         head = [        "// <-- HEADER START -->";
473                                 "mode(3);" ;
474                                 "clear;" ;
475                                 "lines(28,72);";
476                                 "lines(0);" ;
477                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
478                                 "predef(''all'');" ;
479                                 "tmpdirToPrint = msprintf(''TMPDIR1=''''%s''''\n'',TMPDIR);" ;
480                                 "diary(''"+tmp_diafile+"'');";
481                                 "write(%io(2),tmpdirToPrint);";
482                                 "// <-- HEADER END -->"];
483                                 
484         tail = [        "// <-- FOOTER START -->";
485                                 "diary(0);";
486                                 "exit;";
487                                 "// <-- FOOTER END -->"]
488         
489         txt = [head;
490                 txt;
491                 tail];
492         
493         // and save it in a temporary file
494         mputl(txt,tmp_tstfile);
495         
496         // Delete previous dia file
497         if fileinfo(diafile) <> [] then
498                 deletefile(diafile)
499         end
500         
501         // Build the command to launch
502         if MSDOS then
503                 unit_test_cmd = "( """+SCI+"\bin\scilex.exe"+""""+" -nw -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
504         else
505                 unit_test_cmd = "( "+SCI+"/bin/scilab -nw -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
506         end
507         
508         // Launch the test exec
509         host(unit_test_cmd);
510         
511         // First Check
512         tmp_errfile_info = fileinfo(tmp_errfile);
513         
514         if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
515                 status_msg = "failed  : error_output not empty"
516                 status_details = sprintf("     Check the following file : \n     - %s",tmp_errfile);
517                 status_id  = 5;
518                 return;
519         end
520         
521         //  Do some modification in  dia file
522         dia = mgetl(tmp_diafile);
523         
524         // To get TMPDIR line
525         tmpdir1_line = grep(dia,"TMPDIR1");
526         execstr(dia(tmpdir1_line));
527         
528         // Remove Header and Footer
529         dia = remove_headers(dia);
530         
531         //  Do some modification in  dia file
532         dia(grep(dia,"exec("))                     = [];
533         dia(grep(dia,"write(%io(2),tmpdirToPrint"))= [];
534         dia(grep(dia,"TMPDIR1"))                   = [];
535         dia(grep(dia,"diary(0)"))                  = [];
536         
537         dia = strsubst(dia,TMPDIR,'TMPDIR');
538         dia = strsubst(dia,TMPDIR1,'TMPDIR');
539         dia = strsubst(dia,TMPDIR1,'TMPDIR');
540         dia = strsubst(dia,SCI,'SCI');
541         
542         //suppress the prompts
543         dia = strsubst(dia,'-->','');
544         dia = strsubst(dia,'@#>','-->');
545         dia = strsubst(dia,'-1->','');
546         
547         //standardise  number display
548         dia = strsubst(strsubst(strsubst(strsubst(dia,' .','0.'),'E+','D+'),'E-','D-'),'-.','-0.');
549
550         //not to change the ref files
551         dia=strsubst(dia,'bugmes();return','bugmes();quit');
552
553         if create_ref then
554                 
555                 // Delete previous .dia.ref file
556                 if fileinfo(reffile) <> [] then
557                         deletefile(reffile)
558                 end
559                 
560                 mputl(dia,reffile);
561                 
562                 status_msg = "passed : ref created";
563                 status_id  = 20;
564                 
565                 return;
566                 
567         else
568                 
569                 // write down the resulting dia file
570                 mputl(dia,diafile)
571                 
572                 //Check for diff with the .ref file
573                 
574                 [u,ierr] = mopen(reffile,'r');
575                 if ierr== 0 then //ref file exists
576                         
577                         ref=mgetl(u);
578                         mclose(u)
579                         
580                         // suppress blank (diff -nw)
581                         
582                         dia = strsubst(dia,' ','')
583                         ref = strsubst(ref,' ','')
584                         
585                         dia(find(dia=='')) = [];
586                         ref(find(ref=='')) = [];
587                         
588                         dia(find(dia=='')) = [];
589                         ref(find(ref=='')) = [];
590                         
591                         if or(ref<>dia) then
592                                 if MSDOS then
593                                         status_msg     = "failed  : dia and ref are not equal";
594                                         status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
595                                         status_id      = 4;
596                                 else
597                                         status_msg     = "failed  : dia and ref are not equal";
598                                         status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
599                                         status_id      = 4;
600                                 end
601                         end
602                 else
603                         error(sprintf(gettext("The ref file (%s) doesn''t exist"),reffile));
604                 end
605         end
606         
607         return;
608         
609 endfunction
610
611 //-----------------------------------------------------------------------------
612 // Pierre MARECHAL
613 // Scilab team
614 // Copyright INRIA
615 // Date : 8 novembre 2007
616 //
617 // => remove header from the diary txt
618 //
619 //-----------------------------------------------------------------------------
620
621 function dia_out = remove_headers(dia_in)
622         
623         dia_out = dia_in;
624         
625         body_start = grep(dia_out,"// <-- HEADER END -->");
626         
627         if body_start<>[] then
628                 dia_out(1:body_start(1)) = [];
629         end
630         
631         body_end   = grep(dia_out,"// <-- FOOTER START -->");
632         
633         if body_end<>[] then
634                 [dia_nl,dia_nc] = size(dia);
635                 dia_out(body_end(1):dia_nl) = [];
636         end
637         
638         return;
639         
640 endfunction