Version 0.2 :
[scilab.git] / scilab / modules / development_tools / macros / unit_test_run.sci
1 //-----------------------------------------------------------------------------
2 // Pierre MARECHAL
3 // Scilab team
4 // Copyright INRIA
5 // Date : 25/10/2007
6 //
7 // Launch unitary tests
8 //-----------------------------------------------------------------------------
9
10 function unit_test_run(varargin)
11         
12         lhs = argn(1);
13         rhs = argn(2);
14         
15         global test_list;
16         global test_count;
17         global displayed_txt;
18         
19         global check_ref;
20         global create_ref;
21         
22         check_ref          = %T;
23         create_ref         = %F;
24         
25         test_count         = 0;
26         test_passed_count  = 0;
27         test_failed_count  = 0;
28         test_skipped_count = 0;
29         
30         displayed_txt      = '';
31         
32         // =======================================================
33         // Gestion des tests à lancer
34         // =======================================================
35         
36         if (rhs == 0) | ((rhs == 1) & (varargin(1)==[])) then
37                 
38                 // No input argument
39                 // unit_test_run()
40                 // unit_test_run([])
41                 // => Launch each test of each module
42                 
43                 module_list = getmodules();
44                 for k=1:size(module_list,'*')
45                         unit_test_add_module(module_list(k));
46                 end
47         
48         elseif (rhs == 1) ..
49                                 | ((rhs == 2) & (varargin(2)==[])) ..
50                                 | ((rhs == 3) & (varargin(2)==[])) then
51                 
52                 // One input argument
53                 // unit_test_run(<module_name>)
54                 // unit_test_run([<module_name_1>,<module_name_2>])
55                 
56                 // varargin(1) = [<module_name_1>,<module_name_2>]
57                 
58                 module_mat = varargin(1);
59                 
60                 [nl,nc] = size(module_mat);
61                 
62                 // unit_test_run([<module_name_1>,<module_name_2>])
63                 
64                 for i=1:nl
65                         for j=1:nc
66                                 if( with_module(module_mat(i,j)) ) then
67                                         unit_test_add_module(module_mat(i,j));
68                                 else
69                                         error(sprintf(gettext("%s is not an installed module"),module_mat(i,j)));
70                                 end
71                         end
72                 end
73                 
74         elseif (rhs == 2) | (rhs == 3) then
75                 
76                 // Two input arguments
77                 // unit_test_run(<module_name>,<test_name>)
78                 // unit_test_run(<module_name>,[<test_name_1>,<test_name_2>] )
79                 
80                 // varargin(1) = <module_name> ==> string 1x1
81                 // varargin(2) = <test_name_1> ==> mat nl x nc
82                 
83                 module   = varargin(1);
84                 test_mat = varargin(2);
85                 
86                 if (size(module) <> [1,1]) & (size(module) <> [1,1]) then
87                         error(gettext("Input argument sizes are not valid"));
88                 end
89                 
90                 [nl,nc] = size(test_mat);
91                 
92                 for i=1:nl
93                         for j=1:nc
94                                 if( fileinfo(SCI+"/modules/"+module+"/unit_tests/"+test_mat(i,j)+".tst") <> [] ) then
95                                         unit_test_add_onetest(module,test_mat(i,j));
96                                 else
97                                         error(sprintf(gettext("The test ""%s"" is not available from the ""%s"" module"),test_mat(i,j),module));
98                                 end
99                         end
100                 end
101         else
102                 error(gettext('Number of parameters incorrect.'));
103         end
104         
105         // =======================================================
106         // Gestion des options
107         // =======================================================
108         
109         if rhs == 3 then
110                 
111                 option_mat =  varargin(3);
112                 
113                 if grep(option_mat,"no_check_ref") <> [] then
114                         check_ref  = %F;
115                 end
116                 
117                 if grep(option_mat,"create_ref") <> [] then
118                         create_ref = %T;
119                         check_ref  = %F;
120                 end
121                 
122         end
123         
124         // Test launch
125         
126         printf("\n");
127         
128         for i=1:test_count
129                 
130                 printf("\t %02d/%02d - ",i,test_count);
131                 printf("[%s] %s",test_list(i,1),test_list(i,2));
132                 for j = length(test_list(i,2) + test_list(i,1)):50
133                         printf(".");
134                 end
135                 
136                 [status_id,status_msg] = unit_test_run_onetest(test_list(i,1),test_list(i,2));
137                 printf("%s \n",status_msg);
138                 
139                 if status_id == 0 then
140                         test_passed_count = test_passed_count + 1;
141                 elseif status_id == 10 then
142                         test_skipped_count = test_skipped_count + 1;
143                 elseif status_id > 0 then
144                         test_failed_count = test_failed_count + 1;
145                 end
146         end
147         
148         // Summary
149         
150         test_passed_percent  = test_passed_count / test_count * 100;
151         test_skipped_percent = test_skipped_count / test_count * 100;
152         test_failed_percent  = test_failed_count / test_count * 100;
153
154         printf("\n");
155         printf("\t---------------------------------------------------------------------------------------\n");
156         printf("\tSummary\n\n");
157         printf("\ttests                     %4d - 100 percent \n",test_count);
158         printf("\tpassed                    %4d - %3d percent \n",test_passed_count,test_passed_percent);
159         printf("\tfailed                    %4d - %3d percent \n",test_failed_count,test_failed_percent);
160         printf("\tskipped                   %4d - %3d percent \n",test_failed_count,test_failed_percent);
161         printf("\t---------------------------------------------------------------------------------------\n");
162         
163 endfunction
164
165 //-----------------------------------------------------------------------------
166 // Pierre MARECHAL
167 // Scilab team
168 // Copyright INRIA
169 // Date : 28 oct. 2007
170 //
171 // => List all test files in the module <module_mat>
172 // => Add them to the test_mat matrix
173 //-----------------------------------------------------------------------------
174
175 function unit_test_add_module(module_mat)
176         
177         module_test_dir = SCI+"/modules/"+module_mat+"/unit_tests";
178         test_mat        = basename(listfiles(module_test_dir+"/*.tst"));
179         
180         nl = size(test_mat,"*");
181         for i=1:nl
182                 unit_test_add_onetest(module_mat,test_mat(i));
183         end
184         
185 endfunction
186
187 //-----------------------------------------------------------------------------
188 // Pierre MARECHAL
189 // Scilab team
190 // Copyright INRIA
191 // Date : 28 oct. 2007
192 //
193 // => Add the test <test> to the test_mat matrix
194 //-----------------------------------------------------------------------------
195
196 function unit_test_add_onetest(module,test)
197         
198         global test_list;
199         global test_count;
200         
201         test_count = test_count + 1;
202         test_list( test_count , 1 ) = module;
203         test_list( test_count , 2 ) = test;
204         
205 endfunction
206
207 //-----------------------------------------------------------------------------
208 // Pierre MARECHAL
209 // Scilab team
210 // Copyright INRIA
211 // Date : 28 oct. 2007
212 //
213 // => Run one test
214 //-----------------------------------------------------------------------------
215
216 function [status_id,status_msg] = unit_test_run_onetest(module,test)
217         
218         global check_ref;
219         global create_ref;
220         
221         status_id  = 0 ;
222         status_msg = "passed" ;
223         
224         [status_id,status_msg] = unit_test_run_checkerror(module,test);
225         
226         if check_ref | create_ref then
227                 // Check ref or create ref
228                 if status_id == 0 then
229                         [status_id,status_msg] = unit_test_run_proc_ref(module,test);
230                 end
231         end
232         
233         return;
234         
235 endfunction
236
237 //-----------------------------------------------------------------------------
238 // Pierre MARECHAL
239 // Scilab team
240 // Copyright INRIA
241 // Date : 8 novembre 2007
242 //
243 // => Check the test
244 //-----------------------------------------------------------------------------
245
246 function [status_id,status_msg] = unit_test_run_checkerror(module,test)
247         
248         status_id  = 0 ;
249         status_msg = "passed" ;
250         
251         // Some definitions
252         
253         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
254         
255         tmp_tstfile = pathconvert(TMPDIR+"/"+test+"_1.tst",%f,%f);
256         tmp_diafile = pathconvert(TMPDIR+"/"+test+"_1.dia",%f,%f);
257         tmp_resfile = pathconvert(TMPDIR+"/"+test+"_1.res",%f,%f);
258         tmp_errfile = pathconvert(TMPDIR+"/"+test+"_1.err",%f,%f);
259         
260         // Remove the previous tmp files
261         if fileinfo(tmp_tstfile) <> [] then
262                 deletefile(tmp_tstfile)
263         end
264         
265         if fileinfo(tmp_diafile) <> [] then
266                 deletefile(tmp_diafile)
267         end
268         
269         if fileinfo(tmp_resfile) <> [] then
270                 deletefile(tmp_resfile)
271         end
272         
273         if fileinfo(tmp_errfile) <> [] then
274                 deletefile(tmp_errfile)
275         end
276         
277         //Reset standard globals
278         rand('seed',0);
279         rand('uniform');
280         
281         // Get the tst file
282         txt = mgetl(tstfile);
283         
284         // Check if it's an interactive test
285         if grep(txt,"<-- INTERACTIVE TEST -->")<>[] then
286                 status_msg = "skipped : interactive test";
287                 status_id  = 10;
288                 return;
289         end
290         
291         // Do some modification in tst file
292         txt = strsubst(txt,'pause,end','bugmes();quit;end');
293         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
294         txt = strsubst(txt,'halt();','');
295         
296         // Header : the same for each test
297         
298         head = [        "// <-- HEADER START -->";
299                                 "mode(3);" ;
300                                 "clear;" ;
301                                 "lines(28,72);";
302                                 "lines(0);" ;
303                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
304                                 "predef(''all'');" ;
305                                 "try";
306                                 "diary(''"+tmp_diafile+"'');";
307                                 "// <-- HEADER END -->"];
308                                 
309         tail = [        "// <-- FOOTER START -->";
310                                 "catch";
311                                 "       errmsg = ""<--""+""Error on the test script file""+""-->""";
312                                 "       printf(""%s"",errmsg)";
313                                 "end";
314                                 "diary(0);";
315                                 "exit;";
316                                 "// <-- FOOTER END -->"]
317         
318         txt = [head;
319                 txt;
320                 tail];
321         
322         // and save it in a temporary file
323         mputl(txt,tmp_tstfile);
324         
325         // Build the command to launch
326         if MSDOS then
327                 unit_test_cmd = "( """+SCI+"\bin\scilex.exe"+""""+" -nw -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
328         else
329                 unit_test_cmd = "( "+SCI+"/bin/scilab -nw -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
330         end
331         
332         // Launch the test exec
333         
334         host(unit_test_cmd);
335         
336         // First Check
337         tmp_errfile_info = fileinfo(tmp_errfile);
338         
339         if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
340                 status_msg = "failed : error_output not empty"
341                 status_id  = 5;
342                 return;
343         end
344         
345         //  Do some modification in  dia file
346         dia = mgetl(tmp_diafile);
347         
348         //Check for execution errors
349         if grep(dia,"<--Error on the test script file-->")<>[] then
350                 status_msg = "failed : premature end of the test script";
351                 status_id = 3;
352                 return;
353         end
354         
355         // Remove Header and Footer
356         dia = remove_headers(dia);
357         
358         //Check for execution errors
359         
360         if grep(dia,"!--error")<>[] then
361                 status_msg = "failed : the string (!--error) has been detected";
362                 status_id  = 1;
363                 return;
364         end
365         
366         if grep(dia,"error on test")<>[] then
367                 status_msg = "failed : one or several unit tests failed";
368                 status_id  = 2;
369                 return;
370         end
371         
372         return;
373         
374 endfunction
375
376 //-----------------------------------------------------------------------------
377 // Pierre MARECHAL
378 // Scilab team
379 // Copyright INRIA
380 // Date : 28 oct. 2007
381 //
382 // => Check ref or generate ref
383 //-----------------------------------------------------------------------------
384
385 function [status_id,status_msg] = unit_test_run_proc_ref(module,test)
386         
387         global create_ref;
388         global check_ref;
389         
390         status_id  = 0 ;
391         status_msg = "passed" ;
392         
393         // Some definitions
394         
395         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
396         diafile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia",%f,%f);
397         reffile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia.ref",%f,%f);
398         
399         tmp_tstfile = pathconvert(TMPDIR+"/"+test+"_2.tst",%f,%f);
400         tmp_diafile = pathconvert(TMPDIR+"/"+test+"_2.dia",%f,%f);
401         tmp_resfile = pathconvert(TMPDIR+"/"+test+"_2.res",%f,%f);
402         tmp_errfile = pathconvert(TMPDIR+"/"+test+"_2.err",%f,%f);
403         
404         // On test l'existence de la ref si besoin
405         
406         if check_ref then
407                 if fileinfo(reffile) == [] then
408                         status_msg = "failed : the ref file doesn''t exist";
409                         status_id  = 5;
410                         return;
411                 end
412         end
413         
414         // Remove the previous tmp files
415         if fileinfo(tmp_tstfile) <> [] then
416                 deletefile(tmp_tstfile)
417         end
418         
419         if fileinfo(tmp_diafile) <> [] then
420                 deletefile(tmp_diafile)
421         end
422         
423         if fileinfo(tmp_resfile) <> [] then
424                 deletefile(tmp_resfile)
425         end
426         
427         if fileinfo(tmp_errfile) <> [] then
428                 deletefile(tmp_errfile)
429         end
430         
431         //Reset standard globals
432         rand('seed',0);
433         rand('uniform');
434
435         // Do some modification in  tst file
436         
437         txt = mgetl(tstfile);
438         txt = strsubst(txt,'pause,end','bugmes();quit;end');
439         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
440         txt = strsubst(txt,'halt()','');
441         
442         // Header : the same for each test
443         
444         head = [        "// <-- HEADER START -->";
445                                 "mode(3);" ;
446                                 "clear;" ;
447                                 "lines(28,72);";
448                                 "lines(0);" ;
449                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
450                                 "predef(''all'');" ;
451                                 "tmpdirToPrint = msprintf(''TMPDIR1=''''%s''''\n'',TMPDIR);" ;
452                                 "diary(''"+tmp_diafile+"'');";
453                                 "write(%io(2),tmpdirToPrint);";
454                                 "// <-- HEADER END -->"];
455                                 
456         tail = [        "// <-- FOOTER START -->";
457                                 "diary(0);";
458                                 "exit;";
459                                 "// <-- FOOTER END -->"]
460         
461         txt = [head;
462                 txt;
463                 tail];
464         
465         // and save it in a temporary file
466         mputl(txt,tmp_tstfile);
467         
468         // Delete previous dia file
469         if fileinfo(diafile) <> [] then
470                 deletefile(diafile)
471         end
472         
473         // Build the command to launch
474         if MSDOS then
475                 unit_test_cmd = "( """+SCI+"\bin\scilex.exe"+""""+" -nw -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
476         else
477                 unit_test_cmd = "( "+SCI+"/bin/scilab -nw -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
478         end
479         
480         // Launch the test exec
481         host(unit_test_cmd);
482         
483         //  Do some modification in  dia file
484         dia = mgetl(tmp_diafile);
485         
486         // To get TMPDIR line
487         tmpdir1_line = grep(dia,"TMPDIR1");
488         execstr(dia(tmpdir1_line));
489         
490         // Remove Header and Footer
491         dia = remove_headers(dia);
492         
493         //  Do some modification in  dia file
494         dia(grep(dia,"exec("))                     = [];
495         dia(grep(dia,"write(%io(2),tmpdirToPrint"))= [];
496         dia(grep(dia,"TMPDIR1"))                   = [];
497         dia(grep(dia,"diary(0)"))                  = [];
498         
499         dia = strsubst(dia,TMPDIR,'TMPDIR');
500         dia = strsubst(dia,TMPDIR1,'TMPDIR');
501         dia = strsubst(dia,TMPDIR1,'TMPDIR');
502         dia = strsubst(dia,SCI,'SCI');
503         
504         //suppress the prompts
505         dia = strsubst(dia,'-->','');
506         dia = strsubst(dia,'@#>','-->');
507         dia = strsubst(dia,'-1->','');
508         
509         //standardise  number display
510         dia = strsubst(strsubst(strsubst(strsubst(dia,' .','0.'),'E+','D+'),'E-','D-'),'-.','-0.');
511
512         //not to change the ref files
513         dia=strsubst(dia,'bugmes();return','bugmes();quit');
514
515         if create_ref then
516                 
517                 // Delete previous .dia.ref file
518                 if fileinfo(reffile) <> [] then
519                         deletefile(reffile)
520                 end
521                 
522                 mputl(dia,reffile);
523                 
524                 status_msg = "passed : ref created";
525                 status_id  = 20;
526                 
527                 return;
528                 
529         else
530                 
531                 // write down the resulting dia file
532                 mputl(dia,diafile)
533                 
534                 //Check for diff with the .ref file
535                 
536                 [u,ierr] = mopen(reffile,'r');
537                 if ierr== 0 then //ref file exists
538                         
539                         ref=mgetl(u);
540                         mclose(u)
541                         
542                         // suppress blank (diff -nw)
543                         
544                         dia = strsubst(dia,' ','')
545                         ref = strsubst(ref,' ','')
546                         
547                         dia(find(dia=='')) = [];
548                         ref(find(ref=='')) = [];
549                         
550                         dia(find(dia=='')) = [];
551                         ref(find(ref=='')) = [];
552                         
553                         if or(ref<>dia) then
554                                 if MSDOS then
555                                         status_msg = "failed : dia and ref are not equal";
556                                         status_id = 4;
557                                 else
558                                         status_msg = "failed : dia and ref are not equal";
559                                         status_id = 4;
560                                 end
561                         end
562                 else
563                         error(sprintf(gettext("The ref file (%s) doesn''t exist"),reffile));
564                 end
565         end
566         
567         return;
568         
569 endfunction
570
571 //-----------------------------------------------------------------------------
572 // Pierre MARECHAL
573 // Scilab team
574 // Copyright INRIA
575 // Date : 8 novembre 2007
576 //
577 // => remove header from the diary txt
578 //
579 //-----------------------------------------------------------------------------
580
581 function dia_out = remove_headers(dia_in)
582         
583         dia_out = dia_in;
584         
585         body_start = grep(dia_out,"// <-- HEADER END -->");
586         
587         if body_start<>[] then
588                 dia_out(1:body_start(1)) = [];
589         end
590         
591         body_end   = grep(dia_out,"// <-- FOOTER START -->");
592         
593         if body_end<>[] then
594                 [dia_nl,dia_nc] = size(dia);
595                 dia_out(body_end(1):dia_nl) = [];
596         end
597         
598         return;
599         
600 endfunction