It's now possible to have different references according to whether we are under...
[scilab.git] / scilab / modules / development_tools / macros / unit_test_run.sci
1 //-----------------------------------------------------------------------------
2 // Pierre MARECHAL
3 // Scilab team
4 // Copyright INRIA
5 // Date : 25/10/2007
6 //
7 // Launch unitary tests
8 //-----------------------------------------------------------------------------
9
10 function unit_test_run(varargin)
11         
12         lhs = argn(1);
13         rhs = argn(2);
14         
15         global test_list;
16         global test_count;
17         global displayed_txt;
18         
19         // options
20         
21         global check_ref;
22         global check_error_output;
23         global create_ref;
24         global launch_mode;
25
26         
27         check_ref          = %T;
28         check_error_output = %T;
29         create_ref         = %F;
30         launch_mode        = "-nw";
31         just_list_tests    = %F;
32         print_help         = %F;
33         
34         test_count         = 0;
35         test_passed_count  = 0;
36         test_failed_count  = 0;
37         test_skipped_count = 0;
38         
39         displayed_txt      = '';
40         details_failed     = '';
41         
42         // =======================================================
43         // Gestion des tests à lancer
44         // =======================================================
45         
46         if (rhs == 0) ..
47                                 | ((rhs == 1) & (varargin(1)==[])) ..
48                                 | (((rhs == 2)|(rhs == 3)) & (varargin(1)==[]) & (varargin(2)==[])) then
49                 
50                 // No input argument
51                 // unit_test_run()
52                 // unit_test_run([])
53                 // => Launch each test of each module
54                 
55                 module_list = getmodules();
56                 module_list = gsort(module_list,"lr","i");
57                 for k=1:size(module_list,'*')
58                         unit_test_add_module(module_list(k));
59                 end
60         
61         elseif (rhs == 1) ..
62                                 | ((rhs == 2) & (varargin(2)==[])) ..
63                                 | ((rhs == 3) & (varargin(2)==[])) then
64                 
65                 // One input argument
66                 // unit_test_run(<module_name>)
67                 // unit_test_run([<module_name_1>,<module_name_2>])
68                 
69                 // varargin(1) = [<module_name_1>,<module_name_2>]
70                 
71                 module_mat = varargin(1);
72                 
73                 [nl,nc] = size(module_mat);
74                 
75                 // unit_test_run([<module_name_1>,<module_name_2>])
76                 
77                 for i=1:nl
78                         for j=1:nc
79                                 if( with_module(module_mat(i,j)) ) then
80                                         unit_test_add_module(module_mat(i,j));
81                                 else
82                                         error(sprintf(gettext("%s is not an installed module"),module_mat(i,j)));
83                                 end
84                         end
85                 end
86                 
87         elseif (rhs == 2) | (rhs == 3) then
88                 
89                 // Two input arguments
90                 // unit_test_run(<module_name>,<test_name>)
91                 // unit_test_run(<module_name>,[<test_name_1>,<test_name_2>] )
92                 
93                 // varargin(1) = <module_name> ==> string 1x1
94                 // varargin(2) = <test_name_1> ==> mat nl x nc
95                 
96                 module   = varargin(1);
97                 test_mat = varargin(2);
98                 
99                 if ((or(size(module) <> [1,1])) & (test_mat <> [])) then
100                         example = unit_test_examples();
101                         err     = ["" ; gettext("error : Input argument sizes are not valid") ; "" ; example ];
102                         printf("%s\n",err);
103                         return;
104                 end
105                 
106                 [nl,nc] = size(test_mat);
107                 
108                 for i=1:nl
109                         for j=1:nc
110                                 if( fileinfo(SCI+"/modules/"+module+"/unit_tests/"+test_mat(i,j)+".tst") <> [] ) then
111                                         unit_test_add_onetest(module,test_mat(i,j));
112                                 else
113                                         error(sprintf(gettext("The test ""%s"" is not available from the ""%s"" module"),test_mat(i,j),module));
114                                 end
115                         end
116                 end
117         else
118                 error(gettext('Number of parameters incorrect.'));
119         end
120         
121         // =======================================================
122         // Gestion des options
123         // =======================================================
124         
125         if rhs == 3 then
126                 
127                 option_mat =  varargin(3);
128                 
129                 if grep(option_mat,"no_check_ref") <> [] then
130                         check_ref  = %F;
131                 end
132                 
133                 if grep(option_mat,"no_check_error_output") <> [] then
134                         check_error_output  = %F;
135                 end
136                 
137                 if grep(option_mat,"create_ref") <> [] then
138                         create_ref = %T;
139                         check_ref  = %F;
140                 end
141                 
142                 if grep(option_mat,"mode_nw") <> [] then
143                         launch_mode = "-nw";
144                 end
145                 
146                 if grep(option_mat,"mode_nwni") <> [] then
147                         launch_mode = "-nwni";
148                 end
149                 
150                 if grep(option_mat,"list") <> [] then
151                         just_list_tests    = %T;
152                 end
153                 
154                 if grep(option_mat,"help") <> [] then
155                         print_help         = %T;
156                 end
157         end
158
159         if print_help then
160                 
161                 example = unit_test_examples();
162                 printf("%s\n",example);
163                 return;
164         
165         elseif just_list_tests then
166                 
167                 // =======================================================
168                 // Just list tests
169                 // =======================================================
170                 
171                 for i=1:test_count
172                         printf("   %02d - ",i);
173                         printf("[%s] %s\n",test_list(i,1),test_list(i,2));
174                 end
175                 
176         else
177                 
178                 // =======================================================
179                 // Test launch
180                 // =======================================================
181                 
182                 printf("   TMPDIR = %s\n",TMPDIR);
183                 printf("\n");
184                 
185                 for i=1:test_count
186                         
187                         printf("   %02d/%02d - ",i,test_count);
188                         printf("[%s] %s",test_list(i,1),test_list(i,2));
189                         for j = length(test_list(i,2) + test_list(i,1)):50
190                                 printf(".");
191                         end
192                         
193                         [status_id,status_msg,status_details] = unit_test_run_onetest(test_list(i,1),test_list(i,2));
194                         printf("%s \n",status_msg);
195                         
196                         // Recencement des tests
197                         
198                         if status_id == 0 then
199                                 // passed
200                                 test_passed_count = test_passed_count + 1;
201                         
202                         elseif (status_id > 0) & (status_id < 10) then
203                                 // failed
204                                 test_failed_count = test_failed_count + 1;
205                                 details_failed = [ details_failed ; sprintf("   TEST : [%s] %s",test_list(i,1),test_list(i,2))];
206                                 details_failed = [ details_failed ; sprintf("     %s",status_msg) ];
207                                 details_failed = [ details_failed ; status_details ];
208                                 details_failed = [ details_failed ; "" ];
209                         
210                         elseif (status_id > 10) & (status_id < 20) then
211                                 // skipped
212                                 test_skipped_count = test_skipped_count + 1;
213                         end
214                 end
215                 
216                 // Summary
217                 
218                 test_passed_percent  = test_passed_count  / test_count * 100;
219                 test_skipped_percent = test_skipped_count / test_count * 100;
220                 test_failed_percent  = test_failed_count  / test_count * 100;
221                 
222                 printf("\n");
223                 printf("   ---------------------------------------------------------------------------------------\n");
224                 printf("   Summary\n\n");
225                 printf("   tests                     %4d - 100 %% \n",test_count);
226                 printf("   passed                    %4d - %3d %% \n",test_passed_count ,test_passed_percent);
227                 printf("   failed                    %4d - %3d %% \n",test_failed_count ,test_failed_percent);
228                 printf("   skipped                   %4d - %3d %% \n",test_skipped_count,test_skipped_percent);
229                 printf("   ---------------------------------------------------------------------------------------\n");
230                 
231                 if test_failed_count > 0 then
232                         printf("   Details\n\n");
233                         printf("%s\n",details_failed);
234                         printf("\n");
235                         printf("   ---------------------------------------------------------------------------------------\n");
236                 end
237                 
238         end
239         
240 endfunction
241
242 //-----------------------------------------------------------------------------
243 // Pierre MARECHAL
244 // Scilab team
245 // Copyright INRIA
246 // Date : 28 oct. 2007
247 //
248 // => List all test files in the module <module_mat>
249 // => Add them to the test_mat matrix
250 //-----------------------------------------------------------------------------
251
252 function unit_test_add_module(module_mat)
253         
254         module_test_dir = SCI+"/modules/"+module_mat+"/unit_tests";
255         test_mat        = gsort(basename(listfiles(module_test_dir+"/*.tst")),"lr","i");
256         
257         nl = size(test_mat,"*");
258         for i=1:nl
259                 unit_test_add_onetest(module_mat,test_mat(i));
260         end
261         
262 endfunction
263
264 //-----------------------------------------------------------------------------
265 // Pierre MARECHAL
266 // Scilab team
267 // Copyright INRIA
268 // Date : 28 oct. 2007
269 //
270 // => Add the test <test> to the test_mat matrix
271 //-----------------------------------------------------------------------------
272
273 function unit_test_add_onetest(module,test)
274         
275         global test_list;
276         global test_count;
277         
278         test_count = test_count + 1;
279         test_list( test_count , 1 ) = module;
280         test_list( test_count , 2 ) = test;
281         
282 endfunction
283
284 //-----------------------------------------------------------------------------
285 // Pierre MARECHAL
286 // Scilab team
287 // Copyright INRIA
288 // Date : 28 oct. 2007
289 //
290 // => Run one test
291 //-----------------------------------------------------------------------------
292
293 function [status_id,status_msg,status_details] = unit_test_run_onetest(module,test)
294         
295         status_id      = 0 ;
296         status_msg     = "passed" ;
297         status_details = "";
298
299         global check_ref;
300         global create_ref;
301         global launch_mode;
302         global check_error_output;
303         
304         // Some definitions
305         
306         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
307         diafile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia",%f,%f);
308         reffile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia.ref",%f,%f);
309         
310         if MSDOS then
311                 altreffile = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".win.dia.ref",%f,%f);
312         else
313                 altreffile = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".unix.dia.ref",%f,%f);
314         end
315         
316         tmp_tstfile = pathconvert(TMPDIR+"/"+test+".tst",%f,%f);
317         tmp_diafile = pathconvert(TMPDIR+"/"+test+".dia",%f,%f);
318         tmp_resfile = pathconvert(TMPDIR+"/"+test+".res",%f,%f);
319         tmp_errfile = pathconvert(TMPDIR+"/"+test+".err",%f,%f);
320         
321         // Remove the previous tmp files
322         if fileinfo(tmp_tstfile) <> [] then
323                 deletefile(tmp_tstfile)
324         end
325         
326         if fileinfo(tmp_diafile) <> [] then
327                 deletefile(tmp_diafile)
328         end
329         
330         if fileinfo(tmp_resfile) <> [] then
331                 deletefile(tmp_resfile)
332         end
333         
334         if fileinfo(tmp_errfile) <> [] then
335                 deletefile(tmp_errfile)
336         end
337         
338         //Reset standard globals
339         rand('seed',0);
340         rand('uniform');
341         
342         // Get the tst file
343         txt = mgetl(tstfile);
344         
345         // Check if it's an interactive test
346         if grep(txt,"<-- INTERACTIVE TEST -->") <> [] then
347                 status_msg = "skipped : interactive test";
348                 status_id  = 10;
349                 return;
350         end
351         
352         if (grep(txt,"<-- TEST WITH GRAPHIC -->") <> []) & (launch_mode=="-nwni") then
353                 status_msg = "skipped : test with graphic";
354                 status_id  = 11;
355                 return;
356         end
357         
358         // Do some modification in tst file
359         txt = strsubst(txt,'pause,end','bugmes();quit;end');
360         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
361         txt = strsubst(txt,'halt();','');
362         
363         head = [        "// <-- HEADER START -->";
364                                 "mode(3);" ;
365                                 "clear;" ;
366                                 "lines(28,72);";
367                                 "lines(0);" ;
368                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
369                                 "predef(''all'');" ;
370                                 "tmpdirToPrint = msprintf(''TMPDIR1=''''%s''''\n'',TMPDIR);" ;
371                                 "try";
372                                 "diary(''"+tmp_diafile+"'');";
373                                 "write(%io(2),tmpdirToPrint);";
374                                 "// <-- HEADER END -->"];
375                                 
376         tail = [        "// <-- FOOTER START -->";
377                                 "catch";
378                                 "       errmsg = ""<--""+""Error on the test script file""+""-->""";
379                                 "       printf(""%s"",errmsg)";
380                                 "end";
381                                 "diary(0);";
382                                 "exit;";
383                                 "// <-- FOOTER END -->"]
384         
385         txt = [head;
386                 txt;
387                 tail];
388         
389         // and save it in a temporary file
390         mputl(txt,tmp_tstfile);
391         
392         // Gestion de l'emplacement de bin/scilab
393         if (~MSDOS) & (fileinfo(SCI+"/bin/scilab")==[]) then
394                 SCI_BIN = strsubst(SCI,'share/scilab','');
395         else
396                 SCI_BIN = SCI;
397         end
398         
399         // Build the command to launch
400         if MSDOS then
401                 unit_test_cmd = "( """+SCI_BIN+"\bin\scilex.exe"+""""+" "+launch_mode+" -nb -args -nouserstartup -f """+tmp_tstfile+""" > """+tmp_resfile+""" ) 2> """+tmp_errfile+"""";
402         else
403                 unit_test_cmd = "( "+SCI_BIN+"/bin/scilab "+launch_mode+" -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
404         end
405         
406         // Launch the test exec
407         
408         host(unit_test_cmd);
409         
410         // First Check : error output
411         if check_error_output then
412                 tmp_errfile_info = fileinfo(tmp_errfile);
413                 
414                 if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
415                         status_msg = "failed  : error_output not empty"
416                         status_details = sprintf("     Check the following file : \n     - %s",tmp_errfile);
417                         status_id  = 5;
418                         return;
419                 end
420         end
421         
422         //  Get the dia file
423         dia = mgetl(tmp_diafile);
424         
425         // To get TMPDIR value
426         tmpdir1_line = grep(dia,"TMPDIR1");
427         execstr(dia(tmpdir1_line));
428         
429         //Check for execution errors
430         if grep(dia,"<--Error on the test script file-->")<>[] then
431                 status_msg = "failed  : premature end of the test script";
432                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
433                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
434                 status_id = 3;
435                 return;
436         end
437         
438         // Remove Header and Footer
439         dia = remove_headers(dia);
440         
441         //Check for execution errors
442         
443         if grep(dia,"!--error")<>[] then
444                 status_msg     = "failed  : the string (!--error) has been detected";
445                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
446                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
447                 status_id  = 1;
448                 return;
449         end
450         
451         if grep(dia,"error on test")<>[] then
452                 status_msg     = "failed  : one or several unit tests failed";
453                 status_details = sprintf("     Check the following file : \n     - %s",tmp_diafile);
454                 status_details = [ status_details ; sprintf("     Or launch the following command : \n     - exec %s;",tstfile) ];
455                 status_id      = 2;
456                 return;
457         end
458         
459         // On test l'existence de la ref si besoin
460         
461         if check_ref then
462                 if (fileinfo(reffile) == []) & (fileinfo(altreffile) == []) then
463                         status_msg     = "failed  : the ref file doesn''t exist";
464                         status_details = "     Add or create the following file"+reffile+" file";
465                         status_details = sprintf("     Add or create the following file : \n     - %s",reffile);
466                         status_id      = 5;
467                         return;
468                 end
469                 
470                 // Gestion du fichier alternatif ( prioritaire s'il existe )
471                 if fileinfo(altreffile) <> [] then
472                         reffile = altreffile;
473                 end
474                 
475         end
476         
477         // Comparaison ref <--> dia
478         
479         if check_ref | create_ref then
480                 
481                 //  Do some modification in  dia file
482                 dia(grep(dia,"exec("))                     = [];
483                 dia(grep(dia,"write(%io(2),tmpdirToPrint"))= [];
484                 dia(grep(dia,"TMPDIR1"))                   = [];
485                 dia(grep(dia,"diary(0)"))                  = [];
486                 
487                 dia = strsubst(dia,TMPDIR,'TMPDIR');
488                 dia = strsubst(dia,TMPDIR1,'TMPDIR');
489                 dia = strsubst(dia,TMPDIR1,'TMPDIR');
490                 dia = strsubst(dia,SCI,'SCI');
491                 
492                 //suppress the prompts
493                 dia = strsubst(dia,'-->','');
494                 dia = strsubst(dia,'@#>','-->');
495                 dia = strsubst(dia,'-1->','');
496
497
498                 
499                 //standardise  number display
500                 dia = strsubst(strsubst(strsubst(strsubst(dia,' .','0.'),'E+','D+'),'E-','D-'),'-.','-0.');
501                 
502                 //not to change the ref files
503                 dia=strsubst(dia,'bugmes();return','bugmes();quit');
504                 
505                 if create_ref then
506                         
507                         // Delete previous .dia.ref file
508                         if fileinfo(reffile) <> [] then
509                                 deletefile(reffile)
510                         end
511                         
512                         mputl(dia,reffile);
513                         
514                         status_msg = "passed : ref created";
515                         status_id  = 20;
516                         
517                         return;
518                         
519                 else
520                         
521                         // write down the resulting dia file
522                         mputl(dia,diafile)
523                         
524                         //Check for diff with the .ref file
525                         
526                         [u,ierr] = mopen(reffile,'r');
527                         if ierr== 0 then //ref file exists
528                                 
529                                 ref=mgetl(u);
530                                 mclose(u)
531                                 
532                                 // suppress blank (diff -nw)
533                                 
534                                 dia = strsubst(dia,' ','')
535                                 ref = strsubst(ref,' ','')
536                                 
537                                 dia(find(dia=='')) = [];
538                                 ref(find(ref=='')) = [];
539                                 
540                                 dia(find(dia=='')) = [];
541                                 ref(find(ref=='')) = [];
542                                 
543                                 if or(ref<>dia) then
544                                         if MSDOS then
545                                                 status_msg     = "failed  : dia and ref are not equal";
546                                                 status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
547                                                 status_id      = 4;
548                                         else
549                                                 status_msg     = "failed  : dia and ref are not equal";
550                                                 status_details = sprintf("     Compare the following files : \n     - %s\n     - %s",diafile,reffile);
551                                                 status_id      = 4;
552                                         end
553                                 end
554                         else
555                                 error(sprintf(gettext("The ref file (%s) doesn''t exist"),reffile));
556                         end
557                 end
558                 
559         end
560         
561         return;
562         
563 endfunction
564
565 //-----------------------------------------------------------------------------
566 // Pierre MARECHAL
567 // Scilab team
568 // Copyright INRIA
569 // Date : 8 novembre 2007
570 //
571 // => remove header from the diary txt
572 //
573 //-----------------------------------------------------------------------------
574
575 function dia_out = remove_headers(dia_in)
576         
577         dia_out = dia_in;
578         
579         body_start = grep(dia_out,"// <-- HEADER END -->");
580         
581         if body_start<>[] then
582                 dia_out(1:body_start(1)) = [];
583         end
584         
585         body_end   = grep(dia_out,"// <-- FOOTER START -->");
586         
587         if body_end<>[] then
588                 [dia_nl,dia_nc] = size(dia);
589                 dia_out(body_end(1):dia_nl) = [];
590         end
591         
592         return;
593         
594 endfunction
595
596 //-----------------------------------------------------------------------------
597 // Pierre MARECHAL
598 // Scilab team
599 // Copyright INRIA
600 // Date : 28 oct. 2007
601 //
602 // => Check ref or generate ref
603 //-----------------------------------------------------------------------------
604
605 function example = unit_test_examples()
606         
607         example = [ sprintf("Examples :\n\n") ];
608
609         
610         example = [ example ; sprintf("// Launch all tests\n") ];
611         example = [ example ; sprintf("unit_test_run();\n") ];
612         example = [ example ; sprintf("unit_test_run([]);\n") ];
613         example = [ example ; sprintf("unit_test_run([],[]);\n") ];
614         example = [ example ; "" ];
615         example = [ example ; sprintf("// Test one or several module\n") ];
616         example = [ example ; sprintf("unit_test_run(''core'');\n") ];
617         example = [ example ; sprintf("unit_test_run(''core'',[]);\n") ];
618         example = [ example ; sprintf("unit_test_run([''core'',''string'']);\n") ];
619         example = [ example ; "" ];
620         example = [ example ; sprintf("// Launch one or several test in a specified module\n") ];
621         example = [ example ; sprintf("unit_test_run(''core'',[''trycatch'',''opcode'']);\n") ];
622         example = [ example ; "" ];
623         example = [ example ; sprintf("// With options\n") ];
624         example = [ example ; sprintf("unit_test_run([],[],''no_check_ref'');\n") ];
625         example = [ example ; sprintf("unit_test_run([],[],''no_check_error_output'');\n") ];
626         example = [ example ; sprintf("unit_test_run([],[],''create_ref'');\n") ];
627         example = [ example ; sprintf("unit_test_run([],[],''list'');\n") ];
628         example = [ example ; sprintf("unit_test_run([],[],''help'');\n") ];
629         example = [ example ; sprintf("unit_test_run([],[],[''no_check_ref'',''mode_nw'']);\n") ];
630         example = [ example ; "" ];
631         
632 endfunction