Version 0.1 of unit_test launcher
[scilab.git] / scilab / modules / development_tools / macros / unit_test_run.sci
1 //-----------------------------------------------------------------------------
2 // Pierre MARECHAL
3 // Scilab team
4 // Copyright INRIA
5 // Date : 25/10/2007
6 //
7 // Launch unitary tests
8 //-----------------------------------------------------------------------------
9
10 function unit_test_run(varargin)
11         
12         lhs = argn(1);
13         rhs = argn(2);
14         
15         global test_list;
16         global test_count;
17         
18         test_count         = 0;
19         test_passed_count  = 0;
20         test_failed_count  = 0;
21         test_skipped_count = 0;
22         
23         select rhs
24                 case 0
25                         // No input argument
26                         // unit_test_run()
27                         // => Launch each test of each module
28                         
29                         module_list = getmodules();
30                         for k=1:size(module_list,'*')
31                                 unit_test_add_module(module_list(k));
32                         end
33                         
34                         break
35                 
36                 case 1
37                         
38                         // One input argument
39                         // unit_test_run(<module_name>)
40                         // unit_test_run([<module_name_1>,<module_name_2>])
41                         
42                         // varargin(1) = [<module_name_1>,<module_name_2>]
43                         
44                         module_mat = varargin(1);
45                         
46                         [nl,nc] = size(module_mat);
47                         
48                         for i=1:nl
49                                 for j=1:nc
50                                         if( with_module(module_mat(i,j)) ) then
51                                                 unit_test_add_module(module_mat(i,j));
52                                         else
53                                                 error(sprintf(gettext("%s is not an installed module"),module_mat(i,j)));
54                                         end
55                                 end
56                         end
57                         
58                         break
59                         
60                 case 2
61                         
62                         // Two input arguments
63                         // unit_test_run(<module_name>,<test_name>)
64                         // unit_test_run(<module_name>,[<test_name_1>,<test_name_2>] )
65                         
66                         // varargin(1) = <module_name> ==> string 1x1
67                         // varargin(2) = <test_name_1> ==> mat nl x nc
68                         
69                         module   = varargin(1);
70                         test_mat = varargin(2);
71                         
72                         if (size(module) <> [1,1]) & (size(module) <> [1,1]) then
73                                 error(gettext("Input argument sizes are not valid"));
74                         end
75                         
76                         [nl,nc] = size(test_mat);
77                         
78                         for i=1:nl
79                                 for j=1:nc
80                                         if( fileinfo(SCI+"/modules/"+module+"/unit_tests/"+test_mat(i,j)+".tst") <> [] ) then
81                                                 unit_test_add_onetest(module,test_mat(i,j));
82                                         else
83                                                 error(sprintf(gettext("The test ""%s"" is not available from the ""%s"" module"),test_mat(i,j),module));
84                                         end
85                                 end
86                         end
87                         
88                         break
89                         
90                 else
91                         error(gettext('Number of parameters incorrect.'));
92         end
93
94         // Test launch
95         
96         printf("\n");
97         
98         for i=1:test_count
99                 
100                 printf("\t %02d/%02d - ",i,test_count);
101                 printf("[%s] %s",test_list(i,1),test_list(i,2));
102                 for j = length(test_list(i,2) + test_list(i,1)):50
103                         printf(".");
104                 end
105                 
106                 [status_id,status_msg] = unit_test_run_onetest(test_list(i,1),test_list(i,2));
107                 printf("%s\n",status_msg);
108                 
109                 if status_id == 0 then
110                         test_passed_count = test_passed_count + 1;
111                 elseif status_id > 0 then
112                         test_failed_count = test_failed_count + 1;
113                 end
114         end
115         
116         // Summary
117         
118         test_passed_percent = test_passed_count / test_count * 100;
119         test_failed_percent = test_failed_count / test_count * 100;
120
121         printf("\n");
122         printf("\t---------------------------------------------------------------------------------------\n");
123         printf("\tSummary\n\n");
124         printf("\ttests                     %4d - 100 %% \n",test_count);
125         printf("\tpassed                    %4d - %3d %% \n",test_passed_count,test_passed_percent);
126         printf("\tfailed                    %4d - %3d %% \n",test_failed_count,test_failed_percent);
127         printf("\t---------------------------------------------------------------------------------------\n");
128         
129 endfunction
130
131 //-----------------------------------------------------------------------------
132 // Pierre MARECHAL
133 // Scilab team
134 // Copyright INRIA
135 // Date : 28 oct. 2007
136 //
137 // => List all test files in the module <module_mat>
138 // => Add them to the test_mat matrix
139 //-----------------------------------------------------------------------------
140
141 function unit_test_add_module(module_mat)
142         
143         module_test_dir = SCI+"/modules/"+module_mat+"/unit_tests";
144         test_mat        = basename(listfiles(module_test_dir+"/*.tst"));
145         
146         nl = size(test_mat,"*");
147         for i=1:nl
148                 unit_test_add_onetest(module_mat,test_mat(i));
149         end
150         
151 endfunction
152
153 //-----------------------------------------------------------------------------
154 // Pierre MARECHAL
155 // Scilab team
156 // Copyright INRIA
157 // Date : 28 oct. 2007
158 //
159 // => Add the test <test> to the test_mat matrix
160 //-----------------------------------------------------------------------------
161
162 function unit_test_add_onetest(module,test)
163         
164         global test_list;
165         global test_count;
166         
167         test_count = test_count + 1;
168         test_list( test_count , 1 ) = module;
169         test_list( test_count , 2 ) = test;
170         
171 endfunction
172
173 //-----------------------------------------------------------------------------
174 // Pierre MARECHAL
175 // Scilab team
176 // Copyright INRIA
177 // Date : 28 oct. 2007
178 //
179 // => Run one test
180 //-----------------------------------------------------------------------------
181
182 function [status_id,status_msg] = unit_test_run_onetest(module,test)
183         
184         status_id  = 0 ;
185         status_msg = "passed" ;
186         
187         // Some definitions
188         
189         tstfile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".tst",%f,%f);
190         diafile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia",%f,%f);
191         reffile     = pathconvert(SCI+"/modules/"+module+"/unit_tests/"+test+".dia.ref",%f,%f);
192         
193         tmp_tstfile = pathconvert(TMPDIR+"/"+test+".tst",%f,%f);
194         tmp_diafile = pathconvert(TMPDIR+"/"+test+".dia",%f,%f);
195         tmp_resfile = pathconvert(TMPDIR+"/"+test+".res",%f,%f);
196         tmp_errfile = pathconvert(TMPDIR+"/"+test+".err",%f,%f);
197         
198         // DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG
199         // printf("tstfile = %s\n",tstfile);
200         // printf("diafile = %s\n",diafile);
201         // printf("tmp_tstfile = %s\n",tmp_tstfile);
202         // printf("tmp_diafile = %s\n",tmp_diafile);
203         // DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG DEBUG
204         
205         //Reset standard globals
206         rand('seed',0);
207         rand('uniform');
208
209         // Do some modification in  tst file
210         
211         txt = mgetl(tstfile);
212         txt = strsubst(txt,'pause,end','bugmes();quit;end');
213         txt = strsubst(txt,'-->','@#>'); //to avoid suppression of input --> with prompts
214         txt = strsubst(txt,'halt()','');
215         
216         // Header : the same for each test
217         
218         head = [        "// <-- HEADER START -->";
219                                 "mode(3);" ;
220                                 "clear;" ;
221                                 "lines(28,72);";
222                                 "lines(0);" ;
223                                 "deff(''[]=bugmes()'',''write(%io(2),''''error on test'''')'');" ;
224                                 "predef(''all'');" ;
225                                 "tmpdirToPrint = msprintf(''TMPDIR1=''''%s''''\n'',TMPDIR);" ;
226                                 "try";
227                                 "diary(''"+tmp_diafile+"'');";
228                                 "write(%io(2),tmpdirToPrint);";
229                                 "// <-- HEADER END -->"];
230                                 
231         tail = [        "// <-- FOOTER START -->";
232                                 "catch";
233                                 "       printf(""Error on the test script file"")";
234                                 "end";
235                                 "diary(0);";
236                                 "exit;";
237                                 "// <-- FOOTER END -->"]
238         
239         txt = [head;
240                 txt;
241                 tail];
242         
243         // and save it in a temporary file
244         mputl(txt,tmp_tstfile);
245         
246         // Delete previous dia file
247         if fileinfo(diafile) <> [] then
248                 deletefile(diafile)
249         end
250         
251         // Build the command to launch
252         if MSDOS then
253                 unit_test_cmd = """"+SCI+"\bin\scilex.exe"+""""+" -nw -args -nouserstartup -f "+tmp_tstfile+" 1>NUL";
254         else
255                 unit_test_cmd = "( "+SCI+"/bin/scilab -nw -nb -args -nouserstartup -f "+tmp_tstfile+" > "+tmp_resfile+" ) 2> "+tmp_errfile;
256         end
257         
258         // Launch the test exec
259         
260         //printf("%s\n",unit_test_cmd);
261         host(unit_test_cmd);
262
263         // First Check
264         tmp_errfile_info = fileinfo(tmp_errfile);
265         if ( (tmp_errfile_info <> []) & (tmp_errfile_info(1)<>0) ) then
266                 status_msg = "failed : error_output not empty"
267                 status_id  = 5;
268                 return;
269         end
270         
271         //  Do some modification in  dia file
272         dia = mgetl(tmp_diafile);
273         
274         // To get TMPDIR line
275         tmpdir1_line = grep(dia,"TMPDIR1");
276         execstr(dia(tmpdir1_line));
277         
278         // Remove Header and Footer
279         
280         body_start = grep(dia,"// <-- HEADER END -->");
281         
282         if body_start<>[] then
283                 dia(1:body_start(1)) = [];
284         end
285         
286         body_end   = grep(dia,"// <-- FOOTER START -->");
287         
288         if body_end<>[] then
289                 [dia_nl,dia_nc] = size(dia);
290                 dia(body_end(1):dia_nl) = [];
291         end
292         
293         //Check for execution errors
294         
295         if grep(dia,"!--error")<>[] then
296                 status_msg = "failed : the string (!--error) has been detected";
297                 status_id  = 1;
298                 return;
299         end
300         
301         if grep(dia,"error on test")<>[] then
302                 status_msg = "failed : one or several unit tests failed";
303                 status_id  = 2;
304                 return;
305         end
306         
307         if grep(dia,"Error on the test script file")<>[] then
308                 status_msg = "failed : premature end of the test script";
309                 status_id = 3;
310                 return;
311         end
312         
313         //  Do some modification in  dia file
314         dia(grep(dia,"exec("))                     = [];
315         dia(grep(dia,"write(%io(2),tmpdirToPrint"))= [];
316         dia(grep(dia,"TMPDIR1"))                   = [];
317         dia(grep(dia,"diary(0)"))                  = [];
318         
319         dia = strsubst(dia,TMPDIR,'TMPDIR');
320         dia = strsubst(dia,TMPDIR1,'TMPDIR');
321         dia = strsubst(dia,TMPDIR1,'TMPDIR');
322         dia = strsubst(dia,SCI,'SCI');
323         
324         //suppress the prompts
325         dia = strsubst(dia,'-->','');
326         dia = strsubst(dia,'@#>','-->');
327         dia = strsubst(dia,'-1->','');
328         
329         //standardise  number display
330         dia = strsubst(strsubst(strsubst(strsubst(dia,' .','0.'),'E+','D+'),'E-','D-'),'-.','-0.');
331
332         //not to change the ref files
333         dia=strsubst(dia,'bugmes();return','bugmes();quit');
334         
335         // write down the resulting dia file
336         mputl(dia,diafile)
337         
338         //Check for diff with the .ref file
339         
340         [u,ierr] = mopen(reffile,'r');
341         if ierr== 0 then //ref file exists
342                 
343                 ref=mgetl(u);
344                 mclose(u)
345                 
346                 // suppress blank (diff -nw)
347                 dia=strsubst(dia,' ','')
348                 ref=strsubst(ref,' ','')
349                 
350                 if or(ref<>dia) then
351                         if MSDOS then
352                                 status_msg = "failed : dia and ref are not equal";
353                                 status_id = 4;
354                         else
355                                 status_msg = "failed : dia and ref are not equal";
356                                 status_id = 4;
357                         end
358                 end
359         else
360                 error(sprintf(gettext("The ref file (%s) doesn''t exist"),reffile));
361         end
362         
363         return;
364         
365 endfunction