Better comment management through Shell mode call
[scilab.git] / scilab / modules / parse / src / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13 #include "isatty.hxx"
14 #include "parse.hxx"
15 #include "parser_private.hxx"
16
17 #include "context.hxx"
18
19 extern "C"
20 {
21 #include "charEncoding.h"
22 #include "MALLOC.h"
23 }
24
25 static int matrix_level = 0;
26 static int comment_level = 0;
27 static int last_token = 0;
28 static int exit_status = PARSE_ERROR;
29 static std::string current_file;
30 static std::string program_name;
31
32 static std::string *pstBuffer;
33
34 static bool rejected = false;
35
36 #define YY_USER_ACTION                          \
37  yylloc.last_column += yyleng;
38
39 /* -*- Verbose Special Debug -*- */
40 //#define DEV
41 //#define TOKENDEV
42
43 #ifdef DEV
44 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
45 #else
46 #define DEBUG(x) /* Nothing */
47 #endif
48
49 %}
50
51 %option stack
52 %option noyywrap
53
54 %x SIMPLESTRING
55 %x DOUBLESTRING
56 %x REGIONCOMMENT
57 %x LINECOMMENT
58 %x LINEBREAK
59
60 %x MATRIX
61 %x MATRIXMINUSID
62
63 %x SHELLMODE
64 %x BEGINID
65
66 spaces                  [ \t\v\f]+
67 integer                 [0-9]+
68 number                  [0-9]+[\.][0-9]*
69 little                  \.[0-9]+
70
71 floating                ({little}|{number}|{integer})[deDE][+-]?{integer}
72
73 hex             [0]x[0-9a-fA-F]+
74 oct             [0]o[0-7]+
75
76
77 utf2            ([\xC2-\xDF][\x80-\xBF])
78 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
79 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
80 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
81 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
82 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
83 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
84 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
85
86 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
87 utf4            ({utf41}|{utf42}|{utf43})
88
89 utf             ({utf2}|{utf3}|{utf4})
90 id              (([a-zA-Z_%#?$]|{utf})([a-zA-Z_0-9#?$]|{utf})*)
91
92
93 newline                 ("\r"|"\n"|"\r\n")
94 blankline               {spaces}+{newline}
95 emptyline       {newline}({spaces}|[,;])+{newline}
96 next                    (".."|"...")
97
98 boolnot                 ("@"|"~")
99 booltrue                ("%t"|"%T")
100 boolfalse               ("%f"|"%F")
101 booland                 ("&")
102 boolandand              ("&&")
103 boolor                  ("|")
104 booloror                ("||")
105
106 lbrack                  "["
107 rbrack                  "]"
108
109 lparen                  "("
110 rparen                  ")"
111
112 lbrace                  "{"
113 rbrace                  "}"
114
115 dollar                  "$"
116
117 semicolon               ";"
118 comma                   ","
119 colon                   ":"
120
121 startcomment            "//"
122 startblockcomment       "/*"
123 endblockcomment         "*/"
124
125 dquote                  "\""
126 quote                   "'"
127
128 dot             "."
129 dotquote                ".'"
130 dottimes                ".*"
131 dotrdivide              "./"
132 dotldivide              ".\\"
133 dotpower                (".^"|".**")
134
135 plus                    "+"
136 minus                   "-"
137 rdivide                 "/"
138 ldivide                 "\\"
139 times                   "*"
140 power                   ("^"|"**")
141
142 equal                   "=="
143 notequal                ("~="|"@="|"<>")
144 lowerthan               "<"
145 greaterthan             ">"
146 lowerequal              "<="
147 greaterequal            ">="
148
149 krontimes               ".*."
150 kronrdivide             "./."
151 kronldivide             ".\\."
152
153 controltimes    ("*."[^0-9])
154 controlrdivide  ("/."[^0-9])
155 controlldivide  ("\\."[^0-9])
156
157 assign                  "="
158
159 %%
160
161 <INITIAL,BEGINID>"if"            {
162         if (last_token != DOT)
163     {
164         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
165     }
166     DEBUG("BEGIN(INITIAL)");
167     BEGIN(INITIAL);
168     return scan_throw(IF);
169 }
170
171 <INITIAL,BEGINID>"then"          {
172     DEBUG("BEGIN(INITIAL)");
173     BEGIN(INITIAL);
174     return scan_throw(THEN);
175 }
176
177 <INITIAL,BEGINID>"else"          {
178         if (last_token != DOT)
179     {
180         // Pop to step out IF
181         ParserSingleInstance::popControlStatus();
182         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
183     }
184     DEBUG("BEGIN(INITIAL)");
185     BEGIN(INITIAL);
186         return scan_throw(ELSE);
187 }
188
189 <INITIAL,BEGINID>"elseif" {
190         if (last_token != DOT)
191     {
192         ParserSingleInstance::popControlStatus();
193         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
194     }
195     DEBUG("BEGIN(INITIAL)");
196     BEGIN(INITIAL);
197         return scan_throw(ELSEIF);
198 }
199
200 <INITIAL,BEGINID>"end"          {
201         if (last_token != DOT)
202     {
203         ParserSingleInstance::popControlStatus();
204     }
205     DEBUG("BEGIN(INITIAL)");
206     BEGIN(INITIAL);
207     return scan_throw(END);
208 }
209
210 <INITIAL,BEGINID>"select"       {
211         if (last_token != DOT)
212     {
213         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
214     }
215     DEBUG("BEGIN(INITIAL)");
216     BEGIN(INITIAL);
217     return scan_throw(SELECT);
218 }
219
220 <INITIAL,BEGINID>"switch"       {
221         if (last_token != DOT)
222     {
223         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
224     }
225     DEBUG("BEGIN(INITIAL)");
226     BEGIN(INITIAL);
227     return scan_throw(SWITCH);
228 }
229
230 <INITIAL,BEGINID>"otherwise" {
231         if (last_token != DOT)
232     {
233         ParserSingleInstance::popControlStatus();
234         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
235     }
236     DEBUG("BEGIN(INITIAL)");
237     BEGIN(INITIAL);
238         return scan_throw(OTHERWISE);
239 }
240
241 <INITIAL,BEGINID>"case"         {
242         if (last_token != DOT)
243     {
244         ParserSingleInstance::popControlStatus();
245         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
246     }
247     DEBUG("BEGIN(INITIAL)");
248     BEGIN(INITIAL);
249     return scan_throw(CASE);
250 }
251
252 <INITIAL,BEGINID>"function" {
253         if (last_token != DOT)
254     {
255         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
256     }
257     DEBUG("BEGIN(INITIAL)");
258     BEGIN(INITIAL);
259     return scan_throw(FUNCTION);
260 }
261
262 <INITIAL,BEGINID>"endfunction" {
263         if (last_token != DOT)
264     {
265         ParserSingleInstance::popControlStatus();
266     }
267     DEBUG("BEGIN(INITIAL)");
268     BEGIN(INITIAL);
269         return scan_throw(ENDFUNCTION);
270 }
271
272 <INITIAL,BEGINID>"#function"    {
273         if (last_token != DOT)
274     {
275         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
276     }
277     DEBUG("BEGIN(INITIAL)");
278     BEGIN(INITIAL);
279         return scan_throw(HIDDENFUNCTION);
280 }
281
282 <INITIAL,BEGINID>"hidden"       {
283     DEBUG("BEGIN(INITIAL)");
284         BEGIN(INITIAL);
285     return scan_throw(HIDDEN);
286 }
287
288 <INITIAL,BEGINID>"for" {
289         if (last_token != DOT)
290     {
291         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
292     }
293     BEGIN(INITIAL);
294     return scan_throw(FOR);
295 }
296
297 <INITIAL,BEGINID>"while"        {
298         if (last_token != DOT)
299     {
300         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
301     }
302         BEGIN(INITIAL);
303         return scan_throw(WHILE);
304 }
305
306 <INITIAL,BEGINID>"do"           {
307         BEGIN(INITIAL);
308     return scan_throw(DO);
309 }
310
311 <INITIAL,BEGINID>"break"                {
312         BEGIN(INITIAL);
313         return scan_throw(BREAK);
314 }
315
316 <INITIAL,BEGINID>"continue"             {
317         BEGIN(INITIAL);
318         return scan_throw(CONTINUE);
319 }
320
321 <INITIAL,BEGINID>"try" {
322         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
323         BEGIN(INITIAL);
324         return scan_throw(TRY);
325 }
326
327 <INITIAL,BEGINID>"catch" {
328     // Pop to step out TRY
329         ParserSingleInstance::popControlStatus();
330         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
331         BEGIN(INITIAL);
332         return scan_throw(CATCH);
333 }
334
335 <INITIAL,BEGINID>"return"       {
336     BEGIN(INITIAL);
337     return scan_throw(RETURN);
338 }
339
340 ^{spaces}*/({id}){spaces}[^(=<>~@] {
341         BEGIN(BEGINID);
342 }
343
344 <BEGINID>
345 {
346     {id}                        {
347         wchar_t *pwText = to_wide_string(yytext);
348         if (yytext != NULL && pwText == NULL)
349         {
350             std::string str = "can not convert'";
351             str += yytext;
352             str += "' to UTF-8";
353             exit_status = SCAN_ERROR;
354             scan_error("can not convert string to UTF-8");
355         }
356         yylval.str = new std::wstring(pwText);
357         if (symbol::Context::getInstance()->get(*new symbol::Symbol(*yylval.str)) != NULL
358             && symbol::Context::getInstance()->get(*new symbol::Symbol(*yylval.str))->isCallable())
359         {
360             scan_throw(ID);
361             BEGIN(SHELLMODE);
362         }
363         else
364         {
365             BEGIN(INITIAL);
366             return scan_throw(ID);
367         }
368     }
369
370 }
371
372 <INITIAL,MATRIX>{boolnot}               {
373   return scan_throw(NOT);
374 }
375 <INITIAL,MATRIX>{dollar}                {
376   return scan_throw(DOLLAR);
377 }
378 <INITIAL,MATRIX>{booltrue}              {
379   return scan_throw(BOOLTRUE);
380 }
381 <INITIAL,MATRIX>{boolfalse}             {
382   return scan_throw(BOOLFALSE);
383 }
384 <INITIAL,MATRIX>{booland}               {
385   return scan_throw(AND);
386 }
387 <INITIAL,MATRIX>{boolandand}    {
388   return scan_throw(ANDAND);
389 }
390 <INITIAL,MATRIX>{boolor}                {
391   return scan_throw(OR);
392 }
393 <INITIAL,MATRIX>{booloror}              {
394   return scan_throw(OROR);
395 }
396
397
398 <INITIAL,MATRIX>{lparen}                {
399   return scan_throw(LPAREN);
400 }
401 <INITIAL,MATRIX>{rparen}                {
402   return scan_throw(RPAREN);
403 }
404
405
406 <INITIAL,MATRIX>{semicolon}             {
407         scan_step();
408   return scan_throw(SEMI);
409 }
410
411 <INITIAL,MATRIX>{comma}                 {
412         scan_step();
413   return scan_throw(COMMA);
414 }
415
416 <INITIAL,MATRIX>{colon}                 {
417   return scan_throw(COLON);
418 }
419
420
421 <INITIAL,MATRIX>{lbrace}                {
422   yy_push_state(MATRIX);
423   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
424   return scan_throw(LBRACE);
425 }
426
427 {rbrace}                        {
428   return scan_throw(RBRACE);
429 }
430
431
432 <INITIAL,MATRIX>{dotquote}              {
433   return scan_throw(DOTQUOTE);
434 }
435 <INITIAL,MATRIX>{dottimes}              {
436   return scan_throw(DOTTIMES);
437 }
438 <INITIAL,MATRIX>{dotrdivide}            {
439   return scan_throw(DOTRDIVIDE);
440 }
441 <INITIAL,MATRIX>{dotldivide}            {
442   return scan_throw(DOTLDIVIDE);
443 }
444 <INITIAL,MATRIX>{dotpower}              {
445   return scan_throw(DOTPOWER);
446 }
447
448
449 {minus}                                 {
450   return scan_throw(MINUS);
451 }
452 {plus}                                  {
453   return scan_throw(PLUS);
454 }
455 <INITIAL,MATRIX>{times}                 {
456   return scan_throw(TIMES);
457 }
458 <INITIAL,MATRIX>{rdivide}               {
459   return scan_throw(RDIVIDE);
460 }
461 <INITIAL,MATRIX>{ldivide}               {
462   return scan_throw(LDIVIDE);
463 }
464 <INITIAL,MATRIX>{power}                 {
465   return scan_throw(POWER);
466 }
467
468 <INITIAL,MATRIX>{krontimes}             {
469   return scan_throw(KRONTIMES);
470 }
471 <INITIAL,MATRIX>{kronrdivide}           {
472   return scan_throw(KRONRDIVIDE);
473 }
474 <INITIAL,MATRIX>{kronldivide}           {
475   return scan_throw(KRONLDIVIDE);
476 }
477
478
479 <INITIAL,MATRIX>{controltimes}          {
480     unput(yytext[yyleng - 1]);
481     return scan_throw(CONTROLTIMES);
482 }
483 <INITIAL,MATRIX>{controlrdivide}                {
484     unput(yytext[yyleng - 1]);
485     return scan_throw(CONTROLRDIVIDE);
486 }
487 <INITIAL,MATRIX>{controlldivide}                {
488     unput(yytext[yyleng - 1]);
489     return scan_throw(CONTROLLDIVIDE);
490 }
491
492
493 <INITIAL,MATRIX>{equal}                 {
494   return scan_throw(EQ);
495 }
496 <INITIAL,MATRIX>{notequal}              {
497   return scan_throw(NE);
498 }
499 <INITIAL,MATRIX>{lowerthan}             {
500   return scan_throw(LT);
501 }
502 <INITIAL,MATRIX>{greaterthan}           {
503   return scan_throw(GT);
504 }
505 <INITIAL,MATRIX>{lowerequal}            {
506   return scan_throw(LE);
507 }
508 <INITIAL,MATRIX>{greaterequal}          {
509   return scan_throw(GE);
510 }
511
512
513 <INITIAL,MATRIX>{assign}                {
514   return scan_throw(ASSIGN);
515  }
516
517
518 <INITIAL,MATRIX>{lbrack}                {
519   DEBUG("yy_push_state(MATRIX)");
520   yy_push_state(MATRIX);
521   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
522   return scan_throw(LBRACK);
523 }
524
525 <INITIAL>{rbrack}                               {
526   return scan_throw(RBRACK);
527 }
528
529
530 <INITIAL,MATRIX>{dot}                   {
531   return scan_throw(DOT);
532 }
533
534 <INITIAL>{next}                 {
535     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
536     yy_push_state(LINEBREAK);
537 }
538
539 <INITIAL,MATRIX>{integer}               {
540   yylval.number = atof(yytext);
541 #ifdef TOKENDEV
542   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
543 #endif
544 //  scan_step();
545   return scan_throw(VARINT);
546 }
547
548
549 <INITIAL,MATRIX>{floating}              {
550   scan_exponent_convert(yytext);
551   yylval.number = atof(yytext);
552 #ifdef TOKENDEV
553   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
554 #endif
555   scan_step();
556   return scan_throw(VARFLOAT);
557 }
558
559
560 <INITIAL,MATRIX>{number}                {
561   yylval.number = atof(yytext);
562 #ifdef TOKENDEV
563   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
564 #endif
565 //  scan_step();
566   return scan_throw(NUM);
567 }
568
569
570 <INITIAL,MATRIX>{little}                {
571   yylval.number = atof(yytext);
572 #ifdef TOKENDEV
573   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
574 #endif
575   scan_step();
576   return scan_throw(NUM);
577 }
578
579
580 <INITIAL,MATRIX>{id}                    {
581     wchar_t *pwText = to_wide_string(yytext);
582     if (yytext != NULL && pwText == NULL)
583     {
584         std::string str = "can not convert'";
585         str += yytext;
586         str += "' to UTF-8";
587         exit_status = SCAN_ERROR;
588         scan_error("can not convert string to UTF-8");
589     }
590     yylval.str = new std::wstring(pwText);
591 #ifdef TOKENDEV
592   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
593 #endif
594 //  scan_step();
595   return scan_throw(ID);
596 }
597
598
599 <INITIAL,MATRIX,SHELLMODE>{startblockcomment}   {
600   yylval.comment = new std::wstring();
601   comment_level = 1;
602   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
603   yy_push_state(REGIONCOMMENT);
604 }
605
606
607 <INITIAL,MATRIX,SHELLMODE>{startcomment}                {
608   pstBuffer = new std::string();
609   yy_push_state(LINECOMMENT);
610 }
611
612
613 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
614   pstBuffer = new std::string();
615   yy_push_state(DOUBLESTRING);
616 }
617
618
619 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
620   /*
621   ** Matrix Transposition special behaviour
622   ** ID' []' toto()' are transposition call
623   */
624   if (last_token == ID
625       || last_token == RBRACK
626       || last_token == RPAREN
627       || last_token == RBRACE
628       || last_token == VARINT
629       || last_token == VARFLOAT
630       || last_token == NUM
631       || last_token == BOOLTRUE
632       || last_token == BOOLFALSE)
633   {
634       return scan_throw(QUOTE);
635   }
636   else
637   {
638       pstBuffer = new std::string();
639       yy_push_state(SIMPLESTRING);
640   }
641 }
642
643
644 <INITIAL,MATRIX>{spaces}                {
645   scan_step();
646   scan_throw(SPACES);
647 }
648
649
650 <INITIAL>{newline}              {
651   yylloc.last_line += 1;
652   yylloc.last_column = 1;
653   scan_step();
654   if (last_token != EOL) {
655       return scan_throw(EOL);
656   }
657
658 }
659
660
661 <INITIAL,MATRIX>{blankline}             {
662   yylloc.last_line += 1;
663   yylloc.last_column = 1;
664   scan_step();
665   if (last_token != EOL)
666   {
667       return scan_throw(EOL);
668   }
669   scan_throw(EOL);
670 }
671
672 <INITIAL,MATRIX>{emptyline}             {
673   yylloc.last_line += 2;
674   yylloc.last_column = 1;
675   scan_step();
676   if (last_token != EOL)
677   {
678       return scan_throw(EOL);
679   }
680   scan_throw(EOL);
681 }
682 .                                       {
683     std::string str = "unexpected token '";
684     str += yytext;
685     str += "'";
686     exit_status = SCAN_ERROR;
687     scan_error(str);
688 }
689
690
691 <MATRIX>
692 {
693   {spaces}*{lparen} {
694       unput(yytext[yyleng -1]);
695       if (last_token == ID
696           || last_token == RPAREN
697           || last_token == QUOTE
698           || last_token == VARINT
699           || last_token == VARFLOAT
700           || last_token == NUM)
701       {
702           return scan_throw(COMMA);
703       }
704   }
705
706   {spaces}*{colon}{spaces}* {
707       return scan_throw(COLON);
708   }
709
710   {newline} {
711       yylloc.last_line += 1;
712       yylloc.last_column = 1;
713       if(last_token != DOTS && last_token != EOL)
714       {
715           return scan_throw(EOL);
716       }
717       scan_throw(EOL);
718   }
719
720
721   {rbrack}                              {
722     DEBUG("yy_pop_state()");
723     yy_pop_state();
724     ParserSingleInstance::popControlStatus();
725     return scan_throw(RBRACK);
726   }
727
728   {rbrace}                              {
729     yy_pop_state();
730     ParserSingleInstance::popControlStatus();
731     return scan_throw(RBRACE);
732   }
733
734   {plus}                                {
735     return scan_throw(PLUS);
736   }
737
738   {minus}                               {
739     return scan_throw(MINUS);
740   }
741
742   {spaces}({plus}|{minus}){integer}                     {
743    int i;
744     for (i = yyleng - 1 ; i >= 0 ; --i)
745       {
746         unput(yytext[i]);
747       }
748     yy_push_state(MATRIXMINUSID);
749     if (last_token != LBRACK
750         && last_token != EOL
751         && last_token != SEMI)
752       {
753         return scan_throw(COMMA);
754       }
755   }
756
757   {spaces}({plus}|{minus}){number}      {
758       int i;
759       for (i = yyleng - 1 ; i >= 0 ; --i)
760       {
761           unput(yytext[i]);
762       }
763       yy_push_state(MATRIXMINUSID);
764       if (last_token != LBRACK
765           && last_token != EOL
766           && last_token != SEMI)
767       {
768           return scan_throw(COMMA);
769       }
770   }
771
772   {spaces}({plus}|{minus}){floating}    {
773       int i;
774       for (i = yyleng - 1 ; i >= 0 ; --i)
775       {
776           unput(yytext[i]);
777       }
778       yy_push_state(MATRIXMINUSID);
779       if (last_token != LBRACK
780           && last_token != EOL
781           && last_token != SEMI)
782       {
783           return scan_throw(COMMA);
784       }
785   }
786
787   {spaces}({plus}|{minus}){little}      {
788       int i;
789       for (i = yyleng - 1 ; i >= 0 ; --i)
790       {
791           unput(yytext[i]);
792       }
793       yy_push_state(MATRIXMINUSID);
794       if (last_token != LBRACK
795           && last_token != EOL
796           && last_token != SEMI)
797       {
798           return scan_throw(COMMA);
799       }
800   }
801
802   {spaces}({minus}|{plus}){id}          {
803       int i;
804       for (i = yyleng - 1 ; i >= 0 ; --i)
805       {
806           unput(yytext[i]);
807       }
808       yy_push_state(MATRIXMINUSID);
809       if (last_token != LBRACK
810           && last_token != EOL
811           && last_token != SEMI
812           )
813       {
814           return scan_throw(COMMA);
815       }
816   }
817
818   .                                     {
819     std::string str = "unexpected token '";
820     str += yytext;
821     str += "' within a matrix.";
822     exit_status = SCAN_ERROR;
823     scan_error(str);
824   }
825
826   {next}{spaces}*{newline}          {
827       /* Just do nothing */
828       yylloc.last_line += 1;
829       yylloc.last_column = 1;
830       scan_step();
831       scan_throw(EOL);
832   }
833
834   {next}{spaces}*{startcomment}          {
835       /* Just do nothing */
836       pstBuffer = new std::string();
837       yy_push_state(LINECOMMENT);
838       scan_throw(DOTS);
839   }
840
841   <<EOF>>       {
842       yy_pop_state();
843   }
844 }
845
846 <MATRIXMINUSID>
847 {
848   {minus}                               {
849     return scan_throw(MINUS);
850   }
851
852   {plus}                                {
853      /* Do Nothing. */
854   }
855
856   {integer}                             {
857     yy_pop_state();
858     yylval.number = atof(yytext);
859 #ifdef TOKENDEV
860     std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
861 #endif
862     scan_step();
863     return scan_throw(VARINT);
864   }
865
866   {number}                              {
867     yy_pop_state();
868     yylval.number = atof(yytext);
869 #ifdef TOKENDEV
870     std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
871 #endif
872     scan_step();
873     return scan_throw(NUM);
874   }
875
876   {little}                              {
877     yy_pop_state();
878     yylval.number = atof(yytext);
879 #ifdef TOKENDEV
880     std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
881 #endif
882     scan_step();
883     return scan_throw(NUM);
884   }
885
886   {floating}                            {
887     yy_pop_state();
888     scan_exponent_convert(yytext);
889     yylval.number = atof(yytext);
890 #ifdef TOKENDEV
891     std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
892 #endif
893     scan_step();
894     return scan_throw(VARFLOAT);
895   }
896
897   {id}                                  {
898     yy_pop_state();
899     wchar_t* pwText = to_wide_string(yytext);
900     if (yytext != NULL && pwText == NULL)
901     {
902         std::string str = "can not convert'";
903         str += yytext;
904         str += "' to UTF-8";
905         exit_status = SCAN_ERROR;
906         scan_error("can not convert string to UTF-8");
907     }
908     yylval.str = new std::wstring(pwText);
909 #ifdef TOKENDEV
910     std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
911 #endif
912     scan_step();
913     return scan_throw(ID);
914   }
915
916   {spaces}                              {
917     /* Do Nothing. */
918   }
919
920   .                                     {
921     std::string str = "unexpected token '";
922     str += yytext;
923     str += "' within a matrix.";
924     exit_status = SCAN_ERROR;
925     scan_error(str);
926   }
927 }
928
929 <LINEBREAK>
930 {
931   {newline}                             {
932     yylloc.last_line += 1;
933     yylloc.last_column = 1;
934     scan_step();
935     yy_pop_state();
936     ParserSingleInstance::popControlStatus();
937   }
938
939   {startblockcomment}                   {
940     ++comment_level;
941     yy_push_state(REGIONCOMMENT);
942   }
943
944   {startcomment}                        {
945     scan_throw(DOTS);
946     pstBuffer = new std::string();
947     yy_push_state(LINECOMMENT);
948   }
949
950   {spaces}                              {
951       /* Do nothing... */
952   }
953
954   <<EOF>>       {
955       yy_pop_state();
956   }
957   .                                     {
958     std::string str = "unexpected token '";
959     str += yytext;
960     str += "' after line break with .. or ...";
961     exit_status = SCAN_ERROR;
962     scan_error(str);
963   }
964 }
965
966
967 <LINECOMMENT>
968 {
969   {newline}     {
970     //yylloc.last_line += 1;
971     //yylloc.last_column = 1;
972     //scan_step();
973     yy_pop_state();
974     for (int i = yyleng - 1 ; i >= 0 ; --i)
975     {
976         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
977         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
978         unput(yytext[i]);
979         yylloc.last_column--;
980     }
981     /*
982     ** To forgot comments after lines break
983     */
984     if (last_token != DOTS)
985     {
986         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
987         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
988         wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
989         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
990         if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
991         {
992             std::string str = "can not convert'";
993             str += pstBuffer->c_str();
994             str += "' to UTF-8";
995             exit_status = SCAN_ERROR;
996             scan_error("can not convert string to UTF-8");
997         }
998         yylval.comment = new std::wstring(pwstBuffer);
999         delete pstBuffer;
1000         FREE (pwstBuffer);
1001         return scan_throw(COMMENT);
1002     }
1003   }
1004
1005   <<EOF>>       {
1006     yy_pop_state();
1007     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1008     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1009     {
1010         std::string str = "can not convert'";
1011         str += pstBuffer->c_str();
1012         str += "' to UTF-8";
1013         exit_status = SCAN_ERROR;
1014         scan_error("can not convert string to UTF-8");
1015     }
1016     yylval.comment = new std::wstring(pwstBuffer);
1017     delete pstBuffer;
1018     FREE (pwstBuffer);
1019     return scan_throw(COMMENT);
1020   }
1021
1022   .         {
1023      // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
1024      // only translate to WCHAR_T when popping state.
1025      *pstBuffer += yytext;
1026   }
1027
1028 }
1029
1030
1031 <REGIONCOMMENT>
1032 {
1033   {endblockcomment}                             {
1034     --comment_level;
1035     if (comment_level == 0) {
1036       ParserSingleInstance::popControlStatus();
1037       yy_pop_state();
1038       //return scan_throw(BLOCKCOMMENT);
1039     }
1040   }
1041
1042   {startblockcomment}                           {
1043     ++comment_level;
1044     yy_push_state(REGIONCOMMENT);
1045   }
1046
1047   {newline}                                     {
1048     yylloc.last_line += 1;
1049     yylloc.last_column = 1;
1050     scan_step();
1051     *yylval.comment += L"\n//";
1052   }
1053
1054   .                                             {
1055       *yylval.comment += std::wstring(to_wide_string(yytext));
1056   }
1057
1058  <<EOF>>                                        {
1059       yy_pop_state();
1060 //    std::string str = "unexpected end of file in a comment";
1061 //    exit_status = SCAN_ERROR;
1062 //    scan_error(str);
1063   }
1064 }
1065
1066
1067 <SIMPLESTRING>
1068 {
1069   {dquote}{dquote}                              {
1070     *pstBuffer += "\"";
1071   }
1072
1073   {dquote}{quote}                               {
1074     *pstBuffer += "'";
1075   }
1076
1077   {quote}{dquote}                               {
1078     *pstBuffer += "\"";
1079   }
1080
1081   {quote}{quote}                                {
1082     *pstBuffer += "'";
1083   }
1084
1085   {quote}                                       {
1086     yy_pop_state();
1087     scan_step();
1088     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1089     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1090     {
1091         std::string str = "can not convert'";
1092         str += pstBuffer->c_str();
1093         str += "' to UTF-8";
1094         exit_status = SCAN_ERROR;
1095         scan_error("can not convert string to UTF-8");
1096     }
1097     yylval.str = new std::wstring(pwstBuffer);
1098     delete pstBuffer;
1099     FREE(pwstBuffer);
1100     return scan_throw(STR);
1101   }
1102
1103   {dquote}                  {
1104     std::string str = "Heterogeneous string detected, starting with ' and ending with \".";
1105     exit_status = SCAN_ERROR;
1106     scan_error(str);
1107   }
1108
1109   {next}{newline}           {
1110       /* Do nothing... Just skip */
1111   }
1112
1113   {newline}                                     {
1114     std::string str = "unexpected end of line in a string.";
1115     exit_status = SCAN_ERROR;
1116     scan_error(str);
1117     yylloc.last_line += 1;
1118     yylloc.last_column = 1;
1119   }
1120
1121   <<EOF>>                                       {
1122     std::string str = "unexpected end of file in a string.";
1123     exit_status = SCAN_ERROR;
1124     scan_error(str);
1125   }
1126
1127   .                                             {
1128     scan_step();
1129     *pstBuffer += yytext;
1130   }
1131 }
1132
1133
1134 <DOUBLESTRING>
1135 {
1136   {dquote}{dquote}                              {
1137     *pstBuffer += "\"";
1138   }
1139
1140   {dquote}{quote}                               {
1141     *pstBuffer += "'";
1142   }
1143
1144   {quote}{dquote}               {
1145     *pstBuffer += "\"";
1146   }
1147
1148   {quote}{quote}                                {
1149     *pstBuffer += "'";
1150   }
1151
1152   {dquote}                      {
1153     yy_pop_state();
1154     scan_step();
1155     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1156     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1157     {
1158         std::string str = "can not convert'";
1159         str += pstBuffer->c_str();
1160         str += "' to UTF-8";
1161         exit_status = SCAN_ERROR;
1162         scan_error("can not convert string to UTF-8");
1163     }
1164     yylval.str = new std::wstring(pwstBuffer);
1165     delete pstBuffer;
1166     FREE(pwstBuffer);
1167     return scan_throw(STR);
1168   }
1169
1170   {quote}                  {
1171     std::string str = "Heterogeneous string detected, starting with \" and ending with '.";
1172     exit_status = SCAN_ERROR;
1173     scan_error(str);
1174   }
1175
1176   {next}{newline}           {
1177       /* Do nothing... Just skip */
1178   }
1179
1180   {newline} {
1181     std::string str = "unexpected end of line in a string";
1182     exit_status = SCAN_ERROR;
1183     scan_error(str);
1184     yylloc.last_line += 1;
1185     yylloc.last_column = 1;
1186   }
1187
1188   <<EOF>>   {
1189     std::string str = "unexpected end of file in a string";
1190     exit_status = SCAN_ERROR;
1191     scan_error(str);
1192   }
1193
1194   .         {
1195    scan_step();
1196    *pstBuffer += yytext;
1197   }
1198 }
1199
1200
1201 <SHELLMODE>
1202 {
1203     {spaces}                    {
1204         if (last_token == ID)
1205         {
1206             scan_throw(SPACES);
1207             return ID;
1208         }
1209     }
1210
1211     {semicolon}                 {
1212         BEGIN(INITIAL);
1213         scan_step();
1214         return scan_throw(SEMI);
1215     }
1216
1217     {comma}                     {
1218         BEGIN(INITIAL);
1219         scan_step();
1220         return scan_throw(COMMA);
1221     }
1222
1223     {newline}                   {
1224         BEGIN(INITIAL);
1225         yylloc.last_line += 1;
1226         yylloc.last_column = 1;
1227         scan_step();
1228         return scan_throw(EOL);
1229     }
1230
1231     {assign} {
1232         if (last_token == STR)
1233         {
1234             yylval.str = new std::wstring(to_wide_string(yytext));
1235             return scan_throw(STR);
1236         }
1237         else
1238         {
1239             BEGIN(INITIAL);
1240             return scan_throw(ASSIGN);
1241         }
1242     }
1243
1244     {lparen} {
1245         if (last_token == STR)
1246         {
1247             yylval.str = new std::wstring(to_wide_string(yytext));
1248             return scan_throw(STR);
1249         }
1250         else
1251         {
1252             BEGIN(INITIAL);
1253             return scan_throw(LPAREN);
1254         }
1255     }
1256
1257     {lowerthan} {
1258         if (last_token == STR)
1259         {
1260             yylval.str = new std::wstring(to_wide_string(yytext));
1261             return scan_throw(STR);
1262         }
1263         else
1264         {
1265             BEGIN(INITIAL);
1266             return scan_throw(LT);
1267         }
1268     }
1269
1270     {greaterthan} {
1271         if (last_token == STR)
1272         {
1273             yylval.str = new std::wstring(to_wide_string(yytext));
1274             return scan_throw(STR);
1275         }
1276         else
1277         {
1278             BEGIN(INITIAL);
1279             return scan_throw(GT);
1280         }
1281     }
1282
1283     {boolnot} {
1284         if (last_token == STR)
1285         {
1286             yylval.str = new std::wstring(to_wide_string(yytext));
1287             return scan_throw(STR);
1288         }
1289         else
1290         {
1291             BEGIN(INITIAL);
1292             return scan_throw(NOT);
1293         }
1294     }
1295
1296
1297     [^ \t\v\f\r\n,;'"]+               {
1298         yylval.str = new std::wstring(to_wide_string(yytext));
1299         return scan_throw(STR);
1300     }
1301
1302     <<EOF>>                     {
1303         BEGIN(INITIAL);
1304     }
1305
1306 }
1307
1308 %%
1309
1310 int scan_throw(int token) {
1311   last_token = token;
1312 #ifdef DEV
1313   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1314 #endif
1315   return token;
1316 }
1317
1318 void scan_step() {
1319   yylloc.first_line = yylloc.last_line;
1320   yylloc.first_column = yylloc.last_column;
1321 }
1322
1323 void scan_error(std::string msg)
1324 {
1325     wchar_t* pstMsg = to_wide_string(msg.c_str());
1326
1327     //std::wcerr << pstMsg << std::endl;
1328     ParserSingleInstance::PrintError(pstMsg);
1329     ParserSingleInstance::setExitStatus(Parser::Failed);
1330     ParserSingleInstance::resetControlStatus();
1331     FREE(pstMsg);
1332     last_token = YYEOF;
1333     BEGIN(INITIAL);
1334 }
1335
1336 /*
1337 ** convert floating numbers to C standard
1338 ** 1.2d-3 -> 1.2e-3
1339 ** 1.2D-3 -> 1.2e-3
1340 */
1341 void scan_exponent_convert(char *in)
1342 {
1343   char *pString;
1344   while((pString=strpbrk(in,"d"))!=NULL)
1345     {
1346       *pString='e';
1347     }
1348   while((pString=strpbrk(in,"D"))!=NULL)
1349     {
1350       *pString='e';
1351     }
1352 }
1353
1354 #ifdef _MSC_VER
1355 int isatty (int desc)
1356 {
1357   return 0;
1358 }
1359 #endif