fix warning in ast module.
[scilab.git] / scilab / modules / ast / src / cpp / parse / flex / scanscilab.ll
1 %{                                                            /* -*- C++ -*- */
2 /*
3  *  Scilab ( http://www.scilab.org/ ) - This file is part of Scilab
4  *  Copyright (C) 2008-2012 - Scilab Enterprises - Bruno JOFRET
5  *
6  *  This file must be used under the terms of the CeCILL.
7  *  This source file is licensed as described in the file COPYING, which
8  *  you should have received as part of this distribution.  The terms
9  *  are also available at
10  *  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt
11  *
12  */
13 #include "isatty.hxx"
14 #include "parse.hxx"
15 #include "parser_private.hxx"
16
17 #include "context.hxx"
18
19 extern "C"
20 {
21 #include "charEncoding.h"
22 #include "sci_malloc.h"
23 }
24
25 static int comment_level = 0;
26 static int last_token = 0;
27 static int exit_status = PARSE_ERROR;
28 static std::string current_file;
29 static std::string program_name;
30
31 static std::string *pstBuffer;
32
33 #define YY_USER_ACTION                          \
34  yylloc.last_column += yyleng;
35
36 /* -*- Verbose Special Debug -*- */
37 //#define DEV
38 //#define TOKENDEV
39
40 #ifdef DEV
41 #define DEBUG(x) std::cout << "[DEBUG] " << x << std::endl;
42 #else
43 #define DEBUG(x) /* Nothing */
44 #endif
45
46 %}
47
48 %option stack
49 %option noyywrap
50
51 %x SIMPLESTRING
52 %x DOUBLESTRING
53 %x REGIONCOMMENT
54 %x LINECOMMENT
55 %x LINEBREAK
56
57 %x MATRIX
58 %x MATRIXMINUSID
59
60 %x SHELLMODE
61 %x BEGINID
62
63 spaces                  [ \t\v\f]+
64 integer                 [0-9]+
65 number                  [0-9]+[\.][0-9]*
66 little                  \.[0-9]+
67
68 floating                ({little}|{number}|{integer})[deDE][+-]?{integer}
69
70 hex             [0]x[0-9a-fA-F]+
71 oct             [0]o[0-7]+
72
73
74 utf2            ([\xC2-\xDF][\x80-\xBF])
75 utf31           ([\xE0][\xA0-\xBF][\x80-\xBF])
76 utf32           ([\xE1-\xEC][\x80-\xBF][\x80-\xBF])
77 utf33           ([\xED][\x80-\x9F][\x80-\xBF])
78 utf34           ([\xEE-\xEF][\x80-\xBF][\x80-\xBF])
79 utf41           ([\xF0][\x90-\xBF][\x80-\xBF][\x80-\xBF])
80 utf42           ([\xF1-\xF3][\x80-\xBF][\x80-\xBF][\x80-\xBF])
81 utf43           ([\xF4][\x80-\x8F][\x80-\xBF][\x80-\xBF])
82
83 utf3            ({utf31}|{utf32}|{utf33}|{utf34})
84 utf4            ({utf41}|{utf42}|{utf43})
85
86 utf             ({utf2}|{utf3}|{utf4})
87 id              ((([a-zA-Z_%#?]|{utf})([a-zA-Z_0-9#?$]|{utf})*)|([$]([a-zA-Z_0-9#?$]|{utf})+))
88
89
90 newline                 ("\r"|"\n"|"\r\n")
91 blankline               {spaces}+{newline}
92 emptyline       {newline}({spaces}|[,;])+{newline}
93 next                    \.\.+
94
95 boolnot                 ("@"|"~")
96 booltrue                ("%t"|"%T")
97 boolfalse               ("%f"|"%F")
98 booland                 ("&")
99 boolandand              ("&&")
100 boolor                  ("|")
101 booloror                ("||")
102
103 lbrack                  "["
104 rbrack                  "]"
105
106 lparen                  "("
107 rparen                  ")"
108
109 lbrace                  "{"
110 rbrace                  "}"
111
112 dollar                  "$"
113
114 semicolon               ";"
115 comma                   ","
116 colon                   ":"
117
118 startcomment            "//"
119 startblockcomment       "/*"
120 endblockcomment         "*/"
121
122 dquote                  "\""
123 quote                   "'"
124
125 dot             "."
126 dotquote                ".'"
127 dottimes                ".*"
128 dotrdivide              "./"
129 dotldivide              ".\\"
130 dotpower                (".^"|".**")
131
132 plus                    "+"
133 minus                   "-"
134 rdivide                 "/"
135 ldivide                 "\\"
136 times                   "*"
137 power                   ("^"|"**")
138
139 equal                   "="{spaces}*"="
140 notequal                ("~"{spaces}*"="|"@"{spaces}*"="|"<"{spaces}*">")
141 lowerthan               "<"
142 greaterthan             ">"
143 lowerequal              "<"{spaces}*"="
144 greaterequal            ">"{spaces}*"="
145
146 krontimes               ".*."
147 kronrdivide             "./."
148 kronldivide             ".\\."
149
150 controltimes    ("*."[^0-9])
151 controlrdivide  ("/."[^0-9])
152 controlldivide  ("\\."[^0-9])
153
154 assign                  "="
155
156 %%
157
158 <INITIAL,BEGINID>"if"            {
159         if (last_token != DOT)
160     {
161         ParserSingleInstance::pushControlStatus(Parser::WithinIf);
162     }
163     DEBUG("BEGIN(INITIAL)");
164     BEGIN(INITIAL);
165     return scan_throw(IF);
166 }
167
168 <INITIAL,BEGINID>"then"          {
169     DEBUG("BEGIN(INITIAL)");
170     BEGIN(INITIAL);
171     return scan_throw(THEN);
172 }
173
174 <INITIAL,BEGINID>"else"          {
175         if (last_token != DOT)
176     {
177         // Pop to step out IF
178         ParserSingleInstance::popControlStatus();
179         ParserSingleInstance::pushControlStatus(Parser::WithinElse);
180     }
181     DEBUG("BEGIN(INITIAL)");
182     BEGIN(INITIAL);
183         return scan_throw(ELSE);
184 }
185
186 <INITIAL,BEGINID>"elseif" {
187         if (last_token != DOT)
188     {
189         ParserSingleInstance::popControlStatus();
190         ParserSingleInstance::pushControlStatus(Parser::WithinElseIf);
191     }
192     DEBUG("BEGIN(INITIAL)");
193     BEGIN(INITIAL);
194         return scan_throw(ELSEIF);
195 }
196
197 <INITIAL,BEGINID>"end"          {
198         if (last_token != DOT)
199     {
200         ParserSingleInstance::popControlStatus();
201     }
202     DEBUG("BEGIN(INITIAL)");
203     BEGIN(INITIAL);
204     return scan_throw(END);
205 }
206
207 <INITIAL,BEGINID>"select"       {
208         if (last_token != DOT)
209     {
210         ParserSingleInstance::pushControlStatus(Parser::WithinSelect);
211     }
212     DEBUG("BEGIN(INITIAL)");
213     BEGIN(INITIAL);
214     return scan_throw(SELECT);
215 }
216
217 <INITIAL,BEGINID>"switch"       {
218         if (last_token != DOT)
219     {
220         ParserSingleInstance::pushControlStatus(Parser::WithinSwitch);
221     }
222     DEBUG("BEGIN(INITIAL)");
223     BEGIN(INITIAL);
224     return scan_throw(SWITCH);
225 }
226
227 <INITIAL,BEGINID>"otherwise" {
228         if (last_token != DOT)
229     {
230         ParserSingleInstance::popControlStatus();
231         ParserSingleInstance::pushControlStatus(Parser::WithinOtherwise);
232     }
233     DEBUG("BEGIN(INITIAL)");
234     BEGIN(INITIAL);
235         return scan_throw(OTHERWISE);
236 }
237
238 <INITIAL,BEGINID>"case"         {
239         if (last_token != DOT)
240     {
241         ParserSingleInstance::popControlStatus();
242         ParserSingleInstance::pushControlStatus(Parser::WithinCase);
243     }
244     DEBUG("BEGIN(INITIAL)");
245     BEGIN(INITIAL);
246     return scan_throw(CASE);
247 }
248
249 <INITIAL,BEGINID>"function" {
250         if (last_token != DOT)
251     {
252         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
253     }
254     DEBUG("BEGIN(INITIAL)");
255     BEGIN(INITIAL);
256     return scan_throw(FUNCTION);
257 }
258
259 <INITIAL,BEGINID>"endfunction" {
260         if (last_token != DOT)
261     {
262         ParserSingleInstance::popControlStatus();
263     }
264     DEBUG("BEGIN(INITIAL)");
265     BEGIN(INITIAL);
266         return scan_throw(ENDFUNCTION);
267 }
268
269 <INITIAL,BEGINID>"#function"    {
270         if (last_token != DOT)
271     {
272         ParserSingleInstance::pushControlStatus(Parser::WithinFunction);
273     }
274     DEBUG("BEGIN(INITIAL)");
275     BEGIN(INITIAL);
276         return scan_throw(HIDDENFUNCTION);
277 }
278
279 <INITIAL,BEGINID>"hidden"       {
280     DEBUG("BEGIN(INITIAL)");
281         BEGIN(INITIAL);
282     return scan_throw(HIDDEN);
283 }
284
285 <INITIAL,BEGINID>"for" {
286         if (last_token != DOT)
287     {
288         ParserSingleInstance::pushControlStatus(Parser::WithinFor);
289     }
290     BEGIN(INITIAL);
291     return scan_throw(FOR);
292 }
293
294 <INITIAL,BEGINID>"while"        {
295         if (last_token != DOT)
296     {
297         ParserSingleInstance::pushControlStatus(Parser::WithinWhile);
298     }
299         BEGIN(INITIAL);
300         return scan_throw(WHILE);
301 }
302
303 <INITIAL,BEGINID>"do"           {
304         BEGIN(INITIAL);
305     return scan_throw(DO);
306 }
307
308 <INITIAL,BEGINID>"break"                {
309         BEGIN(INITIAL);
310         return scan_throw(BREAK);
311 }
312
313 <INITIAL,BEGINID>"continue"             {
314         BEGIN(INITIAL);
315         return scan_throw(CONTINUE);
316 }
317
318 <INITIAL,BEGINID>"try" {
319         ParserSingleInstance::pushControlStatus(Parser::WithinTry);
320         BEGIN(INITIAL);
321         return scan_throw(TRY);
322 }
323
324 <INITIAL,BEGINID>"catch" {
325     // Pop to step out TRY
326         ParserSingleInstance::popControlStatus();
327         ParserSingleInstance::pushControlStatus(Parser::WithinCatch);
328         BEGIN(INITIAL);
329         return scan_throw(CATCH);
330 }
331
332 <INITIAL,BEGINID>"return"       {
333     BEGIN(INITIAL);
334     return scan_throw(RETURN);
335 }
336
337 <INITIAL,BEGINID>"resume"       {
338     BEGIN(INITIAL);
339     return scan_throw(RETURN);
340 }
341
342 ^{spaces}*/({id}){spaces}[^ \t\v\f(=<>~@] {
343         BEGIN(BEGINID);
344 }
345
346 <BEGINID>
347 {
348     {id}                        {
349         wchar_t *pwText = to_wide_string(yytext);
350         if (yytext != NULL && pwText == NULL)
351         {
352             std::string str = "can not convert'";
353             str += yytext;
354             str += "' to UTF-8";
355             exit_status = SCAN_ERROR;
356             scan_error("can not convert string to UTF-8");
357         }
358         yylval.str = new std::wstring(pwText);
359         FREE(pwText);
360         types::InternalType * pIT = symbol::Context::getInstance()->get(symbol::Symbol(*yylval.str));
361         if (pIT && pIT->isCallable())
362         {
363             scan_throw(ID);
364             BEGIN(SHELLMODE);
365         }
366         else
367         {
368             BEGIN(INITIAL);
369             return scan_throw(ID);
370         }
371     }
372
373 }
374
375 <INITIAL,MATRIX>{boolnot}               {
376   return scan_throw(NOT);
377 }
378 <INITIAL,MATRIX>{dollar}                {
379   return scan_throw(DOLLAR);
380 }
381 <INITIAL,MATRIX>{booltrue}              {
382   return scan_throw(BOOLTRUE);
383 }
384 <INITIAL,MATRIX>{boolfalse}             {
385   return scan_throw(BOOLFALSE);
386 }
387 <INITIAL,MATRIX>{booland}               {
388   return scan_throw(AND);
389 }
390 <INITIAL,MATRIX>{boolandand}    {
391   return scan_throw(ANDAND);
392 }
393 <INITIAL,MATRIX>{boolor}                {
394   return scan_throw(OR);
395 }
396 <INITIAL,MATRIX>{booloror}              {
397   return scan_throw(OROR);
398 }
399
400
401 <INITIAL,MATRIX>{lparen}                {
402   return scan_throw(LPAREN);
403 }
404 <INITIAL,MATRIX>{rparen}                {
405   return scan_throw(RPAREN);
406 }
407
408
409 <INITIAL,MATRIX>{semicolon}             {
410         scan_step();
411   return scan_throw(SEMI);
412 }
413
414 <INITIAL,MATRIX>{comma}                 {
415         scan_step();
416   return scan_throw(COMMA);
417 }
418
419 <INITIAL,MATRIX>{colon}                 {
420   return scan_throw(COLON);
421 }
422
423
424 <INITIAL,MATRIX>{lbrace}                {
425   yy_push_state(MATRIX);
426   ParserSingleInstance::pushControlStatus(Parser::WithinCell);
427   return scan_throw(LBRACE);
428 }
429
430 {rbrace}                        {
431   return scan_throw(RBRACE);
432 }
433
434
435 <INITIAL,MATRIX>{dotquote}              {
436   return scan_throw(DOTQUOTE);
437 }
438 <INITIAL,MATRIX>{dottimes}              {
439   return scan_throw(DOTTIMES);
440 }
441 <INITIAL,MATRIX>{dotrdivide}            {
442   return scan_throw(DOTRDIVIDE);
443 }
444 <INITIAL,MATRIX>{dotldivide}            {
445   return scan_throw(DOTLDIVIDE);
446 }
447 <INITIAL,MATRIX>{dotpower}              {
448   return scan_throw(DOTPOWER);
449 }
450
451
452 {minus}                                 {
453   return scan_throw(MINUS);
454 }
455 {plus}                                  {
456   return scan_throw(PLUS);
457 }
458 <INITIAL,MATRIX>{times}                 {
459   return scan_throw(TIMES);
460 }
461 <INITIAL,MATRIX>{rdivide}               {
462   return scan_throw(RDIVIDE);
463 }
464 <INITIAL,MATRIX>{ldivide}               {
465   return scan_throw(LDIVIDE);
466 }
467 <INITIAL,MATRIX>{power}                 {
468   return scan_throw(POWER);
469 }
470
471 <INITIAL,MATRIX>{krontimes}             {
472   return scan_throw(KRONTIMES);
473 }
474 <INITIAL,MATRIX>{kronrdivide}           {
475   return scan_throw(KRONRDIVIDE);
476 }
477 <INITIAL,MATRIX>{kronldivide}           {
478   return scan_throw(KRONLDIVIDE);
479 }
480
481
482 <INITIAL,MATRIX>{controltimes}          {
483     unput(yytext[yyleng - 1]);
484     return scan_throw(CONTROLTIMES);
485 }
486 <INITIAL,MATRIX>{controlrdivide}                {
487     unput(yytext[yyleng - 1]);
488     return scan_throw(CONTROLRDIVIDE);
489 }
490 <INITIAL,MATRIX>{controlldivide}                {
491     unput(yytext[yyleng - 1]);
492     return scan_throw(CONTROLLDIVIDE);
493 }
494
495
496 <INITIAL,MATRIX>{equal}                 {
497   return scan_throw(EQ);
498 }
499 <INITIAL,MATRIX>{notequal}              {
500   return scan_throw(NE);
501 }
502 <INITIAL,MATRIX>{lowerthan}             {
503   return scan_throw(LT);
504 }
505 <INITIAL,MATRIX>{greaterthan}           {
506   return scan_throw(GT);
507 }
508 <INITIAL,MATRIX>{lowerequal}            {
509   return scan_throw(LE);
510 }
511 <INITIAL,MATRIX>{greaterequal}          {
512   return scan_throw(GE);
513 }
514
515
516 <INITIAL,MATRIX>{assign}                {
517   return scan_throw(ASSIGN);
518  }
519
520
521 <INITIAL,MATRIX>{lbrack}                {
522   DEBUG("yy_push_state(MATRIX)");
523   yy_push_state(MATRIX);
524   ParserSingleInstance::pushControlStatus(Parser::WithinMatrix);
525   return scan_throw(LBRACK);
526 }
527
528 <INITIAL>{rbrack}                               {
529   return scan_throw(RBRACK);
530 }
531
532
533 <INITIAL,MATRIX>{dot}                   {
534   return scan_throw(DOT);
535 }
536
537 <INITIAL>{next}                 {
538     ParserSingleInstance::pushControlStatus(Parser::WithinDots);
539     yy_push_state(LINEBREAK);
540 }
541
542 <INITIAL,MATRIX>{integer}               {
543   yylval.number = atof(yytext);
544 #ifdef TOKENDEV
545   std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
546 #endif
547 //  scan_step();
548   return scan_throw(VARINT);
549 }
550
551
552 <INITIAL,MATRIX>{floating}              {
553   scan_exponent_convert(yytext);
554   yylval.number = atof(yytext);
555 #ifdef TOKENDEV
556   std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
557 #endif
558   scan_step();
559   return scan_throw(VARFLOAT);
560 }
561
562
563 <INITIAL,MATRIX>{number}                {
564   yylval.number = atof(yytext);
565 #ifdef TOKENDEV
566   std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
567 #endif
568 //  scan_step();
569   return scan_throw(NUM);
570 }
571
572
573 <INITIAL,MATRIX>{little}                {
574   yylval.number = atof(yytext);
575 #ifdef TOKENDEV
576   std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
577 #endif
578   scan_step();
579   return scan_throw(NUM);
580 }
581
582
583 <INITIAL,MATRIX>{id}                    {
584     wchar_t *pwText = to_wide_string(yytext);
585     if (yytext != NULL && pwText == NULL)
586     {
587         std::string str = "can not convert'";
588         str += yytext;
589         str += "' to UTF-8";
590         exit_status = SCAN_ERROR;
591         scan_error("can not convert string to UTF-8");
592     }
593     yylval.str = new std::wstring(pwText);
594     FREE(pwText);
595 #ifdef TOKENDEV
596   std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
597 #endif
598 //  scan_step();
599   return scan_throw(ID);
600 }
601
602
603 <INITIAL,MATRIX>{startblockcomment}     {
604   yylval.comment = new std::wstring();
605   comment_level = 1;
606   ParserSingleInstance::pushControlStatus(Parser::WithinBlockComment);
607   yy_push_state(REGIONCOMMENT);
608 }
609
610
611 <INITIAL,MATRIX>{startcomment}          {
612   pstBuffer = new std::string();
613   yy_push_state(LINECOMMENT);
614 }
615
616
617 <INITIAL,MATRIX,SHELLMODE>{dquote}              {
618   pstBuffer = new std::string();
619   yy_push_state(DOUBLESTRING);
620 }
621
622
623 <INITIAL,MATRIX,SHELLMODE>{quote}                       {
624   /*
625   ** Matrix Transposition special behaviour
626   ** ID' []' toto()' are transposition call
627   */
628   if (last_token == ID
629       || last_token == RBRACK
630       || last_token == RPAREN
631       || last_token == RBRACE
632       || last_token == VARINT
633       || last_token == VARFLOAT
634       || last_token == NUM
635       || last_token == BOOLTRUE
636       || last_token == BOOLFALSE)
637   {
638       return scan_throw(QUOTE);
639   }
640   else
641   {
642       pstBuffer = new std::string();
643       yy_push_state(SIMPLESTRING);
644   }
645 }
646
647
648 <INITIAL,MATRIX>{spaces}                {
649   scan_step();
650   scan_throw(SPACES);
651 }
652
653
654 <INITIAL>{newline}              {
655   yylloc.last_line += 1;
656   yylloc.last_column = 1;
657   scan_step();
658   if (last_token != EOL) {
659       return scan_throw(EOL);
660   }
661
662 }
663
664
665 <INITIAL,MATRIX>{blankline}             {
666   yylloc.last_line += 1;
667   yylloc.last_column = 1;
668   scan_step();
669   if (last_token != EOL)
670   {
671       return scan_throw(EOL);
672   }
673   scan_throw(EOL);
674 }
675
676 <INITIAL,MATRIX>{emptyline}             {
677   yylloc.last_line += 2;
678   yylloc.last_column = 1;
679   scan_step();
680   if (last_token != EOL)
681   {
682       return scan_throw(EOL);
683   }
684   scan_throw(EOL);
685 }
686 .                                       {
687     std::string str = "unexpected token '";
688     str += yytext;
689     str += "'";
690     exit_status = SCAN_ERROR;
691     scan_error(str);
692 }
693
694
695 <MATRIX>
696 {
697   {spaces}*{lparen} {
698       unput(yytext[yyleng -1]);
699       if (last_token == ID
700           || last_token == RPAREN
701           || last_token == QUOTE
702           || last_token == VARINT
703           || last_token == VARFLOAT
704           || last_token == NUM)
705       {
706           return scan_throw(COMMA);
707       }
708   }
709
710   {spaces}*{colon}{spaces}* {
711       return scan_throw(COLON);
712   }
713
714   {newline} {
715       yylloc.last_line += 1;
716       yylloc.last_column = 1;
717       if(last_token != DOTS && last_token != EOL)
718       {
719           return scan_throw(EOL);
720       }
721       scan_throw(EOL);
722   }
723
724
725   {rbrack}                              {
726     DEBUG("yy_pop_state()");
727     yy_pop_state();
728     ParserSingleInstance::popControlStatus();
729     return scan_throw(RBRACK);
730   }
731
732   {rbrace}                              {
733     yy_pop_state();
734     ParserSingleInstance::popControlStatus();
735     return scan_throw(RBRACE);
736   }
737
738   {plus}                                {
739     return scan_throw(PLUS);
740   }
741
742   {minus}                               {
743     return scan_throw(MINUS);
744   }
745
746   {spaces}({plus}|{minus}){lparen}*{integer}                    {
747    int i;
748    for (i = yyleng - 1 ; i >= 0 ; --i)
749    {
750        unput(yytext[i]);
751    }
752    yy_push_state(MATRIXMINUSID);
753    if (last_token != LBRACK
754        && last_token != EOL
755        && last_token != SEMI
756        && last_token != COMMA)
757    {
758        return scan_throw(COMMA);
759    }
760   }
761
762   {spaces}({plus}|{minus}){lparen}*{number}     {
763       int i;
764       for (i = yyleng - 1 ; i >= 0 ; --i)
765       {
766           unput(yytext[i]);
767       }
768       yy_push_state(MATRIXMINUSID);
769       if (last_token != LBRACK
770           && last_token != EOL
771           && last_token != SEMI
772           && last_token != COMMA)
773       {
774           return scan_throw(COMMA);
775       }
776   }
777
778   {spaces}({plus}|{minus}){lparen}*{floating}   {
779       int i;
780       for (i = yyleng - 1 ; i >= 0 ; --i)
781       {
782           unput(yytext[i]);
783       }
784       yy_push_state(MATRIXMINUSID);
785       if (last_token != LBRACK
786           && last_token != EOL
787           && last_token != SEMI
788           && last_token != COMMA)
789       {
790           return scan_throw(COMMA);
791       }
792   }
793
794   {spaces}({plus}|{minus}){lparen}*{little}     {
795       int i;
796       for (i = yyleng - 1 ; i >= 0 ; --i)
797       {
798           unput(yytext[i]);
799       }
800       yy_push_state(MATRIXMINUSID);
801       if (last_token != LBRACK
802           && last_token != EOL
803           && last_token != SEMI
804           && last_token != COMMA)
805       {
806           return scan_throw(COMMA);
807       }
808   }
809
810   {spaces}({minus}|{plus}){lparen}*{id}         {
811       int i;
812       for (i = yyleng - 1; i >= 0 ; --i)
813       {
814           unput(yytext[i]);
815       }
816       yy_push_state(MATRIXMINUSID);
817       if (last_token != LBRACK
818           && last_token != EOL
819           && last_token != SEMI
820           && last_token != COMMA)
821       {
822           return scan_throw(COMMA);
823       }
824   }
825   .                                     {
826     std::string str = "unexpected token '";
827     str += yytext;
828     str += "' within a matrix.";
829     exit_status = SCAN_ERROR;
830     scan_error(str);
831   }
832
833   {next}{spaces}*{newline}          {
834       /* Just do nothing */
835       yylloc.last_line += 1;
836       yylloc.last_column = 1;
837       scan_step();
838       scan_throw(EOL);
839   }
840
841   {next}{spaces}*{startcomment}          {
842       /* Just do nothing */
843       pstBuffer = new std::string();
844       yy_push_state(LINECOMMENT);
845       scan_throw(DOTS);
846   }
847
848   <<EOF>>       {
849       yy_pop_state();
850   }
851 }
852
853 <MATRIXMINUSID>
854 {
855   {minus}                               {
856     return scan_throw(MINUS);
857   }
858
859   {plus}                                {
860      /* Do Nothing. */
861   }
862
863   {integer}                             {
864     yy_pop_state();
865     yylval.number = atof(yytext);
866 #ifdef TOKENDEV
867     std::cout << "--> [DEBUG] INTEGER : " << yytext << std::endl;
868 #endif
869     scan_step();
870     return scan_throw(VARINT);
871   }
872
873   {number}                              {
874     yy_pop_state();
875     yylval.number = atof(yytext);
876 #ifdef TOKENDEV
877     std::cout << "--> [DEBUG] NUMBER : " << yytext << std::endl;
878 #endif
879     scan_step();
880     return scan_throw(NUM);
881   }
882
883   {little}                              {
884     yy_pop_state();
885     yylval.number = atof(yytext);
886 #ifdef TOKENDEV
887     std::cout << "--> [DEBUG] LITTLE : " << yytext << std::endl;
888 #endif
889     scan_step();
890     return scan_throw(NUM);
891   }
892
893   {floating}                            {
894     yy_pop_state();
895     scan_exponent_convert(yytext);
896     yylval.number = atof(yytext);
897 #ifdef TOKENDEV
898     std::cout << "--> [DEBUG] FLOATING : " << yytext << std::endl;
899 #endif
900     scan_step();
901     return scan_throw(VARFLOAT);
902   }
903
904   {id}                                  {
905     yy_pop_state();
906     wchar_t* pwText = to_wide_string(yytext);
907     if (yytext != NULL && pwText == NULL)
908     {
909         std::string str = "can not convert'";
910         str += yytext;
911         str += "' to UTF-8";
912         exit_status = SCAN_ERROR;
913         scan_error("can not convert string to UTF-8");
914     }
915     yylval.str = new std::wstring(pwText);
916     FREE(pwText);
917 #ifdef TOKENDEV
918     std::cout << "--> [DEBUG] ID : " << yytext << std::endl;
919 #endif
920     scan_step();
921     return scan_throw(ID);
922   }
923
924   {spaces}                              {
925     /* Do Nothing. */
926   }
927
928   {lparen} {
929       return scan_throw(LPAREN);
930   }
931
932   {rparen} {
933       return scan_throw(RPAREN);
934   }
935   .                                     {
936     std::string str = "unexpected token '";
937     str += yytext;
938     str += "' within a matrix.";
939     exit_status = SCAN_ERROR;
940     scan_error(str);
941   }
942 }
943
944 <LINEBREAK>
945 {
946   {newline}                             {
947     yylloc.last_line += 1;
948     yylloc.last_column = 1;
949     scan_step();
950     yy_pop_state();
951     ParserSingleInstance::popControlStatus();
952   }
953
954   {startblockcomment}                   {
955     ++comment_level;
956     yy_push_state(REGIONCOMMENT);
957   }
958
959   {startcomment}                        {
960     scan_throw(DOTS);
961     pstBuffer = new std::string();
962     yy_push_state(LINECOMMENT);
963   }
964
965   {spaces}                              {
966       /* Do nothing... */
967   }
968
969   <<EOF>>       {
970       yy_pop_state();
971   }
972   .                                     {
973     std::string str = "unexpected token '";
974     str += yytext;
975     str += "' after line break with .. or ...";
976     exit_status = SCAN_ERROR;
977     scan_error(str);
978   }
979 }
980
981
982 <LINECOMMENT>
983 {
984   {newline}     {
985     //yylloc.last_line += 1;
986     //yylloc.last_column = 1;
987     //scan_step();
988     yy_pop_state();
989     for (int i = yyleng - 1 ; i >= 0 ; --i)
990     {
991         //std::cerr << "Unputting i = {" << i << "}" << std::endl;
992         //std::cerr << "Unputting {" << yytext[i] << "}" << std::endl;
993         unput(yytext[i]);
994         yylloc.last_column--;
995     }
996     /*
997     ** To forgot comments after lines break
998     */
999     if (last_token != DOTS)
1000     {
1001         //std::cerr << "pstBuffer = {" << *pstBuffer << "}" << std::endl;
1002         //std::cerr << "pstBuffer->c_str() = {" << pstBuffer->c_str() << "}" << std::endl;
1003         wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1004         //std::wcerr << L"pwstBuffer = W{" << pwstBuffer << L"}" << std::endl;
1005         if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1006         {
1007             std::string str = "can not convert'";
1008             str += pstBuffer->c_str();
1009             str += "' to UTF-8";
1010             exit_status = SCAN_ERROR;
1011             scan_error("can not convert string to UTF-8");
1012         }
1013         yylval.comment = new std::wstring(pwstBuffer);
1014         delete pstBuffer;
1015         FREE (pwstBuffer);
1016         return scan_throw(COMMENT);
1017     }
1018   }
1019
1020   <<EOF>>       {
1021     yy_pop_state();
1022     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1023     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1024     {
1025         std::string str = "can not convert'";
1026         str += pstBuffer->c_str();
1027         str += "' to UTF-8";
1028         exit_status = SCAN_ERROR;
1029         scan_error("can not convert string to UTF-8");
1030     }
1031     yylval.comment = new std::wstring(pwstBuffer);
1032     delete pstBuffer;
1033     FREE (pwstBuffer);
1034     return scan_throw(COMMENT);
1035   }
1036
1037   .         {
1038      // Put the char in a temporary CHAR buffer to go through UTF-8 trouble
1039      // only translate to WCHAR_T when popping state.
1040      *pstBuffer += yytext;
1041   }
1042
1043 }
1044
1045
1046 <REGIONCOMMENT>
1047 {
1048   {endblockcomment}                             {
1049     --comment_level;
1050     if (comment_level == 0) {
1051       ParserSingleInstance::popControlStatus();
1052       yy_pop_state();
1053       //return scan_throw(BLOCKCOMMENT);
1054     }
1055   }
1056
1057   {startblockcomment}                           {
1058     ++comment_level;
1059     yy_push_state(REGIONCOMMENT);
1060   }
1061
1062   {newline}                                     {
1063     yylloc.last_line += 1;
1064     yylloc.last_column = 1;
1065     scan_step();
1066     *yylval.comment += L"\n//";
1067   }
1068
1069   .                                             {
1070       wchar_t *pwText = to_wide_string(yytext);
1071       *yylval.comment += std::wstring(pwText);
1072       FREE(pwText);
1073   }
1074
1075  <<EOF>>                                        {
1076       yy_pop_state();
1077 //    std::string str = "unexpected end of file in a comment";
1078 //    exit_status = SCAN_ERROR;
1079 //    scan_error(str);
1080   }
1081 }
1082
1083
1084 <SIMPLESTRING>
1085 {
1086   {dquote}{dquote}                              {
1087     *pstBuffer += "\"";
1088   }
1089
1090   {dquote}{quote}                               {
1091     *pstBuffer += "'";
1092   }
1093
1094   {quote}{dquote}                               {
1095     *pstBuffer += "\"";
1096   }
1097
1098   {quote}{quote}                                {
1099     *pstBuffer += "'";
1100   }
1101
1102   {quote}                                       {
1103     yy_pop_state();
1104     scan_step();
1105     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1106     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1107     {
1108         std::string str = "can not convert'";
1109         str += pstBuffer->c_str();
1110         str += "' to UTF-8";
1111         exit_status = SCAN_ERROR;
1112         scan_error("can not convert string to UTF-8");
1113     }
1114     yylval.str = new std::wstring(pwstBuffer);
1115     delete pstBuffer;
1116     FREE(pwstBuffer);
1117     return scan_throw(STR);
1118   }
1119
1120   {dquote}                  {
1121     std::string str = "Heterogeneous string detected, starting with ' and ending with \".";
1122     exit_status = SCAN_ERROR;
1123     scan_error(str);
1124   }
1125
1126   {next}{newline}           {
1127       /* Do nothing... Just skip */
1128   }
1129
1130   {newline}                                     {
1131     std::string str = "unexpected end of line in a string.";
1132     exit_status = SCAN_ERROR;
1133     scan_error(str);
1134     yylloc.last_line += 1;
1135     yylloc.last_column = 1;
1136   }
1137
1138   <<EOF>>                                       {
1139     std::string str = "unexpected end of file in a string.";
1140     exit_status = SCAN_ERROR;
1141     scan_error(str);
1142   }
1143
1144   .                                             {
1145     scan_step();
1146     *pstBuffer += yytext;
1147   }
1148 }
1149
1150
1151 <DOUBLESTRING>
1152 {
1153   {dquote}{dquote}                              {
1154     *pstBuffer += "\"";
1155   }
1156
1157   {dquote}{quote}                               {
1158     *pstBuffer += "'";
1159   }
1160
1161   {quote}{dquote}               {
1162     *pstBuffer += "\"";
1163   }
1164
1165   {quote}{quote}                                {
1166     *pstBuffer += "'";
1167   }
1168
1169   {dquote}                      {
1170     yy_pop_state();
1171     scan_step();
1172     wchar_t *pwstBuffer = to_wide_string(pstBuffer->c_str());
1173     if (pstBuffer->c_str() != NULL && pwstBuffer == NULL)
1174     {
1175         std::string str = "can not convert'";
1176         str += pstBuffer->c_str();
1177         str += "' to UTF-8";
1178         exit_status = SCAN_ERROR;
1179         scan_error("can not convert string to UTF-8");
1180     }
1181     yylval.str = new std::wstring(pwstBuffer);
1182     delete pstBuffer;
1183     FREE(pwstBuffer);
1184     return scan_throw(STR);
1185   }
1186
1187   {quote}                  {
1188     std::string str = "Heterogeneous string detected, starting with \" and ending with '.";
1189     exit_status = SCAN_ERROR;
1190     scan_error(str);
1191   }
1192
1193   {next}{newline}           {
1194       /* Do nothing... Just skip */
1195   }
1196
1197   {newline} {
1198     std::string str = "unexpected end of line in a string";
1199     exit_status = SCAN_ERROR;
1200     scan_error(str);
1201     yylloc.last_line += 1;
1202     yylloc.last_column = 1;
1203   }
1204
1205   <<EOF>>   {
1206     std::string str = "unexpected end of file in a string";
1207     exit_status = SCAN_ERROR;
1208     scan_error(str);
1209   }
1210
1211   .         {
1212    scan_step();
1213    *pstBuffer += yytext;
1214   }
1215 }
1216
1217
1218 <SHELLMODE>
1219 {
1220     {spaces}                    {
1221         if (last_token == ID)
1222         {
1223             scan_throw(SPACES);
1224             return ID;
1225         }
1226     }
1227
1228     {semicolon}                 {
1229         BEGIN(INITIAL);
1230         scan_step();
1231         return scan_throw(SEMI);
1232     }
1233
1234     {comma}                     {
1235         BEGIN(INITIAL);
1236         scan_step();
1237         return scan_throw(COMMA);
1238     }
1239
1240     {newline}                   {
1241         BEGIN(INITIAL);
1242         yylloc.last_line += 1;
1243         yylloc.last_column = 1;
1244         scan_step();
1245         return scan_throw(EOL);
1246     }
1247
1248     {assign} {
1249         if (last_token == STR)
1250         {
1251             wchar_t *pwText = to_wide_string(yytext);
1252             yylval.str = new std::wstring(pwText);
1253             FREE(pwText);
1254             return scan_throw(STR);
1255         }
1256         else
1257         {
1258             BEGIN(INITIAL);
1259             return scan_throw(ASSIGN);
1260         }
1261     }
1262
1263     {lparen} {
1264         if (last_token == STR)
1265         {
1266             wchar_t *pwText = to_wide_string(yytext);
1267             yylval.str = new std::wstring(pwText);
1268             FREE(pwText);
1269             return scan_throw(STR);
1270         }
1271         else
1272         {
1273             BEGIN(INITIAL);
1274             return scan_throw(LPAREN);
1275         }
1276     }
1277
1278     {lowerthan} {
1279         if (last_token == STR)
1280         {
1281             wchar_t *pwText = to_wide_string(yytext);
1282             yylval.str = new std::wstring(pwText);
1283             FREE(pwText);
1284             return scan_throw(STR);
1285         }
1286         else
1287         {
1288             BEGIN(INITIAL);
1289             return scan_throw(LT);
1290         }
1291     }
1292
1293     {greaterthan} {
1294         if (last_token == STR)
1295         {
1296             wchar_t *pwText = to_wide_string(yytext);
1297             yylval.str = new std::wstring(pwText);
1298             FREE(pwText);
1299             return scan_throw(STR);
1300         }
1301         else
1302         {
1303             BEGIN(INITIAL);
1304             return scan_throw(GT);
1305         }
1306     }
1307
1308     {boolnot} {
1309         if (last_token == STR)
1310         {
1311             wchar_t *pwText = to_wide_string(yytext);
1312             yylval.str = new std::wstring(pwText);
1313             FREE(pwText);
1314             return scan_throw(STR);
1315         }
1316         else
1317         {
1318             BEGIN(INITIAL);
1319             return scan_throw(NOT);
1320         }
1321     }
1322
1323
1324     [^ \t\v\f\r\n,;'"]+               {
1325         wchar_t *pwText = to_wide_string(yytext);
1326         yylval.str = new std::wstring(pwText);
1327         FREE(pwText);
1328         return scan_throw(STR);
1329     }
1330
1331     <<EOF>>                     {
1332         BEGIN(INITIAL);
1333     }
1334
1335 }
1336
1337 %%
1338
1339 int scan_throw(int token) {
1340   last_token = token;
1341 #ifdef DEV
1342   std::cout << "--> [DEBUG] TOKEN : " << token << std::endl;
1343 #endif
1344   return token;
1345 }
1346
1347 int get_last_token() {
1348     return last_token;
1349 }
1350
1351 void scan_step() {
1352   yylloc.first_line = yylloc.last_line;
1353   yylloc.first_column = yylloc.last_column;
1354 }
1355
1356 void scan_error(std::string msg)
1357 {
1358     wchar_t* pstMsg = to_wide_string(msg.c_str());
1359
1360     //std::wcerr << pstMsg << std::endl;
1361     ParserSingleInstance::PrintError(pstMsg);
1362     ParserSingleInstance::setExitStatus(Parser::Failed);
1363     ParserSingleInstance::resetControlStatus();
1364     FREE(pstMsg);
1365     last_token = YYEOF;
1366     BEGIN(INITIAL);
1367 }
1368
1369 /*
1370 ** convert floating numbers to C standard
1371 ** 1.2d-3 -> 1.2e-3
1372 ** 1.2D-3 -> 1.2e-3
1373 */
1374 void scan_exponent_convert(char *in)
1375 {
1376   char *pString;
1377   while((pString=strpbrk(in,"d"))!=NULL)
1378     {
1379       *pString='e';
1380     }
1381   while((pString=strpbrk(in,"D"))!=NULL)
1382     {
1383       *pString='e';
1384     }
1385 }
1386
1387 #ifdef _MSC_VER
1388 int isatty (int desc)
1389 {
1390   return 0;
1391 }
1392 #endif