]>
Commit | Line | Data |
---|---|---|
1eee94d3 GM |
1 | (* M2LexBuf.mod provides a buffer for m2.lex. |
2 | ||
83ffe9cd | 3 | Copyright (C) 2001-2023 Free Software Foundation, Inc. |
1eee94d3 GM |
4 | Contributed by Gaius Mulley <gaius.mulley@southwales.ac.uk>. |
5 | ||
6 | This file is part of GNU Modula-2. | |
7 | ||
8 | GNU Modula-2 is free software; you can redistribute it and/or modify | |
9 | it under the terms of the GNU General Public License as published by | |
10 | the Free Software Foundation; either version 3, or (at your option) | |
11 | any later version. | |
12 | ||
13 | GNU Modula-2 is distributed in the hope that it will be useful, but | |
14 | WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | General Public License for more details. | |
17 | ||
18 | You should have received a copy of the GNU General Public License | |
19 | along with GNU Modula-2; see the file COPYING3. If not see | |
20 | <http://www.gnu.org/licenses/>. *) | |
21 | ||
22 | IMPLEMENTATION MODULE M2LexBuf ; | |
23 | ||
24 | IMPORT m2flex ; | |
25 | ||
26 | FROM libc IMPORT strlen ; | |
27 | FROM SYSTEM IMPORT ADDRESS ; | |
28 | FROM Storage IMPORT ALLOCATE, DEALLOCATE ; | |
29 | FROM DynamicStrings IMPORT string, InitString, InitStringCharStar, Equal, Mark, KillString ; | |
30 | FROM FormatStrings IMPORT Sprintf1 ; | |
31 | FROM NameKey IMPORT NulName, Name, makekey, MakeKey, KeyToCharStar ; | |
32 | FROM M2Reserved IMPORT toktype, tokToTok ; | |
33 | FROM M2Printf IMPORT printf0, printf1, printf2, printf3 ; | |
34 | FROM M2Debug IMPORT Assert ; | |
35 | FROM NameKey IMPORT makekey ; | |
36 | FROM m2linemap IMPORT location_t, GetLocationBinary ; | |
37 | FROM M2Emit IMPORT UnknownLocation, BuiltinsLocation ; | |
38 | FROM M2Error IMPORT WarnStringAt ; | |
39 | ||
40 | CONST | |
41 | MaxBucketSize = 100 ; | |
42 | Debugging = FALSE ; | |
43 | DebugRecover = FALSE ; | |
44 | InitialSourceToken = 2 ; (* 0 is unknown, 1 is builtin. *) | |
45 | ||
46 | TYPE | |
47 | SourceList = POINTER TO RECORD | |
48 | left, | |
49 | right: SourceList ; | |
50 | name : String ; | |
51 | line : CARDINAL ; | |
52 | col : CARDINAL ; | |
53 | END ; | |
54 | ||
55 | TokenDesc = RECORD | |
56 | token : toktype ; | |
57 | str : Name ; (* ident name or string literal. *) | |
58 | int : INTEGER ; | |
59 | line : CARDINAL ; | |
60 | col : CARDINAL ; | |
61 | file : SourceList ; | |
62 | loc : location_t ; | |
63 | insert: TokenBucket ; (* contains any inserted tokens. *) | |
64 | END ; | |
65 | ||
66 | TokenBucket = POINTER TO RECORD | |
67 | buf : ARRAY [0..MaxBucketSize] OF TokenDesc ; | |
68 | len : CARDINAL ; | |
69 | next: TokenBucket ; | |
70 | END ; | |
71 | ||
72 | ListDesc = RECORD | |
73 | head, | |
74 | tail : TokenBucket ; | |
75 | LastBucketOffset: CARDINAL ; | |
76 | END ; | |
77 | ||
78 | VAR | |
79 | CurrentSource : SourceList ; | |
80 | UseBufferedTokens, | |
81 | CurrentUsed : BOOLEAN ; | |
82 | ListOfTokens : ListDesc ; | |
83 | CurrentTokNo : CARDINAL ; | |
84 | InsertionIndex : CARDINAL ; | |
85 | ||
86 | ||
87 | (* | |
88 | InitTokenList - creates an empty token list, which starts the first source token | |
89 | at position 2. This allows position 0 to be for unknown location | |
90 | and position 1 for builtin token. | |
91 | *) | |
92 | ||
93 | PROCEDURE InitTokenList ; | |
94 | BEGIN | |
95 | NEW (ListOfTokens.head) ; | |
96 | ListOfTokens.tail := ListOfTokens.head ; | |
97 | WITH ListOfTokens.tail^.buf[0] DO | |
98 | token := eoftok ; | |
99 | str := NulName ; | |
100 | int := 0 ; | |
101 | line := 0 ; | |
102 | col := 0 ; | |
103 | file := NIL ; | |
104 | loc := UnknownLocation () | |
105 | END ; | |
106 | WITH ListOfTokens.tail^.buf[1] DO | |
107 | token := eoftok ; | |
108 | str := NulName ; | |
109 | int := 0 ; | |
110 | line := 0 ; | |
111 | col := 0 ; | |
112 | file := NIL ; | |
113 | loc := BuiltinsLocation () | |
114 | END ; | |
115 | ListOfTokens.tail^.len := InitialSourceToken | |
116 | END InitTokenList ; | |
117 | ||
118 | ||
119 | (* | |
120 | Init - initializes the token list and source list. | |
121 | *) | |
122 | ||
123 | PROCEDURE Init ; | |
124 | BEGIN | |
125 | InsertionIndex := 0 ; | |
126 | currenttoken := eoftok ; | |
127 | CurrentTokNo := InitialSourceToken ; | |
128 | CurrentSource := NIL ; | |
129 | ListOfTokens.head := NIL ; | |
130 | ListOfTokens.tail := NIL ; | |
131 | UseBufferedTokens := FALSE ; | |
132 | InitTokenList | |
133 | END Init ; | |
134 | ||
135 | ||
136 | (* | |
137 | AddTo - adds a new element to the end of SourceList, CurrentSource. | |
138 | *) | |
139 | ||
140 | PROCEDURE AddTo (l: SourceList) ; | |
141 | BEGIN | |
142 | l^.right := CurrentSource ; | |
143 | l^.left := CurrentSource^.left ; | |
144 | CurrentSource^.left^.right := l ; | |
145 | CurrentSource^.left := l ; | |
146 | WITH l^.left^ DO | |
147 | line := m2flex.GetLineNo() ; | |
148 | col := m2flex.GetColumnNo() | |
149 | END | |
150 | END AddTo ; | |
151 | ||
152 | ||
153 | (* | |
154 | SubFrom - subtracts, l, from the source list. | |
155 | *) | |
156 | ||
157 | PROCEDURE SubFrom (l: SourceList) ; | |
158 | BEGIN | |
159 | l^.left^.right := l^.right ; | |
160 | l^.right^.left := l^.left | |
161 | END SubFrom ; | |
162 | ||
163 | ||
164 | (* | |
165 | NewElement - returns a new SourceList | |
166 | *) | |
167 | ||
168 | PROCEDURE NewElement (s: ADDRESS) : SourceList ; | |
169 | VAR | |
170 | l: SourceList ; | |
171 | BEGIN | |
172 | NEW (l) ; | |
173 | IF l = NIL | |
174 | THEN | |
175 | HALT | |
176 | ELSE | |
177 | WITH l^ DO | |
178 | name := InitStringCharStar(s) ; | |
179 | left := NIL ; | |
180 | right := NIL | |
181 | END | |
182 | END ; | |
183 | RETURN l | |
184 | END NewElement ; | |
185 | ||
186 | ||
187 | (* | |
188 | NewList - initializes an empty list with the classic dummy header element. | |
189 | *) | |
190 | ||
191 | PROCEDURE NewList () : SourceList ; | |
192 | VAR | |
193 | l: SourceList ; | |
194 | BEGIN | |
195 | NEW (l) ; | |
196 | WITH l^ DO | |
197 | left := l ; | |
198 | right := l ; | |
199 | name := NIL | |
200 | END ; | |
201 | RETURN l | |
202 | END NewList ; | |
203 | ||
204 | ||
205 | (* | |
206 | CheckIfNeedToDuplicate - checks to see whether the CurrentSource has | |
207 | been used, if it has then duplicate the list. | |
208 | *) | |
209 | ||
210 | PROCEDURE CheckIfNeedToDuplicate ; | |
211 | VAR | |
212 | l, h: SourceList ; | |
213 | BEGIN | |
214 | IF CurrentUsed | |
215 | THEN | |
216 | l := CurrentSource^.right ; | |
217 | h := CurrentSource ; | |
218 | CurrentSource := NewList() ; | |
219 | WHILE l#h DO | |
220 | AddTo (NewElement (l^.name)) ; | |
221 | l := l^.right | |
222 | END | |
223 | END | |
224 | END CheckIfNeedToDuplicate ; | |
225 | ||
226 | ||
227 | (* | |
228 | PushFile - indicates that, filename, has just been included. | |
229 | *) | |
230 | ||
231 | PROCEDURE PushFile (filename: ADDRESS) ; | |
232 | VAR | |
233 | l: SourceList ; | |
234 | BEGIN | |
235 | CheckIfNeedToDuplicate ; | |
236 | AddTo (NewElement (filename)) ; | |
237 | IF Debugging | |
238 | THEN | |
239 | IF CurrentSource^.right#CurrentSource | |
240 | THEN | |
241 | l := CurrentSource ; | |
242 | REPEAT | |
243 | printf3('name = %s, line = %d, col = %d\n', l^.name, l^.line, l^.col) ; | |
244 | l := l^.right | |
245 | UNTIL l=CurrentSource | |
246 | END | |
247 | END | |
248 | END PushFile ; | |
249 | ||
250 | ||
251 | (* | |
252 | PopFile - indicates that we are returning to, filename, having finished | |
253 | an include. | |
254 | *) | |
255 | ||
256 | PROCEDURE PopFile (filename: ADDRESS) ; | |
257 | VAR | |
258 | l: SourceList ; | |
259 | BEGIN | |
260 | CheckIfNeedToDuplicate ; | |
261 | IF (CurrentSource#NIL) AND (CurrentSource^.left#CurrentSource) | |
262 | THEN | |
263 | l := CurrentSource^.left ; (* last element *) | |
264 | SubFrom (l) ; | |
265 | DISPOSE (l) ; | |
266 | IF (CurrentSource^.left#CurrentSource) AND | |
267 | (NOT Equal(CurrentSource^.name, Mark(InitStringCharStar(filename)))) | |
268 | THEN | |
269 | (* mismatch in source file names after preprocessing files *) | |
270 | END | |
271 | ELSE | |
272 | (* source file list is empty, cannot pop an include.. *) | |
273 | END | |
274 | END PopFile ; | |
275 | ||
276 | ||
277 | (* | |
278 | KillList - kills the SourceList providing that it has not been used. | |
279 | *) | |
280 | ||
281 | PROCEDURE KillList ; | |
282 | VAR | |
283 | l, k: SourceList ; | |
284 | BEGIN | |
285 | IF (NOT CurrentUsed) AND (CurrentSource#NIL) | |
286 | THEN | |
287 | l := CurrentSource ; | |
288 | REPEAT | |
289 | k := l ; | |
290 | l := l^.right ; | |
291 | DISPOSE(k) | |
292 | UNTIL l=CurrentSource | |
293 | END | |
294 | END KillList ; | |
295 | ||
296 | ||
297 | (* | |
298 | ReInitialize - re-initialize the all the data structures. | |
299 | *) | |
300 | ||
301 | PROCEDURE ReInitialize ; | |
302 | VAR | |
303 | s, t: TokenBucket ; | |
304 | BEGIN | |
305 | IF ListOfTokens.head#NIL | |
306 | THEN | |
307 | t := ListOfTokens.head ; | |
308 | REPEAT | |
309 | s := t ; | |
310 | t := t^.next ; | |
311 | DISPOSE(s) ; | |
312 | UNTIL t=NIL ; | |
313 | CurrentUsed := FALSE ; | |
314 | KillList | |
315 | END ; | |
316 | Init | |
317 | END ReInitialize ; | |
318 | ||
319 | ||
320 | (* | |
321 | SetFile - sets the current filename to, filename. | |
322 | *) | |
323 | ||
324 | PROCEDURE SetFile (filename: ADDRESS) ; | |
325 | BEGIN | |
326 | KillList ; | |
327 | CurrentUsed := FALSE ; | |
328 | CurrentSource := NewList () ; | |
329 | AddTo (NewElement (filename)) | |
330 | END SetFile ; | |
331 | ||
332 | ||
333 | (* | |
334 | OpenSource - Attempts to open the source file, s. | |
335 | The success of the operation is returned. | |
336 | *) | |
337 | ||
338 | PROCEDURE OpenSource (s: String) : BOOLEAN ; | |
339 | BEGIN | |
340 | IF UseBufferedTokens | |
341 | THEN | |
342 | GetToken ; | |
343 | RETURN TRUE | |
344 | ELSE | |
345 | IF m2flex.OpenSource (string (s)) | |
346 | THEN | |
347 | SetFile (string (s)) ; | |
348 | SyncOpenWithBuffer ; | |
349 | GetToken ; | |
350 | RETURN TRUE | |
351 | ELSE | |
352 | RETURN FALSE | |
353 | END | |
354 | END | |
355 | END OpenSource ; | |
356 | ||
357 | ||
358 | (* | |
359 | CloseSource - closes the current open file. | |
360 | *) | |
361 | ||
362 | PROCEDURE CloseSource ; | |
363 | BEGIN | |
364 | IF UseBufferedTokens | |
365 | THEN | |
366 | WHILE currenttoken#eoftok DO | |
367 | GetToken | |
368 | END | |
369 | ELSE | |
370 | (* a subsequent call to m2flex.OpenSource will really close the file *) | |
371 | END | |
372 | END CloseSource ; | |
373 | ||
374 | ||
375 | (* | |
376 | ResetForNewPass - reset the buffer pointers to the beginning ready for | |
377 | a new pass | |
378 | *) | |
379 | ||
380 | PROCEDURE ResetForNewPass ; | |
381 | BEGIN | |
382 | InsertionIndex := 0 ; | |
383 | CurrentTokNo := InitialSourceToken ; | |
384 | UseBufferedTokens := TRUE | |
385 | END ResetForNewPass ; | |
386 | ||
387 | ||
388 | (* | |
389 | DisplayToken - display the token name using printf0 no newline is emitted. | |
390 | *) | |
391 | ||
392 | PROCEDURE DisplayToken (tok: toktype) ; | |
393 | BEGIN | |
394 | CASE tok OF | |
395 | ||
396 | eoftok: printf0('eoftok') | | |
397 | plustok: printf0('plustok') | | |
398 | minustok: printf0('minustok') | | |
399 | timestok: printf0('timestok') | | |
400 | dividetok: printf0('dividetok') | | |
401 | becomestok: printf0('becomestok') | | |
402 | ambersandtok: printf0('ambersandtok') | | |
403 | periodtok: printf0('periodtok') | | |
404 | commatok: printf0('commatok') | | |
405 | semicolontok: printf0('semicolontok') | | |
406 | lparatok: printf0('lparatok') | | |
407 | rparatok: printf0('rparatok') | | |
408 | lsbratok: printf0('lsbratok') | | |
409 | rsbratok: printf0('rsbratok') | | |
410 | lcbratok: printf0('lcbratok') | | |
411 | rcbratok: printf0('rcbratok') | | |
412 | uparrowtok: printf0('uparrowtok') | | |
413 | singlequotetok: printf0('singlequotetok') | | |
414 | equaltok: printf0('equaltok') | | |
415 | hashtok: printf0('hashtok') | | |
416 | lesstok: printf0('lesstok') | | |
417 | greatertok: printf0('greatertok') | | |
418 | lessgreatertok: printf0('lessgreatertok') | | |
419 | lessequaltok: printf0('lessequaltok') | | |
420 | greaterequaltok: printf0('greaterequaltok') | | |
421 | periodperiodtok: printf0('periodperiodtok') | | |
422 | colontok: printf0('colontok') | | |
423 | doublequotestok: printf0('doublequotestok') | | |
424 | bartok: printf0('bartok') | | |
425 | andtok: printf0('andtok') | | |
426 | arraytok: printf0('arraytok') | | |
427 | begintok: printf0('begintok') | | |
428 | bytok: printf0('bytok') | | |
429 | casetok: printf0('casetok') | | |
430 | consttok: printf0('consttok') | | |
431 | definitiontok: printf0('definitiontok') | | |
432 | divtok: printf0('divtok') | | |
433 | dotok: printf0('dotok') | | |
434 | elsetok: printf0('elsetok') | | |
435 | elsiftok: printf0('elsiftok') | | |
436 | endtok: printf0('endtok') | | |
437 | exittok: printf0('exittok') | | |
438 | exporttok: printf0('exporttok') | | |
439 | fortok: printf0('fortok') | | |
440 | fromtok: printf0('fromtok') | | |
441 | iftok: printf0('iftok') | | |
442 | implementationtok: printf0('implementationtok') | | |
443 | importtok: printf0('importtok') | | |
444 | intok: printf0('intok') | | |
445 | looptok: printf0('looptok') | | |
446 | modtok: printf0('modtok') | | |
447 | moduletok: printf0('moduletok') | | |
448 | nottok: printf0('nottok') | | |
449 | oftok: printf0('oftok') | | |
450 | ortok: printf0('ortok') | | |
451 | pointertok: printf0('pointertok') | | |
452 | proceduretok: printf0('proceduretok') | | |
453 | qualifiedtok: printf0('qualifiedtok') | | |
454 | unqualifiedtok: printf0('unqualifiedtok') | | |
455 | recordtok: printf0('recordtok') | | |
456 | repeattok: printf0('repeattok') | | |
457 | returntok: printf0('returntok') | | |
458 | settok: printf0('settok') | | |
459 | thentok: printf0('thentok') | | |
460 | totok: printf0('totok') | | |
461 | typetok: printf0('typetok') | | |
462 | untiltok: printf0('untiltok') | | |
463 | vartok: printf0('vartok') | | |
464 | whiletok: printf0('whiletok') | | |
465 | withtok: printf0('withtok') | | |
466 | asmtok: printf0('asmtok') | | |
467 | volatiletok: printf0('volatiletok') | | |
468 | periodperiodperiodtok: printf0('periodperiodperiodtok') | | |
469 | datetok: printf0('datetok') | | |
470 | linetok: printf0('linetok') | | |
471 | filetok: printf0('filetok') | | |
472 | integertok: printf0('integertok') | | |
473 | identtok: printf0('identtok') | | |
474 | realtok: printf0('realtok') | | |
475 | stringtok: printf0('stringtok') | |
476 | ||
477 | ELSE | |
478 | END | |
479 | END DisplayToken ; | |
480 | ||
481 | ||
482 | (* | |
483 | UpdateFromBucket - updates the global variables: currenttoken, | |
484 | currentstring, currentcolumn and currentinteger | |
485 | from TokenBucket, b, and, offset. | |
486 | *) | |
487 | ||
488 | PROCEDURE UpdateFromBucket (b: TokenBucket; offset: CARDINAL) ; | |
489 | BEGIN | |
490 | IF InsertionIndex > 0 | |
491 | THEN | |
492 | (* we have an inserted token to use. *) | |
493 | Assert (b^.buf[offset].insert # NIL) ; | |
494 | WITH b^.buf[offset].insert^.buf[InsertionIndex] DO | |
495 | currenttoken := token ; | |
496 | currentstring := KeyToCharStar(str) ; | |
497 | currentcolumn := col ; | |
498 | currentinteger := int ; | |
499 | IF Debugging | |
500 | THEN | |
501 | printf3('line %d (# %d %d) ', line, offset, CurrentTokNo) | |
502 | END | |
503 | END ; | |
504 | INC (InsertionIndex) ; | |
505 | IF InsertionIndex = b^.buf[offset].insert^.len | |
506 | THEN | |
507 | InsertionIndex := 0 ; (* finished consuming the inserted tokens. *) | |
508 | INC (CurrentTokNo) | |
509 | END | |
510 | ELSIF (b^.buf[offset].insert # NIL) AND (InsertionIndex = 0) | |
511 | THEN | |
512 | (* this source token has extra tokens appended after it by the error recovery. *) | |
513 | Assert (b^.buf[offset].insert^.len > 0) ; (* we must have at least one token. *) | |
514 | InsertionIndex := 1 ; (* so set the index ready for the next UpdateFromBucket. *) | |
515 | (* and read the original token. *) | |
516 | WITH b^.buf[offset] DO | |
517 | currenttoken := token ; | |
518 | currentstring := KeyToCharStar(str) ; | |
519 | currentcolumn := col ; | |
520 | currentinteger := int ; | |
521 | IF Debugging | |
522 | THEN | |
523 | printf3('line %d (# %d %d) ', line, offset, CurrentTokNo) | |
524 | END | |
525 | END | |
526 | ELSE | |
527 | (* no inserted tokens after this token so read it and move on. *) | |
528 | WITH b^.buf[offset] DO | |
529 | currenttoken := token ; | |
530 | currentstring := KeyToCharStar(str) ; | |
531 | currentcolumn := col ; | |
532 | currentinteger := int ; | |
533 | IF Debugging | |
534 | THEN | |
535 | printf3('line %d (# %d %d) ', line, offset, CurrentTokNo) | |
536 | END | |
537 | END ; | |
538 | INC (CurrentTokNo) | |
539 | END | |
540 | END UpdateFromBucket ; | |
541 | ||
542 | ||
543 | (* | |
544 | DisplayTokenEntry - | |
545 | *) | |
546 | ||
547 | PROCEDURE DisplayTokenEntry (topBucket: TokenBucket; index, total: CARDINAL) ; | |
548 | VAR | |
549 | i: CARDINAL ; | |
550 | BEGIN | |
551 | printf1 ("%d: ", total) ; | |
552 | DisplayToken (topBucket^.buf[index].token) ; | |
553 | printf1 (" %a ", topBucket^.buf[index].str) ; | |
554 | IF total = GetTokenNo () | |
555 | THEN | |
556 | printf0 (" <- current token") | |
557 | END ; | |
558 | printf0 ("\n") ; | |
559 | (* now check for inserted tokens. *) | |
560 | IF topBucket^.buf[index].insert # NIL | |
561 | THEN | |
562 | i := 1 ; | |
563 | WHILE i < topBucket^.buf[index].insert^.len DO | |
564 | printf1 (" %d: ", i) ; | |
565 | DisplayToken (topBucket^.buf[index].insert^.buf[i].token) ; | |
566 | printf1 (" %a\n", topBucket^.buf[index].insert^.buf[i].str) ; | |
567 | INC (i) | |
568 | END | |
569 | END | |
570 | END DisplayTokenEntry ; | |
571 | ||
572 | ||
573 | (* | |
574 | DumpTokens - developer debugging aid. | |
575 | *) | |
576 | ||
577 | PROCEDURE DumpTokens ; | |
578 | VAR | |
579 | tb : TokenBucket ; | |
580 | i, | |
581 | tokenNo, | |
582 | total, | |
583 | length : CARDINAL ; | |
584 | BEGIN | |
585 | tokenNo := GetTokenNo () ; | |
586 | tb := ListOfTokens.head ; | |
587 | total := 0 ; | |
588 | WHILE tb # NIL DO | |
589 | length := tb^.len ; | |
590 | i := 0 ; | |
591 | WHILE i < length DO | |
592 | DisplayTokenEntry (tb, i, total) ; | |
593 | INC (i) ; | |
594 | INC (total) | |
595 | END ; | |
596 | tb := tb^.next | |
597 | END ; | |
598 | printf2 ("%d: tokenNo, %d: total\n", tokenNo, total) ; | |
599 | IF (total # 0) AND (tokenNo = total) | |
600 | THEN | |
601 | printf1 ("%d: end of buffer ", total) ; | |
602 | printf0 (" <- current token") ; | |
603 | printf0 ("\n") ; | |
604 | END ; | |
605 | END DumpTokens ; | |
606 | ||
607 | ||
608 | (* | |
609 | GetToken - gets the next token into currenttoken. | |
610 | *) | |
611 | ||
612 | PROCEDURE GetToken ; | |
613 | VAR | |
614 | t: CARDINAL ; | |
615 | b: TokenBucket ; | |
616 | BEGIN | |
617 | IF UseBufferedTokens | |
618 | THEN | |
619 | t := CurrentTokNo ; | |
620 | b := FindTokenBucket(t) ; | |
621 | UpdateFromBucket (b, t) | |
622 | ELSE | |
623 | IF ListOfTokens.tail=NIL | |
624 | THEN | |
625 | m2flex.GetToken () ; | |
626 | IF ListOfTokens.tail=NIL | |
627 | THEN | |
628 | HALT | |
629 | END | |
630 | END ; | |
631 | IF CurrentTokNo>=ListOfTokens.LastBucketOffset | |
632 | THEN | |
633 | (* CurrentTokNo is in the last bucket or needs to be read *) | |
634 | IF CurrentTokNo-ListOfTokens.LastBucketOffset<ListOfTokens.tail^.len | |
635 | THEN | |
636 | UpdateFromBucket (ListOfTokens.tail, | |
637 | CurrentTokNo-ListOfTokens.LastBucketOffset) | |
638 | ELSE | |
639 | (* call the lexical phase to place a new token into the last bucket *) | |
640 | m2flex.GetToken () ; | |
641 | GetToken ; (* and call ourselves again to collect the token from bucket *) | |
642 | RETURN | |
643 | END | |
644 | ELSE | |
645 | t := CurrentTokNo ; | |
646 | b := FindTokenBucket (t) ; | |
647 | UpdateFromBucket (b, t) | |
648 | END | |
649 | END | |
650 | END GetToken ; | |
651 | ||
652 | ||
653 | (* | |
654 | SyncOpenWithBuffer - synchronise the buffer with the start of a file. | |
655 | Skips all the tokens to do with the previous file. | |
656 | *) | |
657 | ||
658 | PROCEDURE SyncOpenWithBuffer ; | |
659 | BEGIN | |
660 | IF ListOfTokens.tail#NIL | |
661 | THEN | |
662 | WITH ListOfTokens.tail^ DO | |
663 | CurrentTokNo := ListOfTokens.LastBucketOffset+len | |
664 | END | |
665 | END | |
666 | END SyncOpenWithBuffer ; | |
667 | ||
668 | ||
669 | (* | |
670 | GetInsertBucket - returns the insertion bucket associated with token count | |
671 | and the topBucket. It creates a new TokenBucket if necessary. | |
672 | *) | |
673 | ||
674 | PROCEDURE GetInsertBucket (topBucket: TokenBucket; count: CARDINAL) : TokenBucket ; | |
675 | BEGIN | |
676 | IF topBucket^.buf[count].insert = NIL | |
677 | THEN | |
678 | NEW (topBucket^.buf[count].insert) ; | |
679 | topBucket^.buf[count].insert^.buf[0] := topBucket^.buf[count] ; | |
680 | topBucket^.buf[count].insert^.buf[0].insert := NIL ; | |
681 | topBucket^.buf[count].insert^.len := 1 (* empty, slot 0 contains the original token for ease. *) | |
682 | END ; | |
683 | RETURN topBucket^.buf[count].insert | |
684 | END GetInsertBucket ; | |
685 | ||
686 | ||
687 | (* | |
688 | AppendToken - appends desc to the end of the insertionBucket. | |
689 | *) | |
690 | ||
691 | PROCEDURE AppendToken (insertionBucket: TokenBucket; desc: TokenDesc) ; | |
692 | BEGIN | |
693 | IF insertionBucket^.len < MaxBucketSize | |
694 | THEN | |
695 | insertionBucket^.buf[insertionBucket^.len] := desc ; | |
696 | INC (insertionBucket^.len) | |
697 | END | |
698 | END AppendToken ; | |
699 | ||
700 | ||
701 | (* | |
702 | InsertToken - inserts a symbol, token, infront of the current token | |
703 | ready for the next pass. | |
704 | *) | |
705 | ||
706 | PROCEDURE InsertToken (token: toktype) ; | |
707 | VAR | |
708 | topBucket, insertionBucket: TokenBucket ; | |
709 | count : CARDINAL ; | |
710 | desc : TokenDesc ; | |
711 | BEGIN | |
712 | Assert (ListOfTokens.tail # NIL) ; | |
713 | count := GetTokenNo () -1 ; | |
714 | topBucket := FindTokenBucket (count) ; | |
715 | insertionBucket := GetInsertBucket (topBucket, count) ; | |
716 | desc := topBucket^.buf[count] ; | |
717 | desc.token := token ; | |
718 | desc.insert := NIL ; | |
719 | AppendToken (insertionBucket, desc) ; | |
720 | IF DebugRecover | |
721 | THEN | |
722 | DumpTokens | |
723 | END | |
724 | END InsertToken ; | |
725 | ||
726 | ||
727 | (* | |
728 | InsertTokenAndRewind - inserts a symbol, token, infront of the current token | |
729 | and then moves the token stream back onto the inserted token. | |
730 | *) | |
731 | ||
732 | PROCEDURE InsertTokenAndRewind (token: toktype) ; | |
733 | VAR | |
734 | offset : CARDINAL ; | |
735 | topBucket: TokenBucket ; | |
736 | BEGIN | |
737 | IF GetTokenNo () > 0 | |
738 | THEN | |
739 | InsertToken (token) ; | |
740 | offset := CurrentTokNo -2 ; | |
741 | topBucket := FindTokenBucket (offset) ; | |
742 | InsertionIndex := topBucket^.buf[offset].insert^.len -1 ; | |
743 | DEC (CurrentTokNo, 2) ; | |
744 | GetToken | |
745 | END | |
746 | END InsertTokenAndRewind ; | |
747 | ||
748 | ||
749 | (* | |
750 | GetPreviousTokenLineNo - returns the line number of the previous token. | |
751 | *) | |
752 | ||
753 | PROCEDURE GetPreviousTokenLineNo () : CARDINAL ; | |
754 | BEGIN | |
755 | (* | |
756 | IF GetTokenNo()>0 | |
757 | THEN | |
758 | RETURN( TokenToLineNo(GetTokenNo()-1, 0) ) | |
759 | ELSE | |
760 | RETURN( 0 ) | |
761 | END | |
762 | *) | |
763 | RETURN GetLineNo () | |
764 | END GetPreviousTokenLineNo ; | |
765 | ||
766 | ||
767 | (* | |
768 | GetLineNo - returns the current line number where the symbol occurs in | |
769 | the source file. | |
770 | *) | |
771 | ||
772 | PROCEDURE GetLineNo () : CARDINAL ; | |
773 | BEGIN | |
774 | IF CurrentTokNo = 0 | |
775 | THEN | |
776 | RETURN 0 | |
777 | ELSE | |
778 | RETURN TokenToLineNo (GetTokenNo (), 0) | |
779 | END | |
780 | END GetLineNo ; | |
781 | ||
782 | ||
783 | (* | |
784 | GetColumnNo - returns the current column where the symbol occurs in | |
785 | the source file. | |
786 | *) | |
787 | ||
788 | PROCEDURE GetColumnNo () : CARDINAL ; | |
789 | BEGIN | |
790 | IF CurrentTokNo = 0 | |
791 | THEN | |
792 | RETURN 0 | |
793 | ELSE | |
794 | RETURN TokenToColumnNo (GetTokenNo (), 0) | |
795 | END | |
796 | END GetColumnNo ; | |
797 | ||
798 | ||
799 | (* | |
800 | GetTokenNo - returns the current token number. | |
801 | *) | |
802 | ||
803 | PROCEDURE GetTokenNo () : CARDINAL ; | |
804 | BEGIN | |
805 | IF CurrentTokNo = 0 | |
806 | THEN | |
807 | RETURN 0 | |
808 | ELSE | |
809 | RETURN CurrentTokNo-1 | |
810 | END | |
811 | END GetTokenNo ; | |
812 | ||
813 | ||
814 | (* | |
815 | GetTokenName - returns the token name given the tokenno. | |
816 | *) | |
817 | ||
818 | PROCEDURE GetTokenName (tokenno: CARDINAL) : Name ; | |
819 | VAR | |
820 | b: TokenBucket ; | |
821 | n: Name ; | |
822 | BEGIN | |
823 | b := FindTokenBucket (tokenno) ; | |
824 | IF b=NIL | |
825 | THEN | |
826 | RETURN NulName | |
827 | ELSE | |
828 | WITH b^.buf[tokenno] DO | |
829 | n := tokToTok (token) ; | |
830 | IF n=NulName | |
831 | THEN | |
832 | RETURN str | |
833 | ELSE | |
834 | RETURN n | |
835 | END | |
836 | END | |
837 | END | |
838 | END GetTokenName ; | |
839 | ||
840 | ||
841 | (* | |
842 | FindTokenBucket - returns the TokenBucket corresponding to the TokenNo. | |
843 | *) | |
844 | ||
845 | PROCEDURE FindTokenBucket (VAR TokenNo: CARDINAL) : TokenBucket ; | |
846 | VAR | |
847 | b: TokenBucket ; | |
848 | BEGIN | |
849 | b := ListOfTokens.head ; | |
850 | WHILE b#NIL DO | |
851 | WITH b^ DO | |
852 | IF TokenNo<len | |
853 | THEN | |
854 | RETURN b | |
855 | ELSE | |
856 | DEC (TokenNo, len) | |
857 | END | |
858 | END ; | |
859 | b := b^.next | |
860 | END ; | |
861 | RETURN NIL | |
862 | END FindTokenBucket ; | |
863 | ||
864 | ||
865 | (* | |
866 | TokenToLineNo - returns the line number of the current file for the | |
867 | TokenNo. The depth refers to the include depth. | |
868 | A depth of 0 is the current file, depth of 1 is the file | |
869 | which included the current file. Zero is returned if the | |
870 | depth exceeds the file nesting level. | |
871 | *) | |
872 | ||
873 | PROCEDURE TokenToLineNo (TokenNo: CARDINAL; depth: CARDINAL) : CARDINAL ; | |
874 | VAR | |
875 | b: TokenBucket ; | |
876 | l: SourceList ; | |
877 | BEGIN | |
878 | IF (TokenNo = UnknownTokenNo) OR (TokenNo = BuiltinTokenNo) | |
879 | THEN | |
880 | RETURN 0 | |
881 | ELSE | |
882 | b := FindTokenBucket (TokenNo) ; | |
883 | IF b = NIL | |
884 | THEN | |
885 | RETURN 0 | |
886 | ELSE | |
887 | IF depth = 0 | |
888 | THEN | |
889 | RETURN b^.buf[TokenNo].line | |
890 | ELSE | |
891 | l := b^.buf[TokenNo].file^.left ; | |
892 | WHILE depth>0 DO | |
893 | l := l^.left ; | |
894 | IF l=b^.buf[TokenNo].file^.left | |
895 | THEN | |
896 | RETURN 0 | |
897 | END ; | |
898 | DEC (depth) | |
899 | END ; | |
900 | RETURN l^.line | |
901 | END | |
902 | END | |
903 | END | |
904 | END TokenToLineNo ; | |
905 | ||
906 | ||
907 | (* | |
908 | TokenToColumnNo - returns the column number of the current file for the | |
909 | TokenNo. The depth refers to the include depth. | |
910 | A depth of 0 is the current file, depth of 1 is the file | |
911 | which included the current file. Zero is returned if the | |
912 | depth exceeds the file nesting level. | |
913 | *) | |
914 | ||
915 | PROCEDURE TokenToColumnNo (TokenNo: CARDINAL; depth: CARDINAL) : CARDINAL ; | |
916 | VAR | |
917 | b: TokenBucket ; | |
918 | l: SourceList ; | |
919 | BEGIN | |
920 | IF (TokenNo = UnknownTokenNo) OR (TokenNo = BuiltinTokenNo) | |
921 | THEN | |
922 | RETURN 0 | |
923 | ELSE | |
924 | b := FindTokenBucket (TokenNo) ; | |
925 | IF b=NIL | |
926 | THEN | |
927 | RETURN 0 | |
928 | ELSE | |
929 | IF depth = 0 | |
930 | THEN | |
931 | RETURN b^.buf[TokenNo].col | |
932 | ELSE | |
933 | l := b^.buf[TokenNo].file^.left ; | |
934 | WHILE depth>0 DO | |
935 | l := l^.left ; | |
936 | IF l=b^.buf[TokenNo].file^.left | |
937 | THEN | |
938 | RETURN 0 | |
939 | END ; | |
940 | DEC (depth) | |
941 | END ; | |
942 | RETURN l^.col | |
943 | END | |
944 | END | |
945 | END | |
946 | END TokenToColumnNo ; | |
947 | ||
948 | ||
949 | (* | |
950 | TokenToLocation - returns the location_t corresponding to, TokenNo. | |
951 | *) | |
952 | ||
953 | PROCEDURE TokenToLocation (TokenNo: CARDINAL) : location_t ; | |
954 | VAR | |
955 | b: TokenBucket ; | |
956 | BEGIN | |
957 | IF TokenNo = UnknownTokenNo | |
958 | THEN | |
959 | RETURN UnknownLocation () | |
960 | ELSIF TokenNo = BuiltinTokenNo | |
961 | THEN | |
962 | RETURN BuiltinsLocation () | |
963 | ELSE | |
964 | b := FindTokenBucket (TokenNo) ; | |
965 | IF b=NIL | |
966 | THEN | |
967 | RETURN UnknownLocation () | |
968 | ELSE | |
969 | RETURN b^.buf[TokenNo].loc | |
970 | END | |
971 | END | |
972 | END TokenToLocation ; | |
973 | ||
974 | ||
975 | (* | |
976 | FindFileNameFromToken - returns the complete FileName for the appropriate | |
977 | source file yields the token number, TokenNo. | |
978 | The, Depth, indicates the include level: 0..n | |
979 | Level 0 is the current. NIL is returned if n+1 | |
980 | is requested. | |
981 | *) | |
982 | ||
983 | PROCEDURE FindFileNameFromToken (TokenNo: CARDINAL; depth: CARDINAL) : String ; | |
984 | VAR | |
985 | b: TokenBucket ; | |
986 | l: SourceList ; | |
987 | BEGIN | |
988 | b := FindTokenBucket (TokenNo) ; | |
989 | IF b=NIL | |
990 | THEN | |
991 | RETURN NIL | |
992 | ELSE | |
993 | IF TokenNo = UnknownTokenNo | |
994 | THEN | |
995 | RETURN NIL | |
996 | ELSIF TokenNo = BuiltinTokenNo | |
997 | THEN | |
998 | RETURN NIL | |
999 | ELSE | |
1000 | l := b^.buf[TokenNo].file^.left ; | |
1001 | WHILE depth>0 DO | |
1002 | l := l^.left ; | |
1003 | IF l=b^.buf[TokenNo].file^.left | |
1004 | THEN | |
1005 | RETURN NIL | |
1006 | END ; | |
1007 | DEC (depth) | |
1008 | END ; | |
1009 | RETURN l^.name | |
1010 | END | |
1011 | END | |
1012 | END FindFileNameFromToken ; | |
1013 | ||
1014 | ||
1015 | (* | |
1016 | GetFileName - returns a String defining the current file. | |
1017 | *) | |
1018 | ||
1019 | PROCEDURE GetFileName () : String ; | |
1020 | BEGIN | |
1021 | RETURN FindFileNameFromToken (GetTokenNo (), 0) | |
1022 | END GetFileName ; | |
1023 | ||
1024 | ||
1025 | (* | |
1026 | AddTokToList - adds a token to a dynamic list. | |
1027 | *) | |
1028 | ||
1029 | PROCEDURE AddTokToList (t: toktype; n: Name; | |
1030 | i: INTEGER; l: CARDINAL; c: CARDINAL; | |
1031 | f: SourceList; location: location_t) ; | |
1032 | BEGIN | |
1033 | IF ListOfTokens.head=NIL | |
1034 | THEN | |
1035 | NEW (ListOfTokens.head) ; | |
1036 | IF ListOfTokens.head=NIL | |
1037 | THEN | |
1038 | (* list error *) | |
1039 | END ; | |
1040 | ListOfTokens.tail := ListOfTokens.head ; | |
1041 | ListOfTokens.tail^.len := 0 | |
1042 | ELSIF ListOfTokens.tail^.len=MaxBucketSize | |
1043 | THEN | |
1044 | Assert(ListOfTokens.tail^.next=NIL) ; | |
1045 | NEW (ListOfTokens.tail^.next) ; | |
1046 | IF ListOfTokens.tail^.next=NIL | |
1047 | THEN | |
1048 | (* list error *) | |
1049 | ELSE | |
1050 | ListOfTokens.tail := ListOfTokens.tail^.next ; | |
1051 | ListOfTokens.tail^.len := 0 | |
1052 | END ; | |
1053 | INC (ListOfTokens.LastBucketOffset, MaxBucketSize) | |
1054 | END ; | |
1055 | WITH ListOfTokens.tail^ DO | |
1056 | next := NIL ; | |
1057 | WITH buf[len] DO | |
1058 | token := t ; | |
1059 | str := n ; | |
1060 | int := i ; | |
1061 | line := l ; | |
1062 | col := c ; | |
1063 | file := f ; | |
1064 | loc := location ; | |
1065 | insert := NIL ; | |
1066 | END ; | |
1067 | INC (len) | |
1068 | END | |
1069 | END AddTokToList ; | |
1070 | ||
1071 | ||
1072 | (* | |
1073 | IsLastTokenEof - returns TRUE if the last token was an eoftok | |
1074 | *) | |
1075 | ||
1076 | PROCEDURE IsLastTokenEof () : BOOLEAN ; | |
1077 | VAR | |
1078 | b: TokenBucket ; | |
1079 | BEGIN | |
1080 | IF ListOfTokens.tail#NIL | |
1081 | THEN | |
1082 | IF ListOfTokens.tail^.len=0 | |
1083 | THEN | |
1084 | b := ListOfTokens.head ; | |
1085 | IF b=ListOfTokens.tail | |
1086 | THEN | |
1087 | RETURN FALSE | |
1088 | END ; | |
1089 | WHILE b^.next#ListOfTokens.tail DO | |
1090 | b := b^.next | |
1091 | END ; | |
1092 | ELSE | |
1093 | b := ListOfTokens.tail | |
1094 | END ; | |
1095 | WITH b^ DO | |
1096 | Assert (len>0) ; (* len should always be >0 *) | |
1097 | RETURN buf[len-1].token=eoftok | |
1098 | END | |
1099 | END ; | |
1100 | RETURN FALSE | |
1101 | END IsLastTokenEof ; | |
1102 | ||
1103 | ||
1104 | (* | |
1105 | PrintTokenNo - displays token and the location of the token. | |
1106 | *) | |
1107 | ||
1108 | PROCEDURE PrintTokenNo (tokenno: CARDINAL) ; | |
1109 | VAR | |
1110 | s: String ; | |
1111 | BEGIN | |
1112 | printf1 ("tokenno = %d, ", tokenno) ; | |
1113 | s := InitStringCharStar (KeyToCharStar (GetTokenName (tokenno))) ; | |
1114 | printf1 ("%s\n", s) ; | |
1115 | s := KillString (s) | |
1116 | END PrintTokenNo ; | |
1117 | ||
1118 | ||
1119 | (* | |
c8f2be5d GM |
1120 | isSrcToken - returns TRUE if tokenno is associated with |
1121 | program source code. | |
1eee94d3 GM |
1122 | *) |
1123 | ||
1124 | PROCEDURE isSrcToken (tokenno: CARDINAL) : BOOLEAN ; | |
1125 | BEGIN | |
1126 | RETURN (tokenno # UnknownTokenNo) AND (tokenno # BuiltinTokenNo) | |
1127 | END isSrcToken ; | |
1128 | ||
1129 | ||
1130 | (* | |
1131 | MakeVirtualTok - providing caret, left, right are associated with a source file | |
1132 | and exist on the same src line then | |
1133 | create and return a new tokenno which is created from | |
1134 | tokenno range1 and range2. Otherwise return caret. | |
1135 | *) | |
1136 | ||
1137 | PROCEDURE MakeVirtualTok (caret, left, right: CARDINAL) : CARDINAL ; | |
1138 | VAR | |
1139 | bufLeft, bufRight: TokenBucket ; | |
1140 | lc, ll, lr : location_t ; | |
1141 | BEGIN | |
1eee94d3 GM |
1142 | IF isSrcToken (caret) AND isSrcToken (left) AND isSrcToken (right) |
1143 | THEN | |
1144 | lc := TokenToLocation (caret) ; | |
1145 | ll := TokenToLocation (left) ; | |
1146 | lr := TokenToLocation (right) ; | |
1147 | bufLeft := FindTokenBucket (left) ; (* left maybe changed now. *) | |
1148 | bufRight := FindTokenBucket (right) ; (* right maybe changed now. *) | |
1149 | ||
1150 | IF (bufLeft^.buf[left].line = bufRight^.buf[right].line) AND | |
1151 | (bufLeft^.buf[left].file = bufRight^.buf[right].file) | |
1152 | THEN | |
1153 | (* on the same line, create a new token and location. *) | |
1154 | AddTokToList (virtualrangetok, NulName, 0, | |
1155 | bufLeft^.buf[left].line, bufLeft^.buf[left].col, bufLeft^.buf[left].file, | |
1156 | GetLocationBinary (lc, ll, lr)) ; | |
1157 | RETURN ListOfTokens.LastBucketOffset + ListOfTokens.tail^.len - 1 | |
1158 | END | |
1159 | END ; | |
1160 | RETURN caret | |
1161 | END MakeVirtualTok ; | |
1162 | ||
1163 | ||
1164 | (* *********************************************************************** | |
1165 | * | |
1166 | * These functions allow m2.flex to deliver tokens into the buffer | |
1167 | * | |
1168 | ************************************************************************* *) | |
1169 | ||
1170 | (* | |
1171 | AddTok - adds a token to the buffer. | |
1172 | *) | |
1173 | ||
1174 | PROCEDURE AddTok (t: toktype) ; | |
1175 | VAR | |
1176 | s: String ; | |
1177 | BEGIN | |
1178 | IF NOT ((t=eoftok) AND IsLastTokenEof()) | |
1179 | THEN | |
1180 | AddTokToList(t, NulName, 0, | |
1181 | m2flex.GetLineNo(), m2flex.GetColumnNo(), CurrentSource, | |
1182 | m2flex.GetLocation()) ; | |
1183 | CurrentUsed := TRUE ; | |
1184 | IF Debugging | |
1185 | THEN | |
1186 | (* display each token as a warning. *) | |
1187 | s := InitStringCharStar (KeyToCharStar (GetTokenName (GetTokenNo ()))) ; | |
1188 | WarnStringAt (s, GetTokenNo ()) | |
1189 | END | |
1190 | END | |
1191 | END AddTok ; | |
1192 | ||
1193 | ||
1194 | (* | |
1195 | AddTokCharStar - adds a token to the buffer and an additional string, s. | |
1196 | A copy of string, s, is made. | |
1197 | *) | |
1198 | ||
1199 | PROCEDURE AddTokCharStar (t: toktype; s: ADDRESS) ; | |
1200 | BEGIN | |
1201 | AddTokToList(t, makekey(s), 0, m2flex.GetLineNo(), | |
1202 | m2flex.GetColumnNo(), CurrentSource, m2flex.GetLocation()) ; | |
1203 | CurrentUsed := TRUE | |
1204 | END AddTokCharStar ; | |
1205 | ||
1206 | ||
1207 | (* | |
1208 | AddTokInteger - adds a token and an integer to the buffer. | |
1209 | *) | |
1210 | ||
1211 | PROCEDURE AddTokInteger (t: toktype; i: INTEGER) ; | |
1212 | VAR | |
1213 | s: String ; | |
1214 | c, | |
1215 | l: CARDINAL ; | |
1216 | BEGIN | |
1217 | l := m2flex.GetLineNo() ; | |
1218 | c := m2flex.GetColumnNo() ; | |
1219 | s := Sprintf1(Mark(InitString('%d')), i) ; | |
1220 | AddTokToList(t, makekey(string(s)), i, l, c, CurrentSource, m2flex.GetLocation()) ; | |
1221 | s := KillString(s) ; | |
1222 | CurrentUsed := TRUE | |
1223 | END AddTokInteger ; | |
1224 | ||
1225 | ||
1226 | BEGIN | |
1227 | Init | |
1228 | END M2LexBuf. |