head	1.4;
access;
symbols;
locks
	gtoal:1.4; strict;
comment	@ * @;


1.4
date	2022.02.19.02.47.57;	author gtoal;	state Exp;
branches;
next	1.3;

1.3
date	2022.02.18.06.42.58;	author gtoal;	state Exp;
branches;
next	1.2;

1.2
date	2022.02.18.06.37.37;	author gtoal;	state Exp;
branches;
next	1.1;

1.1
date	2022.02.15.00.56.06;	author gtoal;	state Exp;
branches;
next	;


desc
@no longer in synch with parser
@


1.4
log
@*** empty log message ***
@
text
@// For reasons to be explained later, the code in the on-demand source-fetching
// must not use the flex mechanism if there is any possibility of 'c' itself
// being reallocated while evaluating the index of a c(x) reference.  Space for
// tokens in the c[] array must have been pre-claimed by makespace before a
// non-existent c(x) entry is accessed.  Hence c() was not defined by the
// usual FLEX macro and the use of String() within the line reconstruction
// and any code that it calls (eg stores()) must use c[] rather than c(), and
// the alternative CString() ...

#define CString(i) &stringpool(c[i].sIndex)

int is_binary(int ch) {
  return ((ch|1) == '1');
}

int is_octal(int ch) {
  return ('0' <= ch) && (ch <= '7'); // or...  (ch|7) =='7'
}

int is_decimal(int ch) {
  return ('0' <= ch) && (ch <= '9');
}

int is_hex(int ch) {
  return (('0' <= ch) && (ch <= '9')) || (('a' <= ch) && (ch <= 'f')) || (('A' <= ch) && (ch <= 'F'));
}

static int lastch = EOF, lastch_pos = 0, lastch_lineno = 0, lastch_col = 0, lastch_peek = 0;
static int xfgetc (void) {
  static int ch;
  lastch = ch; lastch_pos = nextch_pos; lastch_lineno = nextch_lineno; lastch_col = nextch_col; lastch_peek = peek;
  if (nextch_pos >= source_length) return EOF;
  ch = source_address(nextch_pos);
  if (ch == '\n') {
    nextch_lineno += 1; nextch_col = 0;
  } else nextch_col += 1;
  //if (debug_line_reconstruction) fprintf(stderr, "Ch %d of %d: {%c}\n", nextch_pos, source_length, ch);
  peek = source_address(++nextch_pos);
  
  return ch&255;
}

static void xungetc (int c, FILE * f) { // for a xfgetc char, not for peek.
  // rather than subtracting 1 from postions it may be better to save all
  // the positions in xfgetc and unwind them, thus avoiding the issue of
  // whether we're ungetting a token or whitespace.  Or supply two xungetc's...
  if (nextch_pos > 0) {
    nextch_pos -= 1;
    if (nextch_col == 0) {
      int p;
      nextch_lineno -= 1;
      p = source_line(nextch_lineno);
      while (source_address(p) != '\n') {
        if (p == source_length) break;
        nextch_col += 1; p += 1;
      }
    } else nextch_col -= 1;
  } else {
    // invalid ungetc. nothing to unget at start of file.
  }
  ch = source_address(nextch_pos-1);//+1
  //  if (c == '\n') {
  //    nextch_lineno += 1; nextch_col = 0;
  //  } else nextch_col += 1;
  if (nextch_pos == source_length) peek = EOF; else peek = source_address(nextch_pos);//+1
}

void em_char_in_stmnt(int line) {
  // (a little nod to the past...)
  fprintf(stderr, "EM CHAR IN STMNT - DISASTER (detected at parser line %d)\n", line); // stop-gap measure - needs a longjump to end of compile code to clean up
  exit(EXIT_FAILURE);
}
#define em_char_in_stmnt() em_char_in_stmnt(__LINE__)

void line_reconstruction (void) {
  int (*acceptable_range)(int ch);
  // adds one more item to c[] array.
  // It is up to the caller to keep calling until sufficient data is loaded.
  static int token_nextfree, startcol;
  static int token_arraysize = 0;
  static char *token = NULL;
#define token(x) FLEX(token, x, __LINE__)
  static int init=0;
  
  // Recognise single and double-quoted strings
  // while removing comments and handling line continuation.
  // The c[] array gets the canonicalised version of the text, BUT retains
  // pointers to the original text including whitespace, which comes in
  // handy for any source-to-source transformations such as indent (pretty printing)
  // or transpiling to another language (eg C to Javascript)
  if (init == 0) {
    at_startofline = TRUE;
    makespace(token,1024); // quick hack only for test.imp fix as soon as lexer is written.
    init = 1;
  }
  
  /* Pre-process input ready for parsing.  Tokens are stored in array c[] */
  // NOTE: we are not yet handling include files...

  for (;;) {
    startcol = nextch_col;
    token_nextfree = 0;
    token_lineno = nextch_lineno;
    token_col = nextch_col;
    token_pos = nextch_pos; // actual text of token may start here but whitespace goes back to end of previous token.
 
    if ((ch = xfgetc ()) == EOF) {
      // set up a dummy at the end because we sometimes look ahead by 1
      // in the parsing code and don't want to hit uninitialised data.
      c[c_nextfree].f = curfile;

      c[c_nextfree].sIndex = str_to_pool("<EOF>"); // however we want to be able to print B_EOF and have any blank lines
      c[c_nextfree].t = B_EOF;                     // or comments before the end-of-file be output... so maybe "" would be better?

      c[c_nextfree].lineno_sp = whitespace_lineno;
      c[c_nextfree].col_sp = whitespace_col;
      c[c_nextfree].start_sp = whitespace_pos;

      c[c_nextfree].lineno_tok = nextch_lineno;
      c[c_nextfree].col_tok = nextch_col;
      c[c_nextfree].start_tok = nextch_pos;

      c[c_nextfree].end_tok = nextch_pos;
      c_nextfree++;

      #ifdef NEVER
      if (debug_tokens || debug_line_reconstruction) { // print tokens once parser hits end of file on input stream
        int i;
        fprintf (stderr, "\nLexical token stream:\n\n");
        for (i = 0; i < c_nextfree; i++) {
          fprintf (stderr, "C[%d].f=\"%s\", .lineno=%d, .col=%d, .type=%0d, .s=\"%s\" (at %p)\n",
                           i, c[i].f, c[i].lineno_tok, c[i].col_tok, c[i].t, escape(CString(i),'"'), CString(i));
        }
      }
      #endif
      return;
    }
    

    if ((ch == '+' || ch == '-' || ch == '&' || ch == '|' || ch == '=') && (peek == ch)) {
      startline = FALSE;
      /* ++, --, &&, || */
      // only a very small number of compound graphemes need to be converted to lexical tokens (BIPs) to avoid parsing ambiguities
      makespace (token, token_nextfree+4);
      token(token_nextfree++) = ch;
      token(token_nextfree++) = ch;
      token(token_nextfree++) = '\0';
      ch = xfgetc (); // eat 'peek' ch.
      stores (token,
              token_lineno, startcol, token_pos,
              (ch=='+'?B_pp:ch=='-'?B_mm:ch=='&'?B_andand:ch=='='?B_eqeq:B_oror), curfile);
    } else if (isalpha (ch) || (ch == '_')) {
      startline = FALSE;
      /* token or keyword */

      for (;;) {
        makespace (token, token_nextfree+1);
        if (isalpha (ch) || isdigit (ch) || (ch == '_')) {
          // digits allowed after 1st char.
          token(token_nextfree++) = ch;
        } else break;
        ch = xfgetc ();
      }
      xungetc (ch, sourcefile);
      token(token_nextfree++) = '\0';
      stores (token,
              token_lineno, startcol, token_pos,
              B_internal_identifier, curfile);

    } else if (((ch == '.') && isdigit(peek)) || (isdigit (ch))) {  // -->0B.110101100P12L
      startline = FALSE;
      /* Number */
      acceptable_range = &is_decimal;
      if (ch == '0') { // must be binary, octal, or hex.
        // Store as a string...

        makespace (token, token_nextfree+3);
        if (peek == 'b' || peek == 'B' || (peek == 'x') || (peek == 'X')) {
          token(token_nextfree++) = ch; // '0'
          acceptable_range = (peek == 'b' || peek == 'B' ? &is_binary : &is_hex);
          ch = xfgetc (); // get the B
          token(token_nextfree++) = ch; // b
          if (peek == '.') {
            goto READ_DECIMAL;
          }
        } else if (peek == '.') {
          token(token_nextfree++) = ch; // '0'
          // started with a 0 so octal unless it is 0.19 etc...
          acceptable_range = &is_decimal;
          goto READ_DECIMAL;
        } else {
          acceptable_range = &is_octal; // the 0 has been read.  If nothing acceptable follows, don't read any more chars
          token(token_nextfree++) = ch; // digit
        }
      } else if (ch == '.') {  //   .nnn or 0.nnn
        acceptable_range = &is_decimal;
        xungetc (ch, sourcefile); // peek is now '.'
        goto READ_DECIMAL;
      } else if (isdigit(ch)) {
        acceptable_range = &is_decimal;
        token(token_nextfree++) = ch; // digit
      }

      // rest of number comes through here for all formats
      while (acceptable_range(peek)) {  // 0B-->.110101100P12L
        ch = xfgetc (); // what was peek is now ch
        makespace (token, token_nextfree+2);
        token(token_nextfree++) = ch; // 0
      }
    READ_DECIMAL:
      if (peek == 'p' || peek == 'P') {
        goto BINARY_POWER;
      } else if (peek == '.' || peek == 'e' || peek == 'E') { 
        if (peek == '.') {
          ch = xfgetc ();
          token(token_nextfree++) = ch;
          while (acceptable_range(peek)) {
            ch = xfgetc (); // what was peek is now ch
            makespace (token, token_nextfree+2);
            token(token_nextfree++) = ch; // 0
          }
        }
        if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
          ch = xfgetc ();
          token(token_nextfree++) = ch;
        } else if (((acceptable_range == is_decimal) && (peek == 'e' || peek == 'E'))
            || ((acceptable_range != is_decimal) && (peek == 'p' || peek == 'P'))) {
        BINARY_POWER:
          ch = xfgetc ();
          token(token_nextfree++) = ch; // P
          acceptable_range = &is_decimal; // P+nnn nnn is decimal
          if ((peek == '+') || (peek == '-')) {
            ch = xfgetc ();
            token(token_nextfree++) = ch;
          }
          if (!acceptable_range(peek)) {
            token(token_nextfree) = '\0';
            fprintf(stderr, "Warning: bad floating point literal '%s'\n", token);
          }
          while (acceptable_range(peek)) { // SHOULD pick up P12L etc
            makespace (token, token_nextfree+2);
            ch = xfgetc (); // what was peek is now ch
            token(token_nextfree++) = ch; // 0
          }
          if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
            ch = xfgetc ();
            token(token_nextfree++) = ch;
          }
        }
      } else if ((peek == 'u') || (peek == 'U')) {
        ch = xfgetc ();
        token(token_nextfree++) = ch;
      } else if ((peek == 'l') || (peek == 'L')) {
        ch = xfgetc ();
        token(token_nextfree++) = ch;
        if (ch == peek) { // 'll' or 'LL' allowed but not 'lL' or 'Ll'
          ch = xfgetc ();
          token(token_nextfree++) = ch;
        }
      }
      makespace (token, token_nextfree+1);
      token(token_nextfree++) = '\0';
      if (strchr(token, '.') == NULL) {
        stores (token,
                token_lineno, startcol, token_pos,
                B_integer_constant, curfile); // changed from col to startcol
      } else {
        stores (token,
                token_lineno, startcol, token_pos,
                B_floating_constant, curfile); // changed from col to startcol
      }
    } else {
      switch (ch) {

      case '\'':               // Handle 'c' char const
      case '"':                // Handle "string"
        /* literals */
        startline = FALSE;
        {
          int string_nextfree = 0, string_arraysize = 0, quotech = ch;
          char *string = NULL;

          for (;;) {
            ch = xfgetc ();   // Newlines are allowed
            makespace (string, string_nextfree+3);
            if (ch == '\\') {
              ch = xfgetc ();
              if (ch == '\\') {
                string[string_nextfree++] = ch;
              } else if (ch == '\'') {
                string[string_nextfree++] = '\'';
              } else if (ch == '"') {
                string[string_nextfree++] = '"';
              } else if (ch == 'n') {
                string[string_nextfree++] = '\n';
              } else if (ch == 'r') {
                string[string_nextfree++] = '\r';
              } else if (ch == 't') {
                string[string_nextfree++] = '\t';
              } else if (ch == '0') {
                string[string_nextfree++] = '\0';
              } else {
                // Warn of unknown (to me) \x escape.  Probably an error.
                string[string_nextfree++] = '\\';
                string[string_nextfree++] = ch;
                fprintf(stderr, "Warning: un-handled escape '\\%c'\n", ch);
              }
            } else if (ch != quotech) {
              string[string_nextfree++] = ch;
            } else {
              string[string_nextfree] = '\0';
              break;
            }
          }

          if (quotech == '\'') {
            int stringlen = 0;
            char *s;
            stores (string,
                    token_lineno, token_col, token_pos,
                    B_sqstring, curfile);
            s = string;
            while (*s != '\0') {
              if (*s == '\\') s += 2; else s += 1;
              stringlen += 1;
            }
            if (stringlen == 1) {
              // good char const
            } else if (stringlen <= 4) {
              // Warn that 'xx' as a 32-bit int is a non-standard extension
              fprintf(stderr, "Warning: multi-byte characters constants such as \'%s\' are not well supported.\n", string); // don't escape - already escaped
            } else {
              // Warn that this is probably a string with the wrong type of quote.
              fprintf(stderr, "Warning: wrong kind of string quote used? - \'%s\'.\n", string); // don't escape - already escaped
            }
          } else stores (string,
                         token_lineno, token_col, token_pos,
                         B_dqstring, curfile);
          free (string);
        }
        break;

      case '/':
        /* COMMENTS (or just a divide symbol) */

        startline = FALSE;
        if (peek == '/') { // comment to end of line
          //  '//' comments are part of whitespace - skip over them, but whitespace_start must not be updated
          do {
            ch = xfgetc ();
          } while (ch != '\n');

        } else if (peek == '*') { /* Handle potential multi-line comment */
          ch = xfgetc ();     //  '/* ... */' comments are part of whitespace - skip over them, but whitespace_start must not be updated
          for (;;) {
            ch = xfgetc ();
            if ((ch == '*') && (peek == '/')) {
              ch = xfgetc (); // get the '/' too.
              break;
            }
          }
        } else {
          // a divide symbol, not a comment
          storec (ch,
                  token_lineno, token_col-1, token_pos,
                  B_char, curfile); 
        }
        break;

        // WHITESPACE
      case '\n':
      case '\r':
        startline = TRUE;
        break;

      case '\t':
      case ' ':
        // white space does not affect startline.
        break;

        // DIRECTIVES
      case '#':
        // should allow '#' as a directive if startline is true. (and not in a comment of course)
        // for now we'll just copy # directives straight through to the output
        if (startline) {
          int i = 0, lastch = -1;
          static char CPP_directive[256]; // small for now. Will make flex once tested.
          for (;;) {
            ch = xfgetc ();
            if (ch == EOF) exit(1);// fix
            ch &= 255;
            if ((ch == '\n') && (lastch != '\\')) { // e.g. #define X  some multiline text with '\' at the end of all lines except the last
              break;
            }
            CPP_directive[i++] = ch;
            lastch = ch;
            if (i == 255) {
              // skip any more text for now.
              for (;;) {
                ch = xfgetc ();
                if (ch == EOF) exit(1);// fix
                ch &= 255;
                if (ch == '\n' && lastch != '\\') {
                  break;
                }
                lastch = ch;
              }
              break;
            }
          }
          CPP_directive[i] = '\0';
          fprintf(stderr, "Warning: pre-processor directive skipped: #%s\n", CPP_directive);
          ch = '\n';
          startcol = nextch_col;
          startline = TRUE;
          break;
        }
        // Drop through

      default:
        storec (ch,
                token_lineno, token_col, token_pos,
                B_char, curfile);
        startline = FALSE;
      }
    }
    return;
  }
}
@


1.3
log
@excess removed
@
text
@d37 1
a37 1
  if (debug_line_reconstruction) fprintf(stderr, "Ch %d of %d: {%c}\n", nextch_pos, source_length, ch);
d112 2
a113 2
      c[c_nextfree].sIndex = str_to_pool("<EOF>");
      c[c_nextfree].t = B_EOF;
d125 2
a126 1
  
d135 1
d427 1
@


1.2
log
@both versions at the moment
@
text
@a0 57
/*

  When I first wrote this parser, I had it pull in the entire source up front, and
  do all the line reconstruction, outputting the canonical source into the c[] array 
  before parsing anything.

  For various reasons this turned out to be a bad idea, so what you are looking at here
  is a rewrite in which line reconstruction is performed on the fly and on demand,
  just before a character position in the c[] array is about to be accessed for the
  first time.

  It is not strictly a co-routine and does not fetch only the requested character -
  for the moment its task is to ensure that *at least* that character is in the array -
  but it is permissible for it to fetch a few extra characters as well for example if
  accessing the first character of a keyword, this version of the code will prefetch
  the rest of the keyword as well.

  This is not a design requirement - it was expedient because the internal state of
  the line reconstruction filter was not handled well.  A future update may undo
  this behaviour, but for now the other code in the parser must not rely on the
  line reconstruction doing minimal lookahead.

  The reason this is an issue is because we may want to modify the line reconstruction
  behaviour as a function of the current state of the parse - for instance Imp comments
  following a label should be handled the same as comments at the start of a line,
  but they are not - because the lexical scanner cannot tell that we are not already
  in an Imp statement, these particular comments end up being parsed as a statement,
  which causes really nasty interactions to happen.

  This mechanism may also allow us to handle C's preprocessor behaviour as we
  compile.  It's especially important when handling source to source transformations
  in C that we can match the input text to the expanded macro text so that we can
  track the syntax (and indentation etc) while outputting the pre-expanded source.

  Finally... having on-the-fly line reconstruction allows us to have parse-time
  semantic routines in the grammar which actually work (if interacting with
  line reconstruction)!

  An open question is whether to undo the line reconstruction when the parser
  backtracks, or whether the c[] array should be an unchanging cache.  This also
  affects interactions with parse-time semantic routines in the grammar.
  an unchanging c[] array is more efficient; a malleable c[] array is more
  powerful!

  Unrelated to the above but interacting with the same code - these changes
  have enabled us to add regular expression matching against the line-reconstructed
  source text, by virtue of a heavily modified version of regexp.c which takes
  advantage of the on-demand lookahead and gets its text to be examined from
  the c[] array rather than a string.  (Allowing things like matching Imp
  variables and constants which contain whitespace, without having to explicity
  code that whitespace in the regular expression syntax)

  The use of regular expressions to match terminals in the grammar has not
  yet been incorporated with the parser.

 */

a67 9
#ifdef NEVER
static void xungetc (int ch, FILE *sourcefile) {
  if (ch != lastch) {
    fprintf(stderr, "Warning: dubious xungetc?  ch=%d lastch=%d\n", ch, lastch);
  }
  nextch_pos = lastch_pos; nextch_lineno = lastch_lineno; nextch_col = lastch_col; peek = lastch_peek;
}
#endif

a168 1
//fprintf(stderr, "Going down the right path...\n");
a193 1
//fprintf(stderr, "It's a decimal... ch='%c' peek='%c'\n", ch, peek);
a195 1
//fprintf(stderr, "ungetc: ch='%c' peek='%c'\n", ch, peek);
a208 2
//token(token_nextfree) = '\0'; fprintf(stderr, "So read decimal... '%s'\n", token);
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a209 2
//fprintf(stderr, "Got a P...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a211 2
//fprintf(stderr, "Got that . again...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a212 2
//fprintf(stderr, "So... reading...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a219 2
//fprintf(stderr, "And read...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a221 2
//fprintf(stderr, "Got an F...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a225 2
//fprintf(stderr, "Got an E or P...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a226 2
//fprintf(stderr, "Binary power?\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
a248 1
//fprintf(stderr, "Peek U...\n");
a251 1
//fprintf(stderr, "Peek L...\n");
a257 2
      } else {
//fprintf(stderr, "No ELSE...\n");
a258 1
//token(token_nextfree) = '\0';fprintf(stderr, "STORE!... token='%s'\n", token);
a261 1
//fprintf(stderr, "STORE integer const!... '%s'\n", token);
a265 1
//fprintf(stderr, "STORE floating const!... '%s'\n", token);
a425 478

#ifdef NEVER
    if (ch == '/' && peek == '/') {  // C: // comments
      // For the moment we'll treat '!' comments as whitespace, though in Imp they're parsed statements in the grammar.
      // Whether this will hold up when we hit %comment comments as well is a matter for debate!

      do {
        ch = xfgetc (); if (ch == EOF) em_char_in_stmnt(); // comment that ends in <EOF>???
      } while (ch != '\n');
      // Some specific Imp implementations ended these comments at ';'; others parsed them and obeyed a trailing %c

      // There's also a nasty case of continuation that has to be handled here:
      // %const %integer %array reltag(0:6) =
      //             0,                         ;! no relocation
      //                            IF DATA,    ;! dataseg offset code word

      // The code for removing ';'s after '=' has to be applied here too :-(
      if ((c_nextfree > 1) && (c[c_nextfree-2].t == B_char) && strcmp(CString(c_nextfree-2), ",")==0) {
        if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_char) && strcmp(CString(c_nextfree-1), ";")==0) {
          c_nextfree -= 1;
          // Remove the ; but leave it in the whitespace.  This is sneaky:
          whitespace_lineno =  c[c_nextfree].lineno_sp;
          whitespace_col = c[c_nextfree].col_sp;
          whitespace_pos = c[c_nextfree].start_sp;
          at_startofline = FALSE;
          continue;
        }
      }
      // ch has now been replaced by a '\n'
      // drop through because we still want the newline to have significance
      // (not sure about continuations in this kind of comment)
    } else if (ch == '/' && peek == '*') {
      // similar to above but exit on '*/'
      if (ch == '{') { // C: Handle /* */ here!
        // Skip comments but they will be attached to the following token.
        // skip to '}' or end of line.
        for (;;) {
          ch = xfgetc (); if (ch == EOF) em_char_in_stmnt(); //  { ... <EOF>?
          if (ch == '}' || ch == '\n') break;
        }
        if (ch == '}') continue;
        // ch is now '\n'
      }

    } else if (ch == '\'' || ch == '"') { // start of string. Stored as typed string data without quotes or escaping.
      // It's C, so we need to handle escaped characters.  However unlike Imp, all strings end before NL
      int quotech = ch;
      int backtrack_token = token_nextfree;
      int backtrack_pos = token_pos;
      int backtrack_lineno = token_lineno;
      int backtrack_col = token_col;
      int stringlength = 0;
      at_startofline = FALSE;
      // probably a char constant but can be other things, eg M'HECK' is a 32-bit int.
      for (;;) {
        ch = xfgetc ();
        if (ch == EOF) {
          // backtrack and warn if EOF found in string - almost certainly an unterminated string
          token_nextfree = backtrack_token;
          token_pos = backtrack_pos;
          token_lineno = backtrack_lineno;
          token_col = backtrack_col;
          ch = quotech;
          storec (ch, token_lineno, token_col, token_pos, B_char, curfile); // leave it for the parser to cause a syntax error.
          return;
        }
        if (ch == quotech) { // quotech in string is represented by doubling.  No C-like escape characters.
          if (peek != quotech) break;
          ch = xfgetc (); if (ch == EOF) em_char_in_stmnt();  // \<EOF> ?!!!!
        }
        makespace(token,token_nextfree+1);
        token(token_nextfree++) = ch;
        stringlength += 1;
        if (stringlength == 256) {
          // backtrack and warn if string is longer than 255 characters - almost certainly an unterminated string
          token_nextfree = backtrack_token;
          token_pos = backtrack_pos;
          token_lineno = backtrack_lineno;
          token_col = backtrack_col;
          ch = quotech;
          storec (ch, token_lineno, token_col, token_pos, B_char, curfile); // leave it for the parser to cause a syntax error.
          return;
        }
      }
      // end of string
      token(token_nextfree) = '\0';
      stores (token,
              token_lineno, token_col, token_pos,
              quotech == '"' ? B_dqstring : B_sqstring,
              curfile);
      return;
    }

    // Skip white space
    if ((ch != '\n') && isspace(ch)) continue;

    if (ch == '\n') {
      at_startofline = TRUE;
      storec (ch,
            token_lineno, token_col, token_pos,
            B_NL, curfile);
      return;
    }

    // store the character under B_char

    if (ch == 127 || ch == ('L'&31) || !isprint(ch)) {
      fprintf(stderr, "? Warning: stray character %d in program\n", ch);
      continue; // treat it as white space.
    }

    at_startofline = FALSE;
    storec (ch,
            token_lineno, token_col/*-1*/, token_pos,
            B_char, curfile);

    return; // most single character fetches return here.
  }
#endif
}

#ifdef NEVER
void line_reconstruction (FILE *sourcefile, char *filename) {
  int (*acceptable_range)(int ch);
  int token_nextfree, startcol;
  static int token_arraysize = 0;
  static char *token = NULL;
#define token(x) FLEX(token, x, __LINE__)

  /* Pre-process input ready for parsing.  Tokens are stored in array C[] */
  // go over this again and make sure 'col' is updated correctly at all places where chars are read.
  for (;;) {
    startcol = nextch_col;
    token_nextfree = 0;
    token_lineno = nextch_lineno;
    token_col = nextch_col;
    token_pos = nextch_pos;
    
    if ((ch = xfgetc (sourcefile)) == EOF)
      break;

    ch &= 255;                  // int, positive.

    if ((ch == '+' || ch == '-' || ch == '&' || ch == '|' || ch == '=') && (peek == ch)) {
      startline = FALSE;
      /* ++, --, &&, || */
      // only a very small number of compound graphemes need to be converted to lexical tokens (BIPs) to avoid parsing ambiguities
      makespace (token, token_nextfree+4);
      token(token_nextfree++) = ch;
      token(token_nextfree++) = ch;
      token(token_nextfree++) = '\0';
      ch = xfgetc (sourcefile); // eat 'peek' ch.
      stores (token,
              token_lineno, startcol, token_pos,
              (ch=='+'?B_pp:ch=='-'?B_mm:ch=='&'?B_andand:ch=='='?B_eqeq:B_oror), curfile);
    } else if (isalpha (ch) || (ch == '_')) {
      startline = FALSE;
      /* token or keyword */

      for (;;) {
        makespace (token, token_nextfree+1);
        if (isalpha (ch) || isdigit (ch) || (ch == '_')) {
          // digits allowed after 1st char.
          token(token_nextfree++) = ch;
        } else break;
        ch = xfgetc (sourcefile);
      }
      xungetc (ch, sourcefile);
      token(token_nextfree++) = '\0';
      stores (token,
              token_lineno, startcol, token_pos,
              B_internal_identifier, curfile);

    } else if (((ch == '.') && isdigit(peek)) || (isdigit (ch))) {  // -->0B.110101100P12L
//fprintf(stderr, "Going down the right path...\n");
      startline = FALSE;
      /* Number */
      acceptable_range = &is_decimal;
      if (ch == '0') { // must be binary, octal, or hex.
        // Store as a string...

        makespace (token, token_nextfree+3);
        if (peek == 'b' || peek == 'B' || (peek == 'x') || (peek == 'X')) {
          token(token_nextfree++) = ch; // '0'
          acceptable_range = (peek == 'b' || peek == 'B' ? &is_binary : &is_hex);
          ch = xfgetc (sourcefile); // get the B
          token(token_nextfree++) = ch; // b
          if (peek == '.') {
            goto READ_DECIMAL;
          }
        } else if (peek == '.') {
          token(token_nextfree++) = ch; // '0'
          // started with a 0 so octal unless it is 0.19 etc...
          acceptable_range = &is_decimal;
          goto READ_DECIMAL;
        } else {
          acceptable_range = &is_octal; // the 0 has been read.  If nothing acceptable follows, don't read any more chars
          token(token_nextfree++) = ch; // digit
        }
      } else if (ch == '.') {  //   .nnn or 0.nnn
//fprintf(stderr, "It's a decimal... ch='%c' peek='%c'\n", ch, peek);
        acceptable_range = &is_decimal;
        xungetc (ch, sourcefile); // peek is now '.'
//fprintf(stderr, "ungetc: ch='%c' peek='%c'\n", ch, peek);
        goto READ_DECIMAL;
      } else if (isdigit(ch)) {
        acceptable_range = &is_decimal;
        token(token_nextfree++) = ch; // digit
      }

      // rest of number comes through here for all formats
      while (acceptable_range(peek)) {  // 0B-->.110101100P12L
        ch = xfgetc (sourcefile); // what was peek is now ch
        makespace (token, token_nextfree+2);
        token(token_nextfree++) = ch; // 0
      }
    READ_DECIMAL:
//token(token_nextfree) = '\0'; fprintf(stderr, "So read decimal... '%s'\n", token);
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
      if (peek == 'p' || peek == 'P') {
//fprintf(stderr, "Got a P...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
        goto BINARY_POWER;
      } else if (peek == '.' || peek == 'e' || peek == 'E') { 
//fprintf(stderr, "Got that . again...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
        if (peek == '.') {
//fprintf(stderr, "So... reading...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
          ch = xfgetc (sourcefile);
          token(token_nextfree++) = ch;
          while (acceptable_range(peek)) {
            ch = xfgetc (sourcefile); // what was peek is now ch
            makespace (token, token_nextfree+2);
            token(token_nextfree++) = ch; // 0
          }
//fprintf(stderr, "And read...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
        }
        if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
//fprintf(stderr, "Got an F...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
          ch = xfgetc (sourcefile);
          token(token_nextfree++) = ch;
        } else if (((acceptable_range == is_decimal) && (peek == 'e' || peek == 'E'))
            || ((acceptable_range != is_decimal) && (peek == 'p' || peek == 'P'))) {
//fprintf(stderr, "Got an E or P...\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
        BINARY_POWER:
//fprintf(stderr, "Binary power?\n");
//fprintf(stderr, "ch='%c' peek='%c'\n", ch, peek);
          ch = xfgetc (sourcefile);
          token(token_nextfree++) = ch; // P
          acceptable_range = &is_decimal; // P+nnn nnn is decimal
          if ((peek == '+') || (peek == '-')) {
            ch = xfgetc (sourcefile);
            token(token_nextfree++) = ch;
          }
          if (!acceptable_range(peek)) {
            token(token_nextfree) = '\0';
            fprintf(stderr, "Warning: bad floating point literal '%s'\n", token);
          }
          while (acceptable_range(peek)) { // SHOULD pick up P12L etc
            makespace (token, token_nextfree+2);
            ch = xfgetc (sourcefile); // what was peek is now ch
            token(token_nextfree++) = ch; // 0
          }
          if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
            ch = xfgetc (sourcefile);
            token(token_nextfree++) = ch;
          }
        }
      } else if ((peek == 'u') || (peek == 'U')) {
//fprintf(stderr, "Peek U...\n");
        ch = xfgetc (sourcefile);
        token(token_nextfree++) = ch;
      } else if ((peek == 'l') || (peek == 'L')) {
//fprintf(stderr, "Peek L...\n");
        ch = xfgetc (sourcefile);
        token(token_nextfree++) = ch;
        if (ch == peek) { // 'll' or 'LL' allowed but not 'lL' or 'Ll'
          ch = xfgetc (sourcefile);
          token(token_nextfree++) = ch;
        }
      } else {
//fprintf(stderr, "No ELSE...\n");
      }
//token(token_nextfree) = '\0';fprintf(stderr, "STORE!... token='%s'\n", token);
      makespace (token, token_nextfree+1);
      token(token_nextfree++) = '\0';
      if (strchr(token, '.') == NULL) {
//fprintf(stderr, "STORE integer const!... '%s'\n", token);
        stores (token,
                token_lineno, startcol, token_pos,
                B_integer_constant, curfile); // changed from col to startcol
      } else {
//fprintf(stderr, "STORE floating const!... '%s'\n", token);
        stores (token,
                token_lineno, startcol, token_pos,
                B_floating_constant, curfile); // changed from col to startcol
      }
    } else {
      switch (ch) {

      case '\'':               // Handle 'c' char const
      case '"':                // Handle "string"
        /* literals */
        startline = FALSE;
        {
          int string_nextfree = 0, string_arraysize = 0, quotech = ch;
          char *string = NULL;

          for (;;) {
            ch = xfgetc (sourcefile);   // Newlines are allowed
            makespace (string, string_nextfree+3);
            if (ch == '\\') {
              ch = xfgetc (sourcefile);
              if (ch == '\\') {
                string[string_nextfree++] = ch;
              } else if (ch == '\'') {
                string[string_nextfree++] = '\'';
              } else if (ch == '"') {
                string[string_nextfree++] = '"';
              } else if (ch == 'n') {
                string[string_nextfree++] = '\n';
              } else if (ch == 'r') {
                string[string_nextfree++] = '\r';
              } else if (ch == 't') {
                string[string_nextfree++] = '\t';
              } else if (ch == '0') {
                string[string_nextfree++] = '\0';
              } else {
                // Warn of unknown (to me) \x escape.  Probably an error.
                string[string_nextfree++] = '\\';
                string[string_nextfree++] = ch;
                fprintf(stderr, "Warning: un-handled escape '\\%c'\n", ch);
              }
            } else if (ch != quotech) {
              string[string_nextfree++] = ch;
            } else {
              string[string_nextfree] = '\0';
              break;
            }
          }

          if (quotech == '\'') {
            int stringlen = 0;
            char *s;
            stores (string,
                    token_lineno, token_col, token_pos,
                    B_sqstring, curfile);
            s = string;
            while (*s != '\0') {
              if (*s == '\\') s += 2; else s += 1;
              stringlen += 1;
            }
            if (stringlen == 1) {
              // good char const
            } else if (stringlen <= 4) {
              // Warn that 'xx' as a 32-bit int is a non-standard extension
              fprintf(stderr, "Warning: multi-byte characters constants such as \'%s\' are not well supported.\n", string); // don't escape - already escaped
            } else {
              // Warn that this is probably a string with the wrong type of quote.
              fprintf(stderr, "Warning: wrong kind of string quote used? - \'%s\'.\n", string); // don't escape - already escaped
            }
          } else stores (string,
                         token_lineno, token_col, token_pos,
                         B_dqstring, curfile);
          free (string);
        }
        break;

      case '/':
        /* COMMENTS (or just a divide symbol) */

        startline = FALSE;
        if (peek == '/') { // comment to end of line
          //  '//' comments are part of whitespace - skip over them, but whitespace_start must not be updated
          do {
            ch = xfgetc (sourcefile);
          } while (ch != '\n');

        } else if (peek == '*') { /* Handle potential multi-line comment */
          ch = xfgetc (sourcefile);     //  '/* ... */' comments are part of whitespace - skip over them, but whitespace_start must not be updated
          for (;;) {
            ch = xfgetc (sourcefile);
            if ((ch == '*') && (peek == '/')) {
              ch = xfgetc (sourcefile); // get the '/' too.
              break;
            }
          }
        } else {
          // a divide symbol, not a comment
          storec (ch,
                  token_lineno, token_col-1, token_pos,
                  B_char, curfile); 
        }
        break;

        // WHITESPACE
      case '\n':
      case '\r':
        startline = TRUE;
        break;

      case '\t':
      case ' ':
        // white space does not affect startline.
        break;

        // DIRECTIVES
      case '#':
        // should allow '#' as a directive if startline is true. (and not in a comment of course)
        // for now we'll just copy # directives straight through to the output
        if (startline) {
          int i = 0, lastch = -1;
          static char CPP_directive[256]; // small for now. Will make flex once tested.
          for (;;) {
            ch = xfgetc (sourcefile);
            if (ch == EOF) exit(1);// fix
            ch &= 255;
            if ((ch == '\n') && (lastch != '\\')) { // e.g. #define X  some multiline text with '\' at the end of all lines except the last
              break;
            }
            CPP_directive[i++] = ch;
            lastch = ch;
            if (i == 255) {
              // skip any more text for now.
              for (;;) {
                ch = xfgetc (sourcefile);
                if (ch == EOF) exit(1);// fix
                ch &= 255;
                if (ch == '\n' && lastch != '\\') {
                  break;
                }
                lastch = ch;
              }
              break;
            }
          }
          CPP_directive[i] = '\0';
          fprintf(stderr, "Warning: pre-processor directive skipped: #%s\n", CPP_directive);
          ch = '\n';
          startcol = nextch_col;
          startline = TRUE;
          break;
        }
        // Drop through

      default:
        storec (ch,
                token_lineno, token_col, token_pos,
                B_char, curfile);
        startline = FALSE;
      }
    }
  }
  // set up a dummy at the end because we sometimes look ahead by 1
  // in the parsing code and don't want to hit uninitialised data.

  makespace (c, c_nextfree+1);

  c(c_nextfree).sIndex = str_to_pool("<EOF>");
  c(c_nextfree).t = B_EOF;

  c(c_nextfree).lineno_sp = whitespace_lineno;
  c(c_nextfree).col_sp = whitespace_col;
  c(c_nextfree).start_sp = whitespace_pos;

  c(c_nextfree).lineno_tok = nextch_lineno;
  c(c_nextfree).col_tok = nextch_col;
  c(c_nextfree).start_tok = nextch_pos;

  c(c_nextfree).end_tok = nextch_pos;
  
  // NOTE: we are not yet handling include files...
  
  // token will still be on the heap at exit.
a426 1
#endif
@


1.1
log
@Initial revision
@
text
@d1 1
d3 86
a88 2
static int xfgetc (FILE * f) {
  int ch;
d96 1
a99 1
// Imp line reconstruction doesn't need ungetc, but C does
d125 516
a640 1
void line_reconstruction (FILE *sourcefile, char *filename) {  // for C
d654 6
a659 5
    token_pos = nextch_pos; // actual text of token may start here but whitespace goes back to end of previous token.
 
    if ((ch = xfgetc (sourcefile)) == EOF) {
      break; // The code after this loop saves an EOF record in the c[] array
    }
d692 1
d718 1
d721 1
d735 2
d738 2
d742 2
d745 2
d754 2
d758 2
d764 2
d767 2
d791 1
d795 1
d803 1
d805 1
d809 1
d814 1
d868 1
a868 1
                    B_character_constant, curfile);
d885 1
a885 1
                         B_string, curfile);
d997 1
@
