line_reconstruction-c.c

    // For reasons to be explained later, the code in the on-demand source-fetching
    // must not use the flex mechanism if there is any possibility of 'c' itself
    // being reallocated while evaluating the index of a c(x) reference.  Space for
    // tokens in the c[] array must have been pre-claimed by makespace before a
    // non-existent c(x) entry is accessed.  Hence c() was not defined by the
    // usual FLEX macro and the use of String() within the line reconstruction
    // and any code that it calls (eg stores()) must use c[] rather than c(), and
    // the alternative CString() ...

    #define CString(i) &stringpool(c[i].sIndex)

    int is_binary(int ch) {
      return ((ch|1) == '1');
    }

    int is_octal(int ch) {
      return ('0' <= ch) && (ch <= '7'); // or...  (ch|7) =='7'
    }

    int is_decimal(int ch) {
      return ('0' <= ch) && (ch <= '9');
    }

    int is_hex(int ch) {
      return (('0' <= ch) && (ch <= '9')) || (('a' <= ch) && (ch <= 'f')) || (('A' <= ch) && (ch <= 'F'));
    }

    static int lastch = EOF, lastch_pos = 0, lastch_lineno = 0, lastch_col = 0, lastch_peek = 0;
    static int xfgetc (void) {
      static int ch;
      lastch = ch; lastch_pos = nextch_pos; lastch_lineno = nextch_lineno; lastch_col = nextch_col; lastch_peek = peek;
      if (nextch_pos >= source_length) return EOF;
      ch = source_address(nextch_pos);
      if (ch == '\n') {
        nextch_lineno += 1; nextch_col = 0;
      } else nextch_col += 1;
      //if (debug_line_reconstruction) fprintf(stderr, "Ch %d of %d: {%c}\n", nextch_pos, source_length, ch);
      peek = source_address(++nextch_pos);

      return ch&255;
    }

    static void xungetc (int c, FILE * f) { // for a xfgetc char, not for peek.
      // rather than subtracting 1 from postions it may be better to save all
      // the positions in xfgetc and unwind them, thus avoiding the issue of
      // whether we're ungetting a token or whitespace.  Or supply two xungetc's...
      if (nextch_pos > 0) {
        nextch_pos -= 1;
        if (nextch_col == 0) {
          int p;
          nextch_lineno -= 1;
          p = source_line(nextch_lineno);
          while (source_address(p) != '\n') {
            if (p == source_length) break;
            nextch_col += 1; p += 1;
          }
        } else nextch_col -= 1;
      } else {
        // invalid ungetc. nothing to unget at start of file.
      }
      ch = source_address(nextch_pos-1);//+1
      //  if (c == '\n') {
      //    nextch_lineno += 1; nextch_col = 0;
      //  } else nextch_col += 1;
      if (nextch_pos == source_length) peek = EOF; else peek = source_address(nextch_pos);//+1
    }

    void em_char_in_stmnt(int line) {
      // (a little nod to the past...)
      fprintf(stderr, "EM CHAR IN STMNT - DISASTER (detected at parser line %d)\n", line); // stop-gap measure - needs a longjump to end of compile code to clean up
      exit(EXIT_FAILURE);
    }
    #define em_char_in_stmnt() em_char_in_stmnt(__LINE__)

    void line_reconstruction (void) {
      int (*acceptable_range)(int ch);
      // adds one more item to c[] array.
      // It is up to the caller to keep calling until sufficient data is loaded.
      static int token_nextfree, startcol;
      static int token_arraysize = 0;
      static char *token = NULL;
    #define token(x) FLEX(token, x, __LINE__)
      static int init=0;

      // Recognise single and double-quoted strings
      // while removing comments and handling line continuation.
      // The c[] array gets the canonicalised version of the text, BUT retains
      // pointers to the original text including whitespace, which comes in
      // handy for any source-to-source transformations such as indent (pretty printing)
      // or transpiling to another language (eg C to Javascript)
      if (init == 0) {
        at_startofline = TRUE;
        makespace(token,1024); // quick hack only for test.imp fix as soon as lexer is written.
        init = 1;
      }

      /* Pre-process input ready for parsing.  Tokens are stored in array c[] */
      // NOTE: we are not yet handling include files...

      for (;;) {
        startcol = nextch_col;
        token_nextfree = 0;
        token_lineno = nextch_lineno;
        token_col = nextch_col;
        token_pos = nextch_pos; // actual text of token may start here but whitespace goes back to end of previous token.

        if ((ch = xfgetc ()) == EOF) {
          // set up a dummy at the end because we sometimes look ahead by 1
          // in the parsing code and don't want to hit uninitialised data.
          c[c_nextfree].f = curfile;

          c[c_nextfree].sIndex = str_to_pool("<EOF>"); // however we want to be able to print B_EOF and have any blank lines
          c[c_nextfree].t = B_EOF;                     // or comments before the end-of-file be output... so maybe "" would be better?

          c[c_nextfree].lineno_sp = whitespace_lineno;
          c[c_nextfree].col_sp = whitespace_col;
          c[c_nextfree].start_sp = whitespace_pos;

          c[c_nextfree].lineno_tok = nextch_lineno;
          c[c_nextfree].col_tok = nextch_col;
          c[c_nextfree].start_tok = nextch_pos;

          c[c_nextfree].end_tok = nextch_pos;
          c_nextfree++;

          #ifdef NEVER
          if (debug_tokens || debug_line_reconstruction) { // print tokens once parser hits end of file on input stream
            int i;
            fprintf (stderr, "\nLexical token stream:\n\n");
            for (i = 0; i < c_nextfree; i++) {
              fprintf (stderr, "C[%d].f=\"%s\", .lineno=%d, .col=%d, .type=%0d, .s=\"%s\" (at %p)\n",
                               i, c[i].f, c[i].lineno_tok, c[i].col_tok, c[i].t, escape(CString(i),'"'), CString(i));
            }
          }
          #endif
          return;
        }

        if ((ch == '+' || ch == '-' || ch == '&' || ch == '|' || ch == '=') && (peek == ch)) {
          startline = FALSE;
          /* ++, --, &&, || */
          // only a very small number of compound graphemes need to be converted to lexical tokens (BIPs) to avoid parsing ambiguities
          makespace (token, token_nextfree+4);
          token(token_nextfree++) = ch;
          token(token_nextfree++) = ch;
          token(token_nextfree++) = '\0';
          ch = xfgetc (); // eat 'peek' ch.
          stores (token,
                  token_lineno, startcol, token_pos,
                  (ch=='+'?B_pp:ch=='-'?B_mm:ch=='&'?B_andand:ch=='='?B_eqeq:B_oror), curfile);
        } else if (isalpha (ch) || (ch == '_')) {
          startline = FALSE;
          /* token or keyword */

          for (;;) {
            makespace (token, token_nextfree+1);
            if (isalpha (ch) || isdigit (ch) || (ch == '_')) {
              // digits allowed after 1st char.
              token(token_nextfree++) = ch;
            } else break;
            ch = xfgetc ();
          }
          xungetc (ch, sourcefile);
          token(token_nextfree++) = '\0';
          stores (token,
                  token_lineno, startcol, token_pos,
                  B_internal_identifier, curfile);

        } else if (((ch == '.') && isdigit(peek)) || (isdigit (ch))) {  // -->0B.110101100P12L
          startline = FALSE;
          /* Number */
          acceptable_range = &is_decimal;
          if (ch == '0') { // must be binary, octal, or hex.
            // Store as a string...

            makespace (token, token_nextfree+3);
            if (peek == 'b' || peek == 'B' || (peek == 'x') || (peek == 'X')) {
              token(token_nextfree++) = ch; // '0'
              acceptable_range = (peek == 'b' || peek == 'B' ? &is_binary : &is_hex);
              ch = xfgetc (); // get the B
              token(token_nextfree++) = ch; // b
              if (peek == '.') {
                goto READ_DECIMAL;
              }
            } else if (peek == '.') {
              token(token_nextfree++) = ch; // '0'
              // started with a 0 so octal unless it is 0.19 etc...
              acceptable_range = &is_decimal;
              goto READ_DECIMAL;
            } else {
              acceptable_range = &is_octal; // the 0 has been read.  If nothing acceptable follows, don't read any more chars
              token(token_nextfree++) = ch; // digit
            }
          } else if (ch == '.') {  //   .nnn or 0.nnn
            acceptable_range = &is_decimal;
            xungetc (ch, sourcefile); // peek is now '.'
            goto READ_DECIMAL;
          } else if (isdigit(ch)) {
            acceptable_range = &is_decimal;
            token(token_nextfree++) = ch; // digit
          }

          // rest of number comes through here for all formats
          while (acceptable_range(peek)) {  // 0B-->.110101100P12L
            ch = xfgetc (); // what was peek is now ch
            makespace (token, token_nextfree+2);
            token(token_nextfree++) = ch; // 0
          }
        READ_DECIMAL:
          if (peek == 'p' || peek == 'P') {
            goto BINARY_POWER;
          } else if (peek == '.' || peek == 'e' || peek == 'E') {
            if (peek == '.') {
              ch = xfgetc ();
              token(token_nextfree++) = ch;
              while (acceptable_range(peek)) {
                ch = xfgetc (); // what was peek is now ch
                makespace (token, token_nextfree+2);
                token(token_nextfree++) = ch; // 0
              }
            }
            if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
              ch = xfgetc ();
              token(token_nextfree++) = ch;
            } else if (((acceptable_range == is_decimal) && (peek == 'e' || peek == 'E'))
                || ((acceptable_range != is_decimal) && (peek == 'p' || peek == 'P'))) {
            BINARY_POWER:
              ch = xfgetc ();
              token(token_nextfree++) = ch; // P
              acceptable_range = &is_decimal; // P+nnn nnn is decimal
              if ((peek == '+') || (peek == '-')) {
                ch = xfgetc ();
                token(token_nextfree++) = ch;
              }
              if (!acceptable_range(peek)) {
                token(token_nextfree) = '\0';
                fprintf(stderr, "Warning: bad floating point literal '%s'\n", token);
              }
              while (acceptable_range(peek)) { // SHOULD pick up P12L etc
                makespace (token, token_nextfree+2);
                ch = xfgetc (); // what was peek is now ch
                token(token_nextfree++) = ch; // 0
              }
              if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
                ch = xfgetc ();
                token(token_nextfree++) = ch;
              }
            }
          } else if ((peek == 'u') || (peek == 'U')) {
            ch = xfgetc ();
            token(token_nextfree++) = ch;
          } else if ((peek == 'l') || (peek == 'L')) {
            ch = xfgetc ();
            token(token_nextfree++) = ch;
            if (ch == peek) { // 'll' or 'LL' allowed but not 'lL' or 'Ll'
              ch = xfgetc ();
              token(token_nextfree++) = ch;
            }
          }
          makespace (token, token_nextfree+1);
          token(token_nextfree++) = '\0';
          if (strchr(token, '.') == NULL) {
            stores (token,
                    token_lineno, startcol, token_pos,
                    B_integer_constant, curfile); // changed from col to startcol
          } else {
            stores (token,
                    token_lineno, startcol, token_pos,
                    B_floating_constant, curfile); // changed from col to startcol
          }
        } else {
          switch (ch) {

          case '\'':               // Handle 'c' char const
          case '"':                // Handle "string"
            /* literals */
            startline = FALSE;
            {
              int string_nextfree = 0, string_arraysize = 0, quotech = ch;
              char *string = NULL;

              for (;;) {
                ch = xfgetc ();   // Newlines are allowed
                makespace (string, string_nextfree+3);
                if (ch == '\\') {
                  ch = xfgetc ();
                  if (ch == '\\') {
                    string[string_nextfree++] = ch;
                  } else if (ch == '\'') {
                    string[string_nextfree++] = '\'';
                  } else if (ch == '"') {
                    string[string_nextfree++] = '"';
                  } else if (ch == 'n') {
                    string[string_nextfree++] = '\n';
                  } else if (ch == 'r') {
                    string[string_nextfree++] = '\r';
                  } else if (ch == 't') {
                    string[string_nextfree++] = '\t';
                  } else if (ch == '0') {
                    string[string_nextfree++] = '\0';
                  } else {
                    // Warn of unknown (to me) \x escape.  Probably an error.
                    string[string_nextfree++] = '\\';
                    string[string_nextfree++] = ch;
                    fprintf(stderr, "Warning: un-handled escape '\\%c'\n", ch);
                  }
                } else if (ch != quotech) {
                  string[string_nextfree++] = ch;
                } else {
                  string[string_nextfree] = '\0';
                  break;
                }
              }

              if (quotech == '\'') {
                int stringlen = 0;
                char *s;
                stores (string,
                        token_lineno, token_col, token_pos,
                        B_sqstring, curfile);
                s = string;
                while (*s != '\0') {
                  if (*s == '\\') s += 2; else s += 1;
                  stringlen += 1;
                }
                if (stringlen == 1) {
                  // good char const
                } else if (stringlen <= 4) {
                  // Warn that 'xx' as a 32-bit int is a non-standard extension
                  fprintf(stderr, "Warning: multi-byte characters constants such as \'%s\' are not well supported.\n", string); // don't escape - already escaped
                } else {
                  // Warn that this is probably a string with the wrong type of quote.
                  fprintf(stderr, "Warning: wrong kind of string quote used? - \'%s\'.\n", string); // don't escape - already escaped
                }
              } else stores (string,
                             token_lineno, token_col, token_pos,
                             B_dqstring, curfile);
              free (string);
            }
            break;

          case '/':
            /* COMMENTS (or just a divide symbol) */

            startline = FALSE;
            if (peek == '/') { // comment to end of line
              //  '//' comments are part of whitespace - skip over them, but whitespace_start must not be updated
              do {
                ch = xfgetc ();
              } while (ch != '\n');

            } else if (peek == '*') { /* Handle potential multi-line comment */
              ch = xfgetc ();     //  '/* ... */' comments are part of whitespace - skip over them, but whitespace_start must not be updated
              for (;;) {
                ch = xfgetc ();
                if ((ch == '*') && (peek == '/')) {
                  ch = xfgetc (); // get the '/' too.
                  break;
                }
              }
            } else {
              // a divide symbol, not a comment
              storec (ch,
                      token_lineno, token_col-1, token_pos,
                      B_char, curfile);
            }
            break;

            // WHITESPACE
          case '\n':
          case '\r':
            startline = TRUE;
            break;

          case '\t':
          case ' ':
            // white space does not affect startline.
            break;

            // DIRECTIVES
          case '#':
            // should allow '#' as a directive if startline is true. (and not in a comment of course)
            // for now we'll just copy # directives straight through to the output
            if (startline) {
              int i = 0, lastch = -1;
              static char CPP_directive[256]; // small for now. Will make flex once tested.
              for (;;) {
                ch = xfgetc ();
                if (ch == EOF) exit(1);// fix
                ch &= 255;
                if ((ch == '\n') && (lastch != '\\')) { // e.g. #define X  some multiline text with '\' at the end of all lines except the last
                  break;
                }
                CPP_directive[i++] = ch;
                lastch = ch;
                if (i == 255) {
                  // skip any more text for now.
                  for (;;) {
                    ch = xfgetc ();
                    if (ch == EOF) exit(1);// fix
                    ch &= 255;
                    if (ch == '\n' && lastch != '\\') {
                      break;
                    }
                    lastch = ch;
                  }
                  break;
                }
              }
              CPP_directive[i] = '\0';
              fprintf(stderr, "Warning: pre-processor directive skipped: #%s\n", CPP_directive);
              ch = '\n';
              startcol = nextch_col;
              startline = TRUE;
              break;
            }
            // Drop through

          default:
            storec (ch,
                    token_lineno, token_col, token_pos,
                    B_char, curfile);
            startline = FALSE;
          }
        }
        return;
      }
    }