line_reconstruction-imp80.c

    //68000: fix...
    //%conststring(15)%array codename(1:n)=-
    //   "RomREBOOT",        "RomBOOT",        "RomLOAD",        "RomPHEX",

    /*

      When I first wrote this parser, I had it pull in the entire source up front, and
      do all the line reconstruction, outputting the canonical source into the c[] array
      before parsing anything.

      For various reasons this turned out to be a bad idea, so what you are looking at here
      is a rewrite in which line reconstruction is performed on the fly and on demand,
      just before a character position in the c[] array is about to be accessed for the
      first time.

      It is not strictly a co-routine and does not fetch only the requested character -
      for the moment its task is to ensure that *at least* that character is in the array -
      but it is permissible for it to fetch a few extra characters as well for example if
      accessing the first character of a keyword, this version of the code will prefetch
      the rest of the keyword as well.

      This is not a design requirement - it was expedient because the internal state of
      the line reconstruction filter was not handled well.  A future update may undo
      this behaviour, but for now the other code in the parser must not rely on the
      line reconstruction doing minimal lookahead.

      The reason this is an issue is because we may want to modify the line reconstruction
      behaviour as a function of the current state of the parse - for instance Imp comments
      following a label should be handled the same as comments at the start of a line,
      but they are not - because the lexical scanner cannot tell that we are not already
      in an Imp statement, these particular comments end up being parsed as a statement,
      which causes really nasty interactions to happen.

      This mechanism may also allow us to handle C's preprocessor behaviour as we
      compile.  It's especially important when handling source to source transformations
      in C that we can match the input text to the expanded macro text so that we can
      track the syntax (and indentation etc) while outputting the pre-expanded source.

      Finally... having on-the-fly line reconstruction allows us to have parse-time
      semantic routines in the grammar which actually work (if interacting with
      line reconstruction)!

      An open question is whether to undo the line reconstruction when the parser
      backtracks, or whether the c[] array should be an unchanging cache.  This also
      affects interactions with parse-time semantic routines in the grammar.
      an unchanging c[] array is more efficient; a malleable c[] array is more
      powerful!

      Unrelated to the above but interacting with the same code - these changes
      have enabled us to add regular expression matching against the line-reconstructed
      source text, by virtue of a heavily modified version of regexp.c which takes
      advantage of the on-demand lookahead and gets its text to be examined from
      the c[] array rather than a string.  (Allowing things like matching Imp
      variables and constants which contain whitespace, without having to explicity
      code that whitespace in the regular expression syntax)

      The use of regular expressions to match terminals in the grammar has not
      yet been incorporated with the parser.

     */

    // For reasons to be explained later, the code in the on-demand source-fetching
    // must not use the flex mechanism if there is any possibility of 'c' itself
    // being reallocated while evaluating the index of a c(x) reference.  Space for
    // tokens in the c[] array must have been pre-claimed by makespace before a
    // non-existent c(x) entry is accessed.  Hence c() was not defined by the
    // usual FLEX macro and the use of String() within the line reconstruction
    // and any code that it calls (eg stores()) must use c[] rather than c(), and
    // the alternative CString() ...

    #define CString(i) &stringpool(c[i].sIndex)

    static int xfgetc (void) {
      int ch;
      if (nextch_pos >= source_length) return EOF;
      ch = source_address(nextch_pos);
      if (ch == '\n') {
        nextch_lineno += 1; nextch_col = 0;
      } else nextch_col += 1;
      //if (debug_line_reconstruction) fprintf(stderr, "Ch %d of %d: {%c}\n", nextch_pos, source_length, ch);
      peek = source_address(++nextch_pos);
      return ch&255;
    }

    int is_binary(int ch) {
      return ((ch|1) == '1');
    }

    int is_octal(int ch) {
      return ('0' <= ch) && (ch <= '7'); // or...  (ch|7) =='7'
    }

    int is_decimal(int ch) {
      return ('0' <= ch) && (ch <= '9');
    }

    int is_hex(int ch) {
      return (('0' <= ch) && (ch <= '9')) || (('a' <= ch) && (ch <= 'f')) || (('A' <= ch) && (ch <= 'F'));
    }

    void em_char_in_stmnt(int line) {
      // (a little nod to the past...)
      fprintf(stderr, "EM CHAR IN STMNT - DISASTER (detected at parser line %d)\n", line); // stop-gap measure - needs a longjump to end of compile code to clean up
      exit(EXIT_FAILURE);
    }
    #define em_char_in_stmnt() em_char_in_stmnt(__LINE__)

    void line_reconstruction (void) {
      // adds one more item to c[] array.
      // It is up to the caller to keep calling until sufficient data is loaded.
      static int (*acceptable_range)(int ch);
      static int token_nextfree, startcol;
      static int token_arraysize = 0;
      static char *token = NULL;
    #define token(x) FLEX(token, x, __LINE__)
      static int init=0;

      // Recognise keywords, variable names, single and double-quoted strings
      // while removing comments and handling line continuation.
      // The c[] array gets the canonicalised version of the text, BUT retains
      // pointers to the original text including whitespace, which comes in
      // handy for any source-to-source transformations such as SOAP (pretty printing)
      // or transpiling to another language (eg imp to c)
      if (init == 0) {
        at_startofstatement = TRUE;
        at_startofline = TRUE;
        in_keyword = FALSE;
        makespace(token,1024); // quick hack only for test.imp fix as soon as lexer is written.
        init = 1;
      }

      /* Pre-process input ready for parsing.  Tokens are stored in array c[] */
      // go over this again and make sure 'col' is updated correctly at all places where chars are read.
      for (;;) {
        startcol = nextch_col;
        token_nextfree = 0;
        token_lineno = nextch_lineno;
        token_col = nextch_col;
        token_pos = nextch_pos; // actual text of token may start here but whitespace goes back to end of previous token.

        if ((ch = xfgetc ()) == EOF) {
          // set up a dummy at the end because we sometimes look ahead by 1
          // in the parsing code and don't want to hit uninitialised data.
          //fprintf(stderr, "Setting EOF at c_nextfree=%d {tokens}  (source_length=%d {chars}) &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&\n", c_nextfree, source_length);
          c[c_nextfree].f = curfile;

          c[c_nextfree].sIndex = str_to_pool("<EOF>");
          c[c_nextfree].t = B_EOF;

          c[c_nextfree].lineno_sp = whitespace_lineno;
          c[c_nextfree].col_sp = whitespace_col;
          c[c_nextfree].start_sp = whitespace_pos;

          c[c_nextfree].lineno_tok = nextch_lineno;
          c[c_nextfree].col_tok = nextch_col;
          c[c_nextfree].start_tok = nextch_pos;

          c[c_nextfree].end_tok = nextch_pos;
          c_nextfree++;

          // NOTE: we are not yet handling include files...
          // token will still be on the heap at exit.
          #ifdef NEVER
          if (debug_tokens || debug_line_reconstruction) {
            int i;
            fprintf (stderr, "\nLexical token stream:\n\n");
            for (i = 0; i < c_nextfree; i++) {
              fprintf (stderr, "C[%d].f=\"%s\", .lineno=%d, .col=%d, .type=%0d, .s=\"%s\" (at %p)\n",
                               i, c[i].f, c[i].lineno_tok, c[i].col_tok, c[i].t, escape(CString(i),'"'), CString(i));
            }
          }
          #endif
          return;
        }

        // The test below does not catch   Label: !comment or swlab(n): !comment
        // The problem we have to handle is a ! comment that contains a single " which starts a multi-line string
        // because that comment is not recognised until parse-time, but the " is handled at line reconstruction time...
        // We *could* have a hack that treated ':' as ';' but ':' can also occur in array bounds...
        // and '!' can occur in integer expressions, so marking everything that follows a '!' as B_char won't work either.
        // (and would mess up a final %c in a comment)
        if ((at_startofstatement || at_startofline) && (ch == '!')) {
          // For the moment we'll treat '!' comments as whitespace, though in Imp they're parsed statements in the grammar.
          // Whether this will hold up when we hit %comment comments as well is a matter for debate!

          do {
            ch = xfgetc (); if (ch == EOF) em_char_in_stmnt(); // comment that ends in <EOF>???
          } while (ch != '\n');
          // Some specific Imp implementations ended these comments at ';'; others parsed them and obeyed a trailing %c

          // There's also a nasty case of continuation that has to be handled here:
          // %const %integer %array reltag(0:6) =
          //             0,                         ;! no relocation
          //                            IF DATA,    ;! dataseg offset code word

          // The code for removing ';'s after '=' has to be applied here too :-(
          if ((c_nextfree > 1) && (c[c_nextfree-2].t == B_char) && strcmp(CString(c_nextfree-2), ",")==0) {
            if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_char) && strcmp(CString(c_nextfree-1), ";")==0) {
              c_nextfree -= 1;
              // Remove the ; but leave it in the whitespace.  This is sneaky:
              whitespace_lineno =  c[c_nextfree].lineno_sp;
              whitespace_col = c[c_nextfree].col_sp;
              whitespace_pos = c[c_nextfree].start_sp;
              at_startofstatement = TRUE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
              at_startofline = FALSE;
              continue;
            }
          }
          // ch has now been replaced by a '\n'
          // drop through because we still want the newline to have significance
          // (not sure about continuations in this kind of comment)
        }

        if (ch == '\'' || ch == '"') { // start of string. Stored as typed string data without quotes or escaping.
          int quotech = ch;
          int backtrack_token = token_nextfree;
          int backtrack_pos = token_pos;
          int backtrack_lineno = token_lineno;
          int backtrack_col = token_col;
          int stringlength = 0;
          at_startofstatement = FALSE;
          at_startofline = FALSE;
          // probably a char constant but can be other things, eg M'HECK' is a 32-bit int.
          for (;;) {
            ch = xfgetc ();
            if (ch == EOF) {
              // backtrack and warn if EOF found in string - almost certainly an unterminated string
              token_nextfree = backtrack_token;
              token_pos = backtrack_pos;
              token_lineno = backtrack_lineno;
              token_col = backtrack_col;
              ch = quotech;
              storec (ch, token_lineno, token_col, token_pos, B_char, curfile); // leave it for the parser to cause a syntax error.
              return;
            }
            if (ch == quotech) { // quotech in string is represented by doubling.  No C-like escape characters.
              if (peek != quotech) break;
              ch = xfgetc (); if (ch == EOF) em_char_in_stmnt();  // \<EOF> ?!!!!
            }
            makespace(token,token_nextfree+1);
            token(token_nextfree++) = ch;
            stringlength += 1;
            if (stringlength == 256) {
              // backtrack and warn if string is longer than 255 characters - almost certainly an unterminated string
              token_nextfree = backtrack_token;
              token_pos = backtrack_pos;
              token_lineno = backtrack_lineno;
              token_col = backtrack_col;
              ch = quotech;
              storec (ch, token_lineno, token_col, token_pos, B_char, curfile); // leave it for the parser to cause a syntax error.
              return;
            }
          }
          // end of string
          token(token_nextfree) = '\0';
          stores (token,
                  token_lineno, token_col, token_pos,
                  quotech == '"' ? B_dqstring : B_sqstring,
                  curfile);
          return;
        }

        if (ch == '%') {       // start of case-stropping.
          at_startofstatement = FALSE;
          at_startofline = FALSE;
          in_keyword = TRUE;
          continue;
        }

        if (isalpha(ch)) {
          at_startofstatement = FALSE;
          at_startofline = FALSE;
          if (in_keyword) {
            // store a keyword character under B_stropped
            ch = tolower(ch);
            storec (ch,
                    token_lineno, /*-1*/token_col, token_pos,
                    B_stropped, curfile);
            if (tolower(ch) == 'c') continue; // have to continue in case it is a %c.  But could just be start of %cycle ...
                                              // Later add complex logic for %and and %or
            return;
          } else ch = toupper(ch);
        } else in_keyword = FALSE;

        // Skip white space
        if ((ch != '\n') && isspace(ch)) continue;

        if (ch == '{') {
          // Skip comments but they will be attached to the following token.
          // skip to '}' or end of line.
          for (;;) {
            ch = xfgetc (); if (ch == EOF) em_char_in_stmnt(); //  { ... <EOF>?
            if (ch == '}' || ch == '\n') break;
          }
          if (ch == '}') continue;
          // ch is now '\n'
        }

        if (ch == '\n') { // can we skip this newline?

          // This continuation stuff gets REALLY ugly when you finally remember that none of it
          // is allowed in a comment :-(

          // It is almost enough to make you add '!' comments to the line reconstruction...
          // but even then a %comment ending in a ',' is a pain.  For example, in the pdp15 compiler:
          // C(39): !BEGEND: ENDOFPRIM(1), ENDOFPERM(2), BEGIN(111),
          //        ! END(260), ENDOFPROGRAM(261)
          //      %IF X = 1 %THEN %START
          //          TBASE = TMAX %IF TBASE = 0
          //      %FINISH
          // the comma in the comment which has a label in front of it causes a continuation
          // (because it is a parsed comment, not a skipped comment) but the line following
          //  which is a skipped comment is skipped, so the line after that (a %IF statement)
          // ends up as part of the first comment!  YUCK.
          // So I *REALLY* need to come up with something better...
          // Current idea... a parse-time statement in the grammar after all labels,
          // which resets the 'at start of line' flag... :-(
          // - NOPE! can't work, the line reconstruction happens *before* parse time.
          // Can we delay it so it is done on demand while parsing, perhaps?

          // Note that in a %comment ending in a %c, the %c really is a continuation symbol,
          // even for the stupid case of:  %comment Note that the routine above is %dynamic

          // if last non-skipped,non-whitespace character in c[] was %c then remove it and the newline.
          if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_stropped) && strcmp(CString(c_nextfree-1), "c")==0) {
            c_nextfree -= 1;
            // Remove the %c but leave it in the whitespace.  This is sneaky:
            whitespace_lineno =  c[c_nextfree].lineno_sp;
            whitespace_col = c[c_nextfree].col_sp;
            whitespace_pos = c[c_nextfree].start_sp;
            at_startofstatement = FALSE;
            at_startofline = TRUE;
            continue;
          }

          // if last non-skipped,non-whitespace character in c[] was '-' then skip it and the newline.
          if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_char) && strcmp(CString(c_nextfree-1), "-")==0) {
            // THIS IS UGGGGLY!
            // And there may yet be cases where it is wrong, eg if some idiot wrote:
            //
            // %if a <-
            //         b %then b=a
            //
            // Hamish's 68000 Imp unfortunately supports both of these conventions:
            // a:   '=' or '<-' at the end of a line in a declaration implicitly allows continuation
            // b:   '-' at the end of a line is almost identical to '%c' at the end of a line.
            // The line is continued and the '-' removed.
            // Presumably Hamish handled the ambiguity by making '<-' an indivisible token, not to
            // be confused with '<' followed by '-'...  So I'll try doing the same.
            //
            if ((c_nextfree > 1) && (c[c_nextfree-2].t == B_char) && (strcmp(CString(c_nextfree-2), "<")==0)
                && /* Indivisibility test: */ (c[c_nextfree-1].start_tok == c[c_nextfree-1].start_sp)) {
              // '<-' is handled like '=' and the '-' is *NOT* removed
              at_startofstatement = FALSE;
              at_startofline = TRUE;
              continue;
            } else {
              c_nextfree -= 1;
              // Otherwise, remove the "-" but leave it in the whitespace.
              whitespace_lineno =  c[c_nextfree].lineno_sp;
              whitespace_col = c[c_nextfree].col_sp;
              whitespace_pos = c[c_nextfree].start_sp;
              at_startofstatement = FALSE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
              at_startofline = TRUE;
              continue;
            }
          }

          // if last non-skipped,non-whitespace character in c[] was %or then remove the newline.
          if ((c_nextfree > 1) && (c[c_nextfree-2].t == B_stropped) && strcmp(CString(c_nextfree-2), "o")==0) {
            if ((c[c_nextfree-1].t == B_stropped) && strcmp(CString(c_nextfree-1), "r")==0) {
              at_startofstatement = FALSE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
              at_startofline = TRUE;
              continue;
            }
          }
          // if last non-skipped,non-whitespace character in c[] was %and then remove the newline.
          if ((c_nextfree > 2) && (c[c_nextfree-3].t == B_stropped) && strcmp(CString(c_nextfree-3), "a")==0) {
            if ((c[c_nextfree-2].t == B_stropped) && strcmp(CString(c_nextfree-2), "n")==0) {
              if ((c[c_nextfree-1].t == B_stropped) && strcmp(CString(c_nextfree-1), "d")==0) {
                at_startofstatement = FALSE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
                at_startofline = TRUE;
                continue;
              }
            }
          }
          // if last non-skipped,non-whitespace character in c[] was '=' or ',' then skip the newline.
          if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_char) && strcmp(CString(c_nextfree-1), "=")==0) {
            at_startofstatement = FALSE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
            at_startofline = TRUE;
            continue;
          }
          if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_char) && strcmp(CString(c_nextfree-1), ",")==0) {
            at_startofstatement = FALSE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
            at_startofline = TRUE;
            continue;
          }
          // I believe it should also keep skipping until a non-space character but that's pathological
          // and I'm not going to implement that unless I have to.
          // store a newline (for phrase S) under B_NL

          // Finally a nasty one, should never have been allowed...
          // %OWNBYTEINTEGERARRAY char(0:133) = ;
          if ((c_nextfree > 1) && (c[c_nextfree-2].t == B_char) && strcmp(CString(c_nextfree-2), "=")==0) {
            if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_char) && strcmp(CString(c_nextfree-1), ";")==0) {
              c_nextfree -= 1;
              // Remove the ; but leave it in the whitespace.  This is sneaky:
              whitespace_lineno =  c[c_nextfree].lineno_sp;
              whitespace_col = c[c_nextfree].col_sp;
              whitespace_pos = c[c_nextfree].start_sp;
              at_startofstatement = FALSE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
              at_startofline = TRUE;
              continue;
            }
          }
          at_startofstatement = TRUE; // allow !-style comment to follow
          at_startofline = TRUE;
        } // continued lines have been skipped.

        // if ch was '\n', it will be stored unless we are following a %c - multiple blank likes must be skipped, not just the one following newline
        if (ch == '\n') {

    #ifdef NEVER
          // however this is not the way to do it!
          if ((c_nextfree > 0) && (c[c_nextfree-1].t == B_stropped) && strcmp(CString(c_nextfree-1), "c")==0) {
            c_nextfree -= 1;
            // Remove the %c but leave it in the whitespace.  This is sneaky:
            whitespace_lineno =  c[c_nextfree].lineno_sp;
            whitespace_col = c[c_nextfree].col_sp;
            whitespace_pos = c[c_nextfree].start_sp;
            at_startofstatement = TRUE; // NOT REALLY START OF STATEMENT BUT ! comments NEED TO THINK IT IS
            at_startofline = TRUE;
            continue;
          }
    #endif

          storec (ch,
                token_lineno, token_col, token_pos,
                B_NL, curfile);
          return;
        }

        // store the character under B_char

        if (ch == 127 || ch == ('L'&31) || !isprint(ch)) {
          fprintf(stderr, "? Warning: stray character %d in program\n", ch);
          continue; // treat it as white space.
        }

        at_startofstatement = FALSE;
        at_startofline = FALSE;
        storec (ch,
                token_lineno, /*-1*/token_col, token_pos,
                B_char, curfile);

        if (ch == '-') continue; // have to continue in case it is a terminal '-'.

        if (ch == ';') { // not in quote or comment so must be end of statement.
          at_startofstatement = TRUE; // allow !-style comment to follow
          at_startofline = FALSE;
          // would be nice to also set if following a label at the start of a statement...
        }

        return; // most single character fetches return here.

      }

    }