parser.c

    #include <stdio.h>
    #include <stdlib.h>
    #include <string.h>
    #include <errno.h>
    #include <stdarg.h>
    #include <ctype.h>

    #ifndef FALSE
    #define FALSE (0!=0)
    #define TRUE (0==0)
    #endif

    #define NEGATED_PHRASE (1<<29)
    #define OPTIONAL_PHRASE (1<<30)
    #define DROP_SPECIAL_BITS (~(NEGATED_PHRASE|OPTIONAL_PHRASE))

    #define MAX_PHRASES 1024
    #define WHITESPACE (MAX_PHRASES-1)

    char *progname;
    static char outname[256];
    FILE *outfile;

    #define DEBUG_PARSER 1 // enable some extra data in header file
    #define DEBUG_TOKENS 1 // debug line reconstruction
    #define DEBUG_AREC   1
    #define DEBUG_AST   1
    int cur_listing, listing = 1;

    #define EXTENDED_ERROR_REPORTING 1 // applies to parser.h ...
    #define WANT_COMMENT 1
    #define WANT_SEMANTIC_CODE 1
    #define WANT_SEMANTIC_TABLE 1

    // this area needs review:
    extern int *A;
    extern int cp;                     // code pointer.  Has to be global state.
    extern int ap;                     // Analysis record pointer.
    int typedef_table[123];// for now
    int push(int val, int *table);
    int check(int Aval, int *table);

    #include "parser.h"              // GENERATED GRAMMAR FROM takeon

    #ifndef SEMANTIC_NAME
    #define SEMANTIC MAX_PHRASES
    //#define SEMANTIC (MAX_PHRASE+512+MAX_BIP+1)
    char *semantic_code[1];
    int semantic(int phrase) {
      return FALSE;
    }
    #endif

    #define FLEX(array,x,l) array[CHECKIDX(x,array##_arraysize,#array,l)]
    #define BOUNDS(array,x,lim,l) array[CHECKIDX(x,lim,#array,l)]

    #define gram(x) BOUNDS(gram, x, MAX_GRAMMAR, __LINE__)
    #define keyword(x) BOUNDS(keyword, x, MAX_KEYWORD, __LINE__)
    #define BIP(x) BOUNDS(BIP, x, MAX_BIP, __LINE__)
    #define phrasename(x) BOUNDS(phrasename, x, MAX_PHRASE, __LINE__)
    #define phrase_start(x) BOUNDS(phrase_start, x, MAX_PHRASE-MAX_BIP, __LINE__)

    int debug_parser = FALSE; // use -d option
    int debug_flex = FALSE; // excessive debugging that I needed when finding the problem with using string addresses rather than stringpool offsets...

    void indent (int n, FILE * f) {
      fprintf (f, "%-*s", n*3, " ");
    }

    int bestparse = -1;             // for error reporting.
    char **parse_stack = NULL;
    int parse_stack_arraysize = 0;
    #define parse_stack(x) FLEX(parse_stack, x, __LINE__)

    char **bestparse_stack = NULL;
    int bestparse_stack_arraysize = 0;
    #define bestparse_stack(x) FLEX(bestparse_stack, x, __LINE__)
    int bestparse_depth = -1;

    char *looking_for = "<UNKNOWN>";        // 'while looking for <PHRASENAME>' (or literal) ...

      // We use a stringpool, and strings are indexes into this pool.  This
      // is useful for the same reasons that the AST is an array indexed
      // by integers rather than a struct with pointers.  It may also
      // save space by reusing common strings.  And we get a free tag
      // to describe strings.  (Also we can compare strings just with
      // an integer tag comparison, if we ever want to)

    static void *makespace_ (void *c, int nextfree, int *arraysize, int objsize, char *arrayname) {
    #define MINSIZE 1    // 64 causes an error, so we are not being clean somewhere...
                         // However, whatever the error is, it probably isn't an array bounds error,
                         // as we're checking for those with CHECKIDX ...

      // *arraysize is the upper bound index of the array, i.e. 1 less than the actual number of elements.
      if (debug_flex) fprintf(stderr, "makespace: %s needs space for %d entries (currently %d)\n", arrayname, nextfree, *arraysize);
      if (c == NULL) {    // TOO MANY TESTS! Need to pick the right one!
        if (debug_flex) fprintf (stderr, "makespace: %s = calloc(%d, %d)\n", arrayname, MINSIZE, objsize);
        c = calloc (MINSIZE, objsize);
        if (c == NULL) {
          fprintf (stderr, "makespace: calloc(%d, %d) - %s\n", MINSIZE, objsize, strerror (errno));
          exit (errno);
        }
        *arraysize = MINSIZE - 1;
      }
      if ((*arraysize) > nextfree) return c;
      // *arraysize is the space allocated, nextfree is the space that will used soon.
      // so we want to make sure that *arraysize >= nextfree
      *arraysize += 1; // convert from high index to actual size
      while ((*arraysize) <= nextfree) {
        *arraysize = ((*arraysize) * 2);
        if (debug_flex) fprintf(stderr, "makespace: %s increased to %d entries\n", arrayname, *arraysize);
      }
      // *arraysize = ((*arraysize) * 2);
      if (debug_flex) fprintf (stderr, "makespace: %s = realloc(%d * %d)\n", arrayname, (*arraysize), objsize);
      c = (void *) realloc (c, (*arraysize) * objsize);       // 0:arraysize, inclusive. eg 0:15, 0:127 etc
      *arraysize -= 1; // restore size back down to upper bound index
      if (c == NULL) {
        fprintf (stderr, "makespace: realloc(%d * %d) - %s\n", (*arraysize), objsize, strerror (errno));
        exit (errno);
      }
      return c;
    }
    #define makespace(c, nextfree) c = (typeof(c))makespace_(c, nextfree, &c##_arraysize, sizeof(c[0]), #c)

    char *stringpool;
    int stringpool_arraysize = 0;
    int stringpool_nextfree = 0;

    char *escape(char *s, int ch);
    int str_to_pool (char *s) {
      int tag;
      // assert(stringpool_nextfree <= stringpool_arraysize);
      makespace(stringpool, stringpool_nextfree+strlen(s)+2);

      strcpy (&stringpool[stringpool_nextfree], s);  /* Create a backstop for when not found */
      for (tag = 0; tag <= stringpool_nextfree; tag++) {
        if (strcmp (&stringpool[tag], s) == 0) {
          if (tag == stringpool_nextfree) {
            if (debug_flex) fprintf(stderr, "adding pre-positioned stringpool: %d: \"%s\" (extends to %d)\n",
                                      stringpool_nextfree, escape(&stringpool[stringpool_nextfree], '"'), stringpool_nextfree + strlen (s) + 1);
            stringpool_nextfree += strlen (s) + 1;       /* Not found, add it */
          } else {
            if (debug_flex) fprintf(stderr, "found in existing stringpool: %d: \"%s\"\n", tag, escape(&stringpool[tag], '"'));
            if (debug_flex) fprintf(stderr, "pre-positioned stringpool ignored: %d: \"%s\" (extended to %d)\n",
                                      stringpool_nextfree, escape(&stringpool[stringpool_nextfree], '"'), stringpool_nextfree+strlen (s) + 1);
          }
          return tag;               /* found, one way or another */
        }
      }
    }

      /* C[] is the source character token stream */

    typedef struct sourceinfo {     // ATOMS for processed input stream
      //char *s;                    // string contents
      int sIndex;                   // string contents
      int start_sp, start_tok, end_tok; // starting 2 ptrs are inclusive, end ptr is exclusive.
                                        // if s_whitespace==s_text then whitespaces is empty,
                                        // if s_text==s_end, then s_text is empty.
      int lineno_sp, lineno_tok;    // lineno
      int col_sp, col_tok;          // column
      int t;                        // type - tag, "string", 'charconst', or char, so far
      char *f;                      // source or includefile name
    } sourceinfo;

    static sourceinfo *c = NULL;
    int c_nextfree = 0, c_arraysize = 0;
    #define c(x) FLEX(c, x, __LINE__)

    /* A() is the Analysis record.
         Contents point to a C[] structure when the item is a BIP,
         otherwise the format is [phraseno] [altno] [no-of-subphrases] [subphrases and/or BIPs...]
         for example, if A(25) contained 10, that would mean that the
         token for A(25) was stored in C[10].]
         (And the C string would be at &stringpool[c(A(25))]?)
       */

    int *A = NULL;           /* Flex array, expanded using 'makespace' below */
    int next_free_a = 0, A_arraysize = 0;
    #define A(x) FLEX(A, x, __LINE__)

    int CHECKIDX(int x, int arraysize, char *name, int line) {
      if ((x < 0) || (x >= arraysize)) {
        fprintf(stderr, "* Array bounds exceeded at line %d: %s[%d] is outside %s[%d]\n",
                line, name, x, name, arraysize);
        exit(4);
      }
      return x;
    }

    int *AST;
    int AST_arraysize;
    int AST_nextfree = 0;
    #define AST(x) FLEX(AST, x, __LINE__)
    ////////////////////////////////////////////////////////////////////////////

    int mktuple(int op, int alt, int count, int T[]) {
      /*  Create a tuple for an n-ary operator.  Pre-generated code
          currently just mirrors the Analysis record. */
      int trip = -1;
      int parm;

      makespace (AST, AST_nextfree+4+count+1);

      trip = AST_nextfree;
      AST(AST_nextfree++) = 0;  // reserved field
      AST(AST_nextfree++) = op;
      AST(AST_nextfree++) = alt;
      AST(AST_nextfree++) = count;
      for (parm = 0; parm < count; parm++) {
        AST(AST_nextfree++) = T[parm+1];
      }
      return trip;
    }

    char *escape(char *s, int ch) {
      const char *hexdigit = "0123456789ABCDEF";
      static char escaped[512];
      char *t = escaped;
      *t = '\0';
      if (s == NULL) return escaped;
      while (*s != '\0') {
        if (*s == '\n') {
          *t++ = '\\'; *t++ = 'n'; s++;
        } else if (*s == '\t') {
          *t++ = '\\'; *t++ = 't'; s++;
        } else if (*s == '\r') {
          *t++ = '\\'; *t++ = 'r'; s++;
        } else if (*s == '\\') {
          *t++ = '\\'; *t++ = '\\'; s++;
        } else if (*s == ch) {
          *t++ = '\\'; *t++ = ch; s++;
        } else if ( ((int)((*s)&255) < 32) || ((int)((*s)&255) > 126) ) {
          *t++ = '\\'; *t++ = '0'; *t++ = 'x';
          *t++ = hexdigit[((*s)>>4)&15];
          *t++ = hexdigit[(*s)&15];
          s++;
        } else {
          if (t-escaped < 510) *t++ = *s++; else *t = '\0';
        }
      }
      *t = *s;
      if (t-escaped == 510) {
        fprintf(stderr, "WARNING! Escaped string is too large! (over %d chars): \"%s\"...\n", t-escaped, escaped);
        exit(1);
      }
      return escaped;
    }

    int identifier(int ap)
    {
      int data[2];
      data[1] =  A(A(ap)+3);
      if (listing) fprintf(stderr, " %s ", &stringpool[c(data[1]).sIndex]);
      return mktuple(B_identifier, 1, 1, data);
    }

    int string(int ap)
    {
      int data[2];
      data[1] =  A(A(ap)+3);
      if (listing) fprintf(stderr, "\"%s\"", escape(&stringpool[c(data[1]).sIndex], '"'));
      return mktuple(B_string, 1, 1, data);
    }

    int floating_constant(int ap)
    {
      int data[2];
      data[1] =  A(ap+4);
      data[1] =  A(A(ap)+3);
      if (listing) fprintf(stderr, "%s", &stringpool[c(data[1]).sIndex]);
      return mktuple(B_floating_constant, 1, 1, data);
    }

    int integer_constant(int ap)
    {
      int data[2];
      data[1] =  A(ap+4);
      data[1] =  A(A(ap)+3);
      if (listing) fprintf(stderr, "%s", &stringpool[c(data[1]).sIndex]);
      return mktuple(B_integer_constant, 1, 1, data);
    }

    int character_constant(int ap)
    {
      int data[2];
      data[1] =  A(ap+4);
      data[1] =  A(A(ap)+3);
      if (listing) fprintf(stderr, "'%s'", escape(&stringpool[c(data[1]).sIndex], '\''));
      return mktuple(B_character_constant, 1, 1, data);
    }

    // TO DO: make these elements in the c() array as well.  It'll simplify debugging the A() tree and later, the AST...
    int bip(int B, int aptr)
    {
      int data[3];

      if (listing) {
        if (B == B_pp) fprintf(stderr, " ++ ");
        if (B == B_mm) fprintf(stderr, " -- ");
        if (B == B_andand) fprintf(stderr, " && ");
        if (B == B_oror) fprintf(stderr, " || ");
        if (B == B_eqeq) fprintf(stderr, " == ");
      }

      switch (B) {
      case B_pp:
      case B_mm:
      case B_andand:
      case B_oror:
      case B_eqeq:
        data[1] = B;
        //    data[2] = B;
        return mktuple(B, 1, /*2*/1, data);

      case B_identifier:         return identifier(aptr);
      case B_character_constant: return character_constant(aptr);
      case B_floating_constant:  return floating_constant(aptr);
      case B_integer_constant:   return integer_constant(aptr);
      case B_string:             return string(aptr);
      case B_EOF:                return mktuple(B_EOF, 1, 1, data);
      default: break;
      }

      fprintf(stderr, "*** WARNING: Unhandled BIP %d\n", B);
      return mktuple(B, 1, 1, data);
    }

    int kw(int idx)
    {
      int data[2];
      data[1] =  idx;
      if (listing) fprintf(stderr, " %s ", keyword(data[1]));
      return mktuple(B_keyword, 1, 1, data);
    }

    int lit(int ch)
    {
      int data[2];
      data[1] =  ch;
      if (listing) {
        fprintf(stderr, "%c", data[1]);
        if (ch == ';' || ch == '{') fprintf(stderr, "\n");
      }
      return mktuple(B_char, 1, 1, data);
    }

    ////////////////////////////////////////////////////////////////////////////

    /* variables used by line-reconstruction (lexer) */
    FILE *sourcefile;
    char *curfile;
    int startline = TRUE, whitespace = TRUE, lineno = 1, col = 0, ch, peek; // peek is always the *immediate* next char, no skipping of spaces

    static int xi = 0, xcur_line = 1;
    static char *xline = NULL;
    static int xline_arraysize = 0;
    #define xline(x) FLEX(xline, x, __LINE__)

    static void xungetc (int c, FILE * f) { // for a xfgetc char, not for peek.
      if (xi > 0) {
        xi -= 1; col -= 1;
      } else if (c == '\n') {
        xcur_line -= 1; lineno -= 1;
        xi = strlen (xline);
        col = xi+1;
      }
      ungetc (c, f);
      peek = c; // this will be the next char that would be returned by xfgetc...
    }

    static int last_line = -1;
    static int xfgetc (FILE * f) {
      int c, ch;
      peek = EOF;
      if ((c = fgetc (f)) == EOF)
        return EOF;
      peek = fgetc (f); ungetc(peek, f); // peek is always up to date whenever we getc.
      ch = c & 255;
      if (ch == '\n') {
        makespace (xline, xi+1);
        xline(xi) = '\0';
        col = 1;
        xi = 0;
        if (last_line != xcur_line) {
          // (void)make_binary_tuple(LINE, xcur_line, str_to_pool(xline));
          last_line = xcur_line;
        }
        xcur_line += 1; lineno += 1;
      } else {
        makespace (xline, xi+1);
        xline(xi++) = ch;
        col += 1;
      }
      // if (xi == 1023) xi = 1022;
      makespace (xline, xi+1);
      xline(xi) = '\0';
      return c;
    }

    void stores (char *s, int lineno, int col, int type, char *fname) {
      int tag;
      tag = str_to_pool (s);

      makespace (c, c_nextfree+1);
      /*
      char *s;                      // string contents
      int start_sp, start_tok, end_tok; // starting 2 ptrs are inclusive, end ptr is exclusive.
                                        // if s_whitespace==s_text then whitespaces is empty,
                                        // if s_text==s_end, then s_text is empty.
      int lineno_sp, lineno_tok;    // lineno
      int col_sp, col_tok;          // column
      int t;                        // type - tag, "string", 'charconst', or char, so far
      char *f;                      // source or includefile name
     */

      c(c_nextfree).sIndex = tag; // pointer to index of string in stringpool - NOT string because stringpool can be reallocked.
      c(c_nextfree).lineno_tok = lineno;
      c(c_nextfree).col_tok = col;
      c(c_nextfree).f = fname;
      c(c_nextfree).t = type;
      if (debug_flex) {
        fprintf(stderr, "Setting c[%d]: lineno=%d col=%d f=\"%s\" t=%d s=\"%s\" (at %p)\n",
                         c_nextfree, c(c_nextfree).lineno_tok,
                         c(c_nextfree).col_tok, c(c_nextfree).f, c(c_nextfree).t,
                                escape(&stringpool[c(c_nextfree).sIndex], '"'), &stringpool[c(c_nextfree).sIndex]);
        fprintf(stderr, "CHECK c[%d]: lineno=%d col=%d f=\"%s\" t=%d s=\"%s\" (at %p)\n",
                         0, c(0).lineno_tok,
                         c(0).col_tok, c(0).f, c(0).t,
                                escape(&stringpool[c(0).sIndex], '"'), &stringpool[c(0).sIndex]);
      }
      c_nextfree++;
    }

    void storec (int ch, int lineno, int col, int type, char *fname) {
      char str[2];
      str[0] = ch&255; str[1] = '\0';        // convert char to 1-char string before saving.
      stores (str, lineno, col, type, fname);
    }

    int is_binary(int ch) {
      return ((ch|1) == '1');
    }

    int is_octal(int ch) {
      return ('0' <= ch) && (ch <= '7'); // or...  (ch|7) =='7'
    }

    int is_decimal(int ch) {
      return ('0' <= ch) && (ch <= '9');
    }

    int is_hex(int ch) {
      return (('0' <= ch) && (ch <= '9')) || (('a' <= ch) && (ch <= 'f')) || (('A' <= ch) && (ch <= 'F'));
    }

    void line_reconstruction (FILE *sourcefile, char *filename) {
      int (*acceptable_range)(int ch);
      int token_nextfree, startcol;
      static int token_arraysize = 0;
      static char *token = NULL;
    #define token(x) FLEX(token, x, __LINE__)

      /* Pre-process input ready for parsing.  Tokens are stored in array C[] */
      // go over this again and make sure 'col' is updated correctly at all places where chars are read.
      for (;;) {
        startcol = col;
        token_nextfree = 0;

        if ((ch = xfgetc (sourcefile)) == EOF)
          break;

        ch &= 255;                  // int, positive.

        if ((ch == '+' || ch == '-' || ch == '&' || ch == '|' || ch == '=') && (peek == ch)) {
          /* ++, --, &&, || */
          // only a very small number of compound graphemes need to be converted to lexical tokens (BIPs) to avoid parsing amniguities
          whitespace = FALSE;
          makespace (token, token_nextfree+4);
          token(token_nextfree++) = ch;
          token(token_nextfree++) = ch;
          token(token_nextfree++) = '\0';
          ch = xfgetc (sourcefile); // eat 'peek' ch.
          stores (token, lineno, startcol, (ch=='+'?B_pp:ch=='-'?B_mm:ch=='&'?B_andand:ch=='='?B_eqeq:B_oror), curfile);
        } else if (isalpha (ch) || (ch == '_')) {
          /* token or keyword */

          whitespace = FALSE;
          for (;;) {
            makespace (token, token_nextfree+1);
            if (isalpha (ch) || isdigit (ch) || (ch == '_')) {
              // digits allowed after 1st char.
              token(token_nextfree++) = ch;
            } else {
              xungetc (ch, sourcefile);
              break;
            }
            ch = xfgetc (sourcefile);
          }
          token(token_nextfree++) = '\0';
          stores (token, lineno, startcol, B_identifier, curfile);

        } else if (((ch == '.') && isdigit(peek)) || (isdigit (ch))) {  // -->0B.110101100P12L
          /* Number */
          acceptable_range = &is_decimal;
          if (ch == '0') { // must be binary, octal, or hex.
            // Store as a string...
            whitespace = FALSE;

            makespace (token, token_nextfree+3);
            if (peek == 'b' || peek == 'B' || (peek == 'x') || (peek == 'X')) {
              token(token_nextfree++) = ch; // '0'
              acceptable_range = (peek == 'b' || peek == 'B' ? &is_binary : &is_hex);
              ch = xfgetc (sourcefile); // get the B
              token(token_nextfree++) = ch; // b
              if (peek == '.') {
                goto READ_DECIMAL;
              }
            } else if (peek == '.') {
              token(token_nextfree++) = ch; // '0'
              // started with a 0 so octal unless it is 0.19 etc...
              acceptable_range = &is_decimal;
              goto READ_DECIMAL;
            } else {
              acceptable_range = &is_octal; // the 0 has been read.  If nothing acceptable follows, don't read any more chars
              token(token_nextfree++) = ch; // digit
            }
          } else if (ch == '.') {  //   .nnn or 0.nnn
            acceptable_range = &is_decimal;
            xungetc (ch, sourcefile); // peek is now '.'
            goto READ_DECIMAL;
          } else if (isdigit(ch)) {
            acceptable_range = &is_decimal;
            token(token_nextfree++) = ch; // digit
          }

          // rest of number comes through here for all formats
          while (acceptable_range(peek)) {  // 0B-->.110101100P12L
            ch = xfgetc (sourcefile); // what was peek is now ch
            makespace (token, token_nextfree+2);
            token(token_nextfree++) = ch; // 0
          }
        READ_DECIMAL:
          if (peek == 'p' || peek == 'P') {
            goto BINARY_POWER;
          } else if (peek == '.' || peek == 'e' || peek == 'E') {
            if (peek == '.') {
              ch = xfgetc (sourcefile);
              token(token_nextfree++) = ch;
              while (acceptable_range(peek)) {
                ch = xfgetc (sourcefile); // what was peek is now ch
                makespace (token, token_nextfree+2);
                token(token_nextfree++) = ch; // 0
              }
            }
            if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
              ch = xfgetc (sourcefile);
              token(token_nextfree++) = ch;
            } else if (((acceptable_range == is_decimal) && (peek == 'e' || peek == 'E'))
                || ((acceptable_range != is_decimal) && (peek == 'p' || peek == 'P'))) {
            BINARY_POWER:
              ch = xfgetc (sourcefile);
              token(token_nextfree++) = ch; // P
              acceptable_range = &is_decimal; // P+nnn nnn is decimal
              if ((peek == '+') || (peek == '-')) {
                ch = xfgetc (sourcefile);
                token(token_nextfree++) = ch;
              }
              if (!acceptable_range(peek)) {
                token(token_nextfree) = '\0';
                fprintf(stderr, "Warning: bad floating point literal '%s'\n", token);
              }
              while (acceptable_range(peek)) { // SHOULD pick up P12L etc
                makespace (token, token_nextfree+2);
                ch = xfgetc (sourcefile); // what was peek is now ch
                token(token_nextfree++) = ch; // 0
              }
              if ((peek == 'f') || (peek == 'F') || (peek == 'l') || (peek == 'L')) {
                ch = xfgetc (sourcefile);
                token(token_nextfree++) = ch;
              }
            }
          } else if ((peek == 'u') || (peek == 'U')) {
            ch = xfgetc (sourcefile);
            token(token_nextfree++) = ch;
          } else if ((peek == 'l') || (peek == 'L')) {
            ch = xfgetc (sourcefile);
            token(token_nextfree++) = ch;
            if (ch == peek) { // 'll' or 'LL' allowed but not 'lL' or 'Ll'
              ch = xfgetc (sourcefile);
              token(token_nextfree++) = ch;
            }
          }
          makespace (token, token_nextfree+1);
          token(token_nextfree++) = '\0';
          if (strchr(token, '.') == NULL) {
            stores (token, lineno, startcol, B_integer_constant, curfile); // changed from col to startcol
          } else {
            stores (token, lineno, startcol, B_floating_constant, curfile); // changed from col to startcol
          }
        } else {
          switch (ch) {

          case '\'':               // Handle 'c' char const
          case '"':                // Handle "string"
            /* literals */
            {
              int string_nextfree = 0, string_arraysize = 0, quotech = ch;
              char *string = NULL;

              whitespace = FALSE;
              for (;;) {
                ch = xfgetc (sourcefile);   // Newlines are allowed
                makespace (string, string_nextfree+3);
                if (ch == '\\') {
                  ch = xfgetc (sourcefile);
                  if (ch == '\\') {
                    string[string_nextfree++] = ch;
                  } else if (ch == '\'') {
                    string[string_nextfree++] = '\'';
                  } else if (ch == '"') {
                    string[string_nextfree++] = '"';
                  } else if (ch == 'n') {
                    string[string_nextfree++] = '\n';
                  } else if (ch == 'r') {
                    string[string_nextfree++] = '\r';
                  } else if (ch == 't') {
                    string[string_nextfree++] = '\t';
                  } else {
                    // Warn of unknown (to me) \x escape.  Probably an error.
                    string[string_nextfree++] = '\\';
                    string[string_nextfree++] = ch;
                  }
                } else if (ch != quotech) {
                  string[string_nextfree++] = ch;
                } else {
                  string[string_nextfree] = '\0';
                  break;
                }
              }

              if (quotech == '\'') {
                stores (string, lineno, col, B_character_constant, curfile);
                if (strlen (string) == 1) {
                  // good char const
                } else if (strlen (string) <= 4) {
                  // Warn that 'xx' as a 32-bit int is a non-standard extension
                  fprintf(stderr, "Warning: multi-byte characters constants such as \'%s\' are not well supported.\n", escape(string, '\''));
                } else {
                  // Warn that this is probably a string with the wrong type of quote.
                  fprintf(stderr, "Warning: wrong kind of string quote used? - \'%s\'.\n", escape(string, '\''));
                }
              } else stores (string, lineno, col, B_string, curfile);
              free (string);
            }
            break;

          case '/':
            /* COMMENTS (or just a divide symbol) */
            whitespace = FALSE;
            if (peek == '/') {
              // Handle line comment
              do {
                ch = xfgetc (sourcefile);
              } while (ch != '\n');
              whitespace = TRUE;
            } else if (peek == '*') {
              /* Handle potential multi-line comment */
              ch = xfgetc (sourcefile);     // Now we have read '/*'
              for (;;) {
                ch = xfgetc (sourcefile);
                if ((ch == '*') && (peek == '/')) {
                  ch = xfgetc (sourcefile); // get the '/' too.
                  break;
                }
              }
              whitespace = TRUE;
            } else {
              storec (ch, lineno, col-1, B_char, curfile); // the '/'
            }
            break;

            // WHITESPACE
          case '\n':
          case '\r':
            startline = TRUE;
            whitespace = TRUE;
            break;

          case '\t':
          case ' ':
            startcol = col;
            whitespace = TRUE;
            // to check: startline
            break;

            // DIRECTIVES
          case '#':
            // should allow '#' as a directive if startline is true. (and not in a comment of course)

            // If we interpret any #-directives while lexing, we don't want to
            // do an expensive test on every token, so what we can do is set
            // a countdown timer on the introductory token (either this '#'
            // or the actual keyword such as 'ifdef') and then test that the
            // *previous* tokens match when the timer hits 0, eg
            // C[cp-3] == '#' && C[cp-2] == 'include' ... etc

            if (!whitespace) {
              // WARN: probably an error... should not be any '#' symbols in the
              // middle of a line. (This language uses "!=" for not-equal)
            }
            // Drop through

          default:
            if (whitespace) storec (' ', lineno, col-1, B_char, curfile);
            whitespace = FALSE;
            storec (ch, lineno, col, B_char, curfile);
          }
        }
      }
      // set up a dummy at the end because we sometimes look ahead by 1
      // in the parsing code and don't want to hit uninitialised data.

      makespace (c, c_nextfree+1);
      /*
      char *s;                      // string contents
      int start_sp, start_tok, end_tok; // starting 2 ptrs are inclusive, end ptr is exclusive.
                                        // if s_whitespace==s_text then whitespaces is empty,
                                        // if s_text==s_end, then s_text is empty.
      int lineno_sp, lineno_tok;    // lineno
      int col_sp, col_tok;          // column
      int t;                        // type - tag, "string", 'charconst', or char, so far
      char *f;                      // source or includefile name
     */

      c(c_nextfree).sIndex = str_to_pool("<EOF>");
      c(c_nextfree).t = B_EOF;
      c(c_nextfree).lineno_tok = lineno;
      c(c_nextfree).col_tok = col;

      // token will still be on the heap at exit.
    }

    /* Parsing.  (Table driven top-down recursive-descent parser) */

    /*  The parser used here is based on a design by Tony Brooker which was
        originally used in Atlas Autocode and the "Compiler Compiler".
        It generates a concrete syntax tree rather than the abstract
        syntax tree more popular in modern compilers.  A later phase
        converts from concrete to abstract.

        Note that the parsing procedure here is just a piece of code
        to walk a pre-built table.  There is nothing in this section
        which reflects the grammar, if that is what you are looking for.
        You'll find the grammar embedded in the 'compile()' procedure
        in the following section.
     */

    int cp = 0;                     // code pointer.  Has to be global state.
    int ap = 0;                     // Analysis record pointer.

    int push(int val, int *table) {
    }

    int check(int Aval, int *table) { // quick hack while testing...
      static int seq = 0;
      fprintf(stderr, "******************************** SEQ=%d ******************************* Temporary Hack\n", seq);
      if (seq++ == 0) return FALSE;
      return TRUE;
    }

    int parse (int pp, int depth)   // depth is only for indentation in diags
    {
      /* Main parsing procedure.  This is a table-driven parser, with the tables
         being generated from the grammar rules embedded in cexpr.c */
      int saved_cp, saved_ap, i, gp, alts, match, where=0;
    #define MAX_DESC 1024
      char saved_desc[(MAX_DESC*3)/2];

      if (pp == WHITESPACE) {
        int optional = pp&OPTIONAL_PHRASE, negated = pp&NEGATED_PHRASE;
        fprintf(stderr, "Should never get here. Trying to parse <%ssp> as an actual phrase rule.\n",
                optional?"?":negated?"!":"");
        exit(1);
      }

      if (pp & (OPTIONAL_PHRASE|NEGATED_PHRASE)) {
        int optional = pp&OPTIONAL_PHRASE, negated = pp&NEGATED_PHRASE;
        fprintf(stderr, "Should never get here. Trying to parse <%s%s> as an actual phrase rule.\n",
                optional?"?":negated?"!":"", phrasename((pp&DROP_SPECIAL_BITS) - 512));
        exit(4);
      }

      makespace (parse_stack, depth+1);
      parse_stack(depth) = phrasename(pp - 512);

      /* Initialisation */
      gp = phrase_start(pp - 512 - MAX_BIP);
      alts = gram(gp);

      /* Debugging */
      if (debug_parser) {
        int optional = pp&OPTIONAL_PHRASE, negated = pp&NEGATED_PHRASE;
        fprintf (outfile, "\n%-*sPhrase %s%s/%d (%d alternatives) = ",
                 depth*3, " ",
                 optional?"?":negated?"!":"", phrasename((pp&DROP_SPECIAL_BITS) - 512),
                 pp&DROP_SPECIAL_BITS, alts);
        fflush (outfile);
      }

      gp++;                         // gp now points to first element (length) of first alt

      saved_cp = cp; // save current input pointer for backtracking if an alt fails
      saved_ap = ap;

      for (i = 0; i < alts; i++) {
        /* Starting with the root phrase, recursively examine each alternative */
        int each, phrases = gram(gp++), phrase_count, gap = 0;

        cp = saved_cp; // backtrack to where input pointer was when we started parsing the previous alt.
        ap = saved_ap;

        if (ap + 3 > next_free_a)
          next_free_a = ap + 3;
        makespace (A, next_free_a+32);

        A(ap++) = pp;               // record which phrase (could be done outside loop)
        A(ap++) = i;                // and which alt.

        // Count slots needed.  *Could* be precalculated and stored
        // in the grammar, either embedded (after the ALT) or as a
        // separate table

        for (each = 0; each < phrases; each++)
          if (((gram(gp + each)&DROP_SPECIAL_BITS) >= 512) && ((gram(gp + each)&DROP_SPECIAL_BITS) < SEMANTIC))
            gap++;

        A(ap++) = gap;              // Count of alts (gap)
        // ap+gap now points to the slot after the space required, which
        // is where the first subphrase will be stored.
        ap = ap + gap;              // recursive subphrases are stored after this phrase.
        // ap is left updated if successful.

        // successfully parsed phrases are stored in A(saved_ap+3+n)

        if (saved_ap + 3 + gap > next_free_a)
          next_free_a = saved_ap + 3 + gap;
        makespace (A, next_free_a+32);

        if (debug_parser) {    // this loop is only for diagnostics
          char *saved_descp;

          fprintf (outfile, "\n%-*s", depth*3, " ");
          if (phrases == 0) {
            fprintf (outfile, "Alternative %d: (NULL phrase) ", i + 1);
          } else {
            fprintf (outfile, "Alternative %d: (%d phrases) ", i + 1, phrases);
          }
          saved_descp = saved_desc;
          *saved_descp = '\0';
          for (each = 0; each < phrases; each++) {
            int phrase = gram(gp + each)&DROP_SPECIAL_BITS, negated = gram(gp + each)&NEGATED_PHRASE, optional = gram(gp + each)&OPTIONAL_PHRASE;

            if (phrase < 256) {
              saved_descp += sprintf (saved_descp, " '%c'", phrase); // to handle white space, we want to check for it before a *sequence* of chars
            } else if (phrase < 512) {
              saved_descp += sprintf (saved_descp, " \"%s\"/%d", keyword(phrase - 256), phrase - 256); // or a string
            } else if (phrase < 512+MAX_BIP) {
              saved_descp += sprintf (saved_descp, " {%s/BIP%d}", phrasename(phrase - 512), BIP(phrase - 512)); // or a BIP
            } else if ((phrase) < SEMANTIC) {
              saved_descp += sprintf (saved_descp, " <%s%s/%d>", optional?"?":negated?"!":"", phrase==WHITESPACE ? "sp" : phrasename(phrase - 512), phrase);
            } else {
              // SEMANTIC PHRASE
              saved_descp += sprintf (saved_descp, " <%s%s/%d>", optional?"?":negated?"!":"", semantic_code[phrase-SEMANTIC], phrase);
            }
          }
          fprintf (outfile, "%s\n", saved_desc);
          if (strlen(saved_desc) > MAX_DESC) {
            fprintf(stderr, "length of saved_desc is getting excessive.  Recode it to use makespace() (and free())\n");
            exit(3);
          }
          fflush (outfile);
        }

        match = TRUE;               // stays true if all subphrases match
        phrase_count = 0;           // only phrases which make it into the A record,
                                    // i.e. excluding literals and keywords
                                    // (but does include guards and negated phrases)
        for (each = 0; each < phrases; each++) {
          /* Within a single grammar rule (alternative), ensure that each subphrase is present */
          int phrase = gram(gp + each)&DROP_SPECIAL_BITS, negated = gram(gp + each)&NEGATED_PHRASE, optional = gram(gp + each)&OPTIONAL_PHRASE;

          if (debug_parser) {
            fprintf (outfile,
                     "%-*sInput token stream = '%s' '%s' '%s' ...\n", depth*3, " ",
                     (cp < c_nextfree ? &stringpool[c(cp).sIndex] : "EOF"), (cp + 1 < c_nextfree ? &stringpool[c(cp + 1).sIndex] : "EOF"), (cp + 2 < c_nextfree ? &stringpool[c(cp + 2).sIndex]: "EOF"));
          }

          if (cp >= bestparse) {
            static char s[1024];           // off heap instead? - binary size
            int i;
            makespace (bestparse_stack, depth+2);
            for (i = 0; i <= depth; i++) bestparse_stack(i) = parse_stack(i);
            if (phrase < 256) {
              sprintf (s, "<%s%s> '%c'", optional?"?":negated?"!":"", phrasename(pp - 512), phrase);
            } else if (phrase < 512) {
              sprintf (s, "<%s%s> \"%s\"", optional?"?":negated?"!":"", phrasename(pp - 512), keyword(phrase - 256));
            } else if (phrase < 512+MAX_BIP) {
              sprintf (s, "<%s%s> {%s}", optional?"?":negated?"!":"", phrasename(pp - 512), phrasename(phrase - 512));
            } else if ((phrase) < SEMANTIC) {
              sprintf (s, "<%s%s> <%s>", optional?"?":negated?"!":"", phrasename(pp - 512), phrase==WHITESPACE ? "sp" : phrasename(phrase - 512));
            } else {
              // SEMANTIC PHRASE
              sprintf (s, "<%s%s> {%s}", optional?"?":negated?"!":"", phrasename(pp - 512), semantic_code[phrase-SEMANTIC]);
            }

            looking_for = s;
            bestparse = cp;
            bestparse_depth = depth;
          }

          if (debug_parser)
            indent (depth, outfile);

          if (phrase < 256) {
            where = ap;               // next phrase to be parsed will be stored at current 'ap'.
            /* Literal */

            if (debug_parser)
              fprintf (outfile, "'%c'", phrase);

            if ((c(cp).t != B_char) || (*&stringpool[c(cp).sIndex] != phrase)) {
              match = FALSE;
            } else {
    #ifndef ORIGINAL_AREC_FORMAT
              A(ap++) = phrase;
              A(ap++) = 1;
              A(ap++) = 1;
              A(ap++) = cp++;
              A(saved_ap + 3 + phrase_count++) = where;     // Record literal
    #else
              cp++;
              // Don't record literals
    #endif
            }
          } else if (phrase < 512) {  /* Keyword */
            where = ap;               // next phrase to be parsed will be stored at current 'ap'.

            if (debug_parser)
              fprintf (outfile, "\"%s\"/%d", keyword(phrase - 256), phrase - 256);

            if ((c(cp).sIndex == -1) || (strcmp (keyword(phrase - 256), &stringpool[c(cp).sIndex]) != 0)) {
              //^^^^^^^^^^^^^^^^^^
              // loses something in the translation... check...
              match = FALSE;
            } else {
    #ifndef ORIGINAL_AREC_FORMAT
              A(ap++) = phrase;
              A(ap++) = 1;
              A(ap++) = 1;
              A(ap++) = cp++;
              A(saved_ap + 3 + phrase_count++) = where;     // Record keyword
    #else
              cp++;
              // Don't record keywords
    #endif

            }
          } else if (phrase < (512+MAX_BIP)) {  /* Built-in phrase */
            where = ap;               // next phrase to be parsed will be stored at current 'ap'.

            if (debug_parser)
              fprintf (outfile, "{%s/BIP%d}", phrasename(phrase - 512), BIP(phrase - 512));

            if (c(cp).t != BIP(phrase - 512)) {
              match = FALSE;
            } else {
              A(ap++) = phrase;
              A(ap++) = 1;
              A(ap++) = 1;
              A(ap++) = cp++;
              A(saved_ap + 3 + phrase_count++) = where;     // Record BIP
            }
          } else if (phrase==WHITESPACE) {
            if (debug_parser) {
              fprintf (outfile, "Skip whitespace");
            }
            while ((c(cp).t == B_char) && (&stringpool[c(cp).sIndex] != NULL) && (*&stringpool[c(cp).sIndex] == ' ')) {
              //                                                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^   CHECK.  Also, tabs??? folded during line reconstruction???
              cp += 1;
              if (debug_parser) {
                fprintf (outfile, ".");
              }
            }
            match = TRUE;
          } else if (phrase < SEMANTIC) {
            /* Recursive call to parser for a subphrase */
            int here = cp; // we don't advance cp if an optional alt.
            where = ap;               // next phrase to be parsed will be stored at current 'ap'.

            if (debug_parser) {
              fprintf (outfile, "Try next alt: <%s%s/%d>", optional?"?":negated?"!":"", phrasename(phrase - 512), phrase);
            }

            // for now we only apply ? or ! to full P<phrases> ... in principle keywords and BIPs could also
            // take ? or !, but if we add that, need to find *all* the places the change would affect.
            match = parse (phrase, depth + 1); // options have been removed from recursive call. COULD USE 'match' HERE!
            if (negated) match = !match;
            if (match) {
              if (debug_parser) {
                indent(depth, outfile);
                fprintf (outfile, "Alt <%s%s/%d> succeeded", optional?"?":negated?"!":"", phrasename(phrase - 512), phrase);
              }
              if (negated) { // there was no subparse data
                ap = where; // backtrack over anything that might have been added by negated phrase
                // let's create a dummy analysis record entry for the unsuccessful parse
                A(ap++) = phrase|negated;
                A(ap++) = 0; // selected alternative - there was none
                A(ap++) = 0; // number of terms in successful alternative
                A(saved_ap + 3 + phrase_count++) = where;     // Record BIP   (not sure if phrase_count is correct here?)
              } else {
                if (optional) A(where) |= OPTIONAL_PHRASE;
                A(saved_ap + 3 + phrase_count++) = where; // we want the result even if it is a guard phrase.
              }
              if (negated || optional) cp = here; // don't advance text pointer for guard phrase.  However we do store the parse data.
              match = TRUE;
            } else {
              if (debug_parser) {
                indent(depth, outfile);
                fprintf (outfile, "Alt <%s%s/%d> failed", optional?"?":negated?"!":"", phrasename(phrase - 512), phrase);
              }
              match = FALSE;
            }

          } else {
            // SEMANTIC PHRASE
            // we don't store anything for a successful parse, we just use it to accept or reject an alternative
            if (semantic(phrase-SEMANTIC)) {
              if (debug_parser) {
                indent(depth, outfile);
                fprintf (outfile, "Alt <%d:%s> succeeded", phrase, semantic_code[phrase-SEMANTIC]);
              }
              match = TRUE;
            } else {
              if (debug_parser) {
                indent(depth, outfile);
                fprintf (outfile, "Alt <%d:%s> failed", phrase, semantic_code[phrase-SEMANTIC]);
              }
              match = FALSE;
            }
          }

          if (debug_parser) {
            fprintf (outfile, "\n");
            indent (depth, outfile);
            fprintf (outfile, "Tried phrase %d of %s - result was %s\n", each + 1, saved_desc, (match ? "TRUE" : "FALSE"));
            fflush (outfile);
          }

          if (!match)
            break;

        }
        gp += phrases;              // move over all phrases, to next alt

        if (match)
          break;
        else if (debug_parser) {
          indent (depth, outfile);
          fprintf (outfile, "** Alternative %d FAILED.\n", i + 1);
        }
        // gp now points to first element (length) of next alt, or start of next phrase

      }

      return (match);

    }

    int build_ast_inner(int ap, int depth, int caller)
    {
    #define build_ast(ap, depth) build_ast_inner(ap, depth, __LINE__)
      int T[100];

      int saved_ap;
      int phrase;  // A(ap) is the phrase number. A(ap+1) is the alt.
      int alt;     // For consistency, in BIPs, the Alt should always be 1
                   // although that may not be the case at the moment :-(
      int phrases; // defined subphrases

      saved_ap = ap;
      phrase = A(ap++) & DROP_SPECIAL_BITS;
      alt = A(ap++);
      phrases = A(ap++);

      if ((phrase < 512) || (phrase > MAX_PHRASE+512)) {
        fprintf(stderr, "* Internal error: build_ast(%d) called from line %d - phrase = %d\n",
                ap, caller, phrase);
        //exit(1);
        saved_ap = saved_ap/0; // force backtrace
      }

      switch (phrase) {

    // This grammar is loosely based on the grammar from:

    //   https://cs.wmich.edu/~gupta/teaching/cs4850/sumII06/The%20syntax%20of%20C%20in%20Backus-Naur%20form.htm

    // - unfortunately when you get down to using it, there are several errors and omissions in that grammar
    // which had to be fixed.  Also, that grammar is written assuming a left-recursive style of parsing, whereas
    // our parser has to be right-recursive or it will get into infinite loops.

    // A few constructs (especially concerning lvalues vs rvalues) which were handled totally by
    // semantic rules if using the grammar above have been handled using a slightly rewritten
    // syntax in my modification of the grammar.

    // Some of the ambiguities of that grammar were also fixed using the information in:

    //   https://en.cppreference.com/w/c/language/operator_precedence

    // The grammar is extracted from the comments in c-gram.c using grep, and is converted
    // into a set of tables using the program "takeon.c".

    // The code to take the parse tree and construct an AST from it was generated from the
    // grammar, using the utility "create_parser.c".  The first iteration of the parse tree
    // is really a "Concrete Syntax Tree" rather than an Abstract Syntax Tree, i.e. the
    // generated code below initially builds an AST which reflects the raw text that was
    // matched by the parser.  Eventually this code will be modified to produce a more
    // realistic AST where the "A" truly does stand for "Abstract".

    // At some point once the C code in extended-grammar-format.c is modified, it will
    // no longer make sense to rebuild it from the compiled grammar.

    // Some work may be needed to re-order tree branches to ensure left-to-right and
    // right-to-left order of precedence for some of the operations.

    // And here is the entire grammar.  The entry point is phrase "SS" (Source Statement).

    #include INCLUDE
      default:
        fprintf(stderr, "*** UNHANDLED GRAMMAR PHRASE %d\n", phrase);
        exit(EXIT_FAILURE);
      }
    }

    void print_astop_name(int astp) {
      int i, spare, astop, alt, count, data0, data1, optional, negated;

      if (astp < 0) {
        return; // -phrase denotes an optional item we don't want to print when reconstituting the source
                // Not an idea mechanism. May change it later to add optional bits to 'op' field.
      }
      spare = AST(astp);
      astop = AST(astp+1)&DROP_SPECIAL_BITS;
      optional = AST(astp+1)&OPTIONAL_PHRASE;
      negated = AST(astp+1)&NEGATED_PHRASE;
      alt = AST(astp+2);
      count = AST(astp+3);
      data0 = AST(astp+4);
      data1 = AST(astp+5);

      if (astop < MAX_BIP) {
        switch (astop) {
        case B_char: // 0
          if (data0) fprintf(stderr, "'%c' ", data0); else fprintf(stderr, "'\\0' ");
          break;
        case B_identifier: // 1
          fprintf(stderr, "<%s%s=", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
          fprintf(stderr, "%s> ", &stringpool[c(data0).sIndex]);
          break;
        case B_integer_constant: // 2
          fprintf(stderr, "<%s%s=", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
          fprintf(stderr, "%s> ", &stringpool[c(data0).sIndex]);
          break;
        case B_character_constant: // 3
          fprintf(stderr, "<%s%s=", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
          fprintf(stderr, "'%s'> ", &stringpool[c(data0).sIndex]);
          break;
        case B_floating_constant: // 4
          fprintf(stderr, "<%s%s=", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
          fprintf(stderr, "%s> ", &stringpool[c(data0).sIndex]);
          break;
        case B_string: // 5
          fprintf(stderr, "<%s%s=", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
          fprintf(stderr, "\"%s\"> ", &stringpool[c(data0).sIndex]);
          break;
        case B_keyword: // 6
          fprintf(stderr, "\"%s\" ", keyword(data0));
          break;
        case B_pp: // 7
          fprintf(stderr, " ++ ");
          break;
        case B_mm: // 8
          fprintf(stderr, " -- ");
          break;
        case B_andand: // 9
          fprintf(stderr, " && ");
          break;
        case B_oror: // 10
          fprintf(stderr, " || ");
          break;
        case B_eqeq: // 11
          fprintf(stderr, " == ");
          break;
        case B_EOF: // 12
          fprintf(stderr, "<%s%s=", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
          fprintf(stderr, "EOF> ");
          return;
          break;
        default:
          fprintf(stderr, "<%s%s=", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
          fprintf(stderr, "{? %d ?} > ", astop);
        }
      } else {
        //fprintf(stderr, "<phrase %d> ", astop);
        fprintf(stderr, " <%s%s> ", optional?"?":negated?"!":"", astop>512?phrasename(astop-512):phrasename(astop));
      }
    }

    void print_terminal(int astp) {
      int i, spare, astop, alt, count, data0, data1;

      if (astp < 0) {
        return; // -phrase denotes an optional item we don't want to print when reconstituting the source
      }

      spare = AST(astp);
      astop = AST(astp+1);  // &DROP_SPECIAL_BITS???
      alt = AST(astp+2);
      count = AST(astp+3);
      data0 = AST(astp+4);
      data1 = AST(astp+5);

      if (astop < MAX_BIP) {
        switch (astop) {
        case B_char: // 0
          // this is for printing the source again.
          // unfortunately nested literal arrays not formatted very well by gnu indent :-/
          if ((data0 == '{') || (data0 == '}')) fprintf(stdout, "\n");
          fprintf(stdout, "%c", data0);
          if (data0 == '=' || data0 == '-' || data0 == '+' || data0 == '|' || data0 == '&' || data0 == '<' || data0 == '>') fprintf(stdout, " ");
          if ((data0 == ';') || (data0 == '{') || (data0 == '}')) fprintf(stdout, "\n");
          break;
        case B_identifier: // 1
          fprintf(stdout, " %s ", &stringpool[c(data0).sIndex]);
          break;
        case B_integer_constant: // 2
          fprintf(stdout, " %s ", &stringpool[c(data0).sIndex]);
          break;
        case B_character_constant: // 3
          fprintf(stdout, "'%s'", escape(&stringpool[c(data0).sIndex], '\''));
          break;
        case B_floating_constant: // 4
          fprintf(stdout, " %s ", &stringpool[c(data0).sIndex]);
          break;
        case B_string: // 5
          fprintf(stdout, "\"%s\"", escape(&stringpool[c(data0).sIndex], '"'));
          break;
        case B_keyword: // 6
          fprintf(stdout, " %s ", keyword(data0));
          break;
        case B_pp: // 7
          fprintf(stdout, " ++ ");
          break;
        case B_mm: // 8
          fprintf(stdout, " -- ");
          break;
        case B_andand: // 9
          fprintf(stdout, " && ");
          break;
        case B_oror: // 10
          fprintf(stdout, " || ");
          break;
        case B_eqeq: // 11
          fprintf(stdout, " == ");
          break;
        case B_EOF: // 12
          return;
        default:
          fprintf(stdout, "/* %d */ ", astop);
        }
      }
    }

    // NOTE: a variation of the code below could output graphviz/dot format for
    // printing parse trees graphically...
    void print_ast(int astp, int depth) {
      // Although currently the AST looks a lot like the AREC, this will not always be
      // the case, as the AST is simplified...
      int i, astop, alt, count, subphrase;
      // p is the return from mktuple() for proper phrases.
      astop  = AST(astp+1);
      if (astop < MAX_BIP) {
        print_astop_name(astp);
        fprintf(stderr, "\n");
        return;
      }
      alt = AST(astp+2);
      count = AST(astp+3);
      indent(depth, stderr);
      print_astop_name(astp);
      fprintf(stderr, " ::= ");
      for (i = astp+4; i < astp+4+count; i++) {
        if (i >= astp+4) print_astop_name(AST(i));
      }
      fprintf(stderr, ";\n");
      for (i = astp+4; i < astp+4+count; i++) {
        if ((AST(i)>=0) && (AST(AST(i)+1)>=MAX_BIP)) print_ast(AST(i), depth+1);  // MAX_BIP? Not 512+MAX_BIP?  CHECK!!!
      }
    }

    void print_prog(int astp, int depth) {  // IMPORTANT!!!! This works on AST, not AREC.
      int i, count;
      count = AST(astp+3);
      for (i = astp+4; i < astp+4+count; i++) {
        int subphrase_addr = AST(i);
        if ((subphrase_addr >= 0) && (AST(subphrase_addr+1)>=512+MAX_BIP)) {
          print_prog(subphrase_addr, depth+1);
        } else print_terminal(subphrase_addr);
      }
    }

    void dump_concrete_tree(int p, int depth) {
      int i, phrase, alt, phrases, negated, optional;

      static int hack = 0;
      hack++;
      if (((hack > 1) && (p == 0)) || (p == -1)) {
        fprintf(stderr, "!!!!!!!!!!! Hack was needed :-(\n");
        exit(5);
        return;
      }

      i = p;
      phrase = A(p)&DROP_SPECIAL_BITS;
      alt = A(p+1);
      phrases = A(p+2);
      negated = A(p)&NEGATED_PHRASE;
      optional = A(p)&OPTIONAL_PHRASE;

    /* A() is the Analysis record.
         Contents point to a C[] structure when the item is a BIP,
         otherwise the format is [phraseno] [altno] [no-of-subphrases] [subphrases and/or BIPs...]
         for example, if A(25) contained 10, that would mean that the
         token for A(25) was stored in C[10].]
         (And the C string would be at &stringpool[c(A(25))]?)

              A(ap++) = phrase;
              A(ap++) = 1; // altno
              A(ap++) = 1; // #subphrases
              A(ap++) = cp++;
              A(saved_ap + 3 + phrase_count++) = where;     // Record BIP

       */

      if (phrase < 256) {
        indent(depth, stderr);
        fprintf(stderr, "A(%d)=%d char '%s'\n", i, A(i), &stringpool[c(A(i+3)).sIndex]);
      } else if (phrase < 512) {
        indent(depth, stderr);
        fprintf(stderr, "A(%d)=%d keyword \"%s\"\n", i, A(i), &stringpool[c(A(i+3)).sIndex]);
      } else if (phrase < 512+MAX_BIP) {
        indent(depth, stderr);
        fprintf(stderr, "A(%d)=%d B<%s%s>, ", i, A(i), optional?"?":negated?"!":"", phrasename(A(i)-512));
        if (phrases) {
          fprintf(stderr, "choice %d: '%s'\n", A(i+1), &stringpool[c(A(i+3)).sIndex]);
        } else {
          fprintf(stderr, "Null\n");
        }
      } else if (phrase < SEMANTIC) {
        indent(depth, stderr);
        fprintf(stderr, "A(%d)=%d P<%s%s>, ", i, phrase, optional?"?":negated?"!":"", phrase==WHITESPACE ? "sp" : phrasename(phrase-512));
        if (phrases) {
          fprintf(stderr, "choice %d: ", A(i+1));
          for (p = i+3; p < i+3+phrases; p++) {
            fprintf(stderr, "A(%d)=%d [X] ", p, A(p));
          }
          fprintf(stderr, "\n");
          for (p = i+3; p < i+3+phrases; p++) {
            if (A(p) == -1) {
              fprintf(stderr, "/* NOTBUG 0 */");
            } else if (A(p) == P_SS) {
              fprintf(stderr, "/* BUG 1 */");
            } else {
              // do something here with special bits
              if (A(p)&OPTIONAL_PHRASE) fprintf(stderr, "/* BUG 2 */");
              if (A(p)&NEGATED_PHRASE) fprintf(stderr, "/* BUG 3 */");
              if (negated == 0) {
                dump_concrete_tree(A(p)&DROP_SPECIAL_BITS, depth+1);
              }
            }
          }
        } else {
          fprintf(stderr, "Null\n");
        }
      } else {
        // SEMANTIC PHRASE
      }
    }

    int main (int argc, char **argv) {
      int errors, statements, lines;
      int program;
      char *s;
      int SS;

      /* Get clean version of executable name.  Should work on most existing systems (2006) */
      progname = strdup (argv[0]);
      if ((s = strrchr (progname, '/')) != NULL)
        progname = s + 1;           // Unix
      if ((s = strrchr (progname, '\\')) != NULL)
        progname = s + 1;           // M$
      if ((s = strrchr (progname, ']')) != NULL)
        progname = s + 1;           // Dec
      if ((s = strrchr (progname, ';')) != NULL)
        *s = '\0';                  // Version no's
      if (((s = strrchr (progname, '.')) != NULL)
          && (strcasecmp (s, ".exe") == 0))
        *s = '\0';
      if (((s = strrchr (progname, '.')) != NULL)
          && (strcasecmp (s, ".com") == 0))
        *s = '\0';

      /* Handle program arguments */

      if (argc >= 2) {
        if (strcmp(argv[1], "-d") == 0) {
          debug_parser = TRUE;
          argc -= 1; argv += 1; // shift
        }
      }

      if (argc != 2) {
        fprintf (stderr, "syntax: %s [-d] filename\n", progname);
        exit (1);
      }

      if ((sourcefile = fopen (argv[1], "r")) == NULL) {
        fprintf (stderr, "%s: %s - %s\n", progname, strerror (errno), argv[1]);
        exit (errno);
      }

      stringpool = NULL;
      stringpool_arraysize = 0;
      stringpool_nextfree = 0;

      AST = NULL;
      AST_arraysize = 0;
      AST_nextfree = 0;

      parse_stack = NULL;
      parse_stack_arraysize = 0;

      bestparse_stack = NULL;
      bestparse_stack_arraysize = 0;

      outfile = stderr;
      curfile = argv[1];
      startline = TRUE;
      whitespace = TRUE;

      line_reconstruction (sourcefile, argv[1]);       // Effectively, lexing.

      if (debug_parser) {
        int i;                        // DEBUG ONLY
        fprintf (stderr, "\nLexical token stream:\n\n");
        for (i = 0; i < c_nextfree; i++) {
          fprintf (stderr, "C[%d].f=\"%s\", .lineno=%d, .col=%d, .type=%0d, .s=\"%s\" (at %p)\n", i, c(i).f, c(i).lineno_tok, c(i).col_tok, c(i).t, escape(&stringpool[c(i).sIndex],'"'), &stringpool[c(i).sIndex]);
        }
      }

      /* Call the parser */

      errors = 0; statements = 0;
      //for (;;) {   // Use this loop if we are only parsing a basic statement at a time. Don't, if you want to parse the entire file.

        //if (cp == c_nextfree) break; // End of file?
        ap = 0; // reset analysis record on each line. We're not parsing the entire program here.
        if ((SS = parse (P_SS, 0)) != 0) {
          int i;

    #ifdef DEBUG_TOKENS
          fflush(stderr); fflush(stdout);
          fprintf(stderr, "Tokens (line reconstruction and BIPs):\n");
          for (i = 0; i <= c_nextfree; i++) {
            fprintf(stderr, "c(%d).s = \"%s\" t=%d\n", i, escape(&stringpool[c(i).sIndex], '"'), c(i).t);
          }
    #endif

    #ifdef DEBUG_AREC
          fflush(stderr); fflush(stdout);
          fprintf(stderr, "A-record (flat): ap = %d\n", ap&DROP_SPECIAL_BITS);
          i = 0;
          while (i < (ap&DROP_SPECIAL_BITS)) {
            int p, phrase = A(i)&DROP_SPECIAL_BITS, phrases = A(i+2), negated = A(i)&NEGATED_PHRASE, optional = A(i)&OPTIONAL_PHRASE;

            if (phrase < 256) {
              fprintf(stderr, "A(%3d)=%3d char '%s'\n", i, A(i), &stringpool[c(A(i+3)).sIndex]);
              i = i + 3 + phrases;
            } else if (phrase < 512) {
              fprintf(stderr, "A(%3d)=%3d keyword \"%s\"\n", i, A(i), &stringpool[c(A(i+3)).sIndex]);
              i = i + 3 + phrases;
            } else if (phrase < 512+MAX_BIP) {
              fprintf(stderr, "A(%3d)=%3d B<%s%s>, ", i, A(i), optional?"?":negated?"!":"", phrasename(A(i)-512));
              if (phrases) {
                fprintf(stderr, "choice %d: '%s'\n", A(i+1), &stringpool[c(A(i+3)).sIndex]);
              } else {
                fprintf(stderr, "Null\n");
              }
              i = i + 3 + phrases;
            } else if (phrase < SEMANTIC) {
              fprintf(stderr, "A(%3d)=%3d P<%s%s>, ", i, phrase, optional?"?":negated?"!":"", phrase==WHITESPACE ? "sp" : phrasename(phrase-512));
              if (phrases) {
                fprintf(stderr, "choice %d: ", A(i+1));
                //fprintf(stderr, "A(%d)=Phrases %d ", i+2, phrases);
                for (p = i+3; p < i+3+phrases; p++) fprintf(stderr, "A(%d)=%d ", p, A(p));
                fprintf(stderr, "\n");
              } else {
                fprintf(stderr, "Null\n");
              }
              i = i + 3 + phrases;
            } else {
              // SEMANTIC PHRASE
              fprintf(stderr, "@@@ i=%d A(i)=%d @@@\n", i, A(i));
            }
          }
          fflush(stderr); fflush(stdout);

          fprintf(stderr, "Concrete tree:\n");
          dump_concrete_tree(0, 0);
          fprintf(stderr, "\n\n");
          fflush(stderr); fflush(stdout);
    #endif

          statements += 1;

          fprintf(stderr, "Build ast:\n");
          program = build_ast(/*SS*/0, 0);
          fprintf(stderr, "\n\n");
          fflush(stderr); fflush(stdout);

    #ifdef DEBUG_AST
          fprintf(stderr, "Print AST: %d\n", program);
          fflush(stderr); fflush(stdout);
          print_ast(program, 0);
          fprintf(stderr, "\n\n");
          fflush(stderr); fflush(stdout);
    #endif

          fprintf(stderr, "Regenerate source line: %d\n", program);
          fflush(stderr); fflush(stdout);
          print_prog(program, 0);
          fprintf(stderr, "\n\n");
          fflush(stderr); fflush(stdout);

        } else {
          errors += 1;
          //break;
        }
      //}

      if (errors == 0) {
        fprintf(stderr, "%d Lines, %d Statement%s compiled.\n", last_line, statements, statements == 1 ? "" : "s");
        // now do something with it :-)
      } else {
        /* Attempt to print a sensible error if the parse failed */
        if (bestparse == c_nextfree) {
          fprintf (stderr, "\"%s\", Line %d, Col %d: Premature end of file while looking for %s\n",
                   argv[1], c(bestparse).lineno_tok, c(bestparse).col_tok + 1, looking_for);
        } else {
          int i, ic = 0;
          for (i = 0; i < bestparse; i++) {
            if (c(i).lineno_tok == c(bestparse).lineno_tok) {
              while (ic < c(i).col_tok) {
                fprintf (stderr, " ");
                ic += 1;
              }
              //fprintf (stderr, "%s{%d}", c(i).s, c(i).col_tok);
              fprintf (stderr, "%s ", &stringpool[c(i).sIndex]);  // when MINSIZE in makespace is too low, this string is corrupted...
              ic += strlen(&stringpool[c(i).sIndex]);
            }
          }
          fprintf (stderr, " ***ERROR*** ");
          while (c(i).lineno_tok == c(bestparse).lineno_tok) {
            while (ic < c(i).col_tok) {
              fprintf (stderr, " ");
              ic += 1;
            }
            fprintf (stderr, "%s ", &stringpool[c(i).sIndex]);
            ic += strlen(&stringpool[c(i).sIndex]);
            i++;
          }
          fprintf (stderr, "\n\"%s\", Line %d, Col %d: Syntax error while looking for %s\n",
                   argv[1], c(bestparse).lineno_tok, c(bestparse).col_tok + 1, looking_for);
          makespace (bestparse_stack, bestparse_depth+1);
          for (i = 0; i <= bestparse_depth; i++) {
            if (bestparse_stack(i)) {
              fprintf(stderr, "  %s\n", bestparse_stack(i));
            }
          }
        }
        exit (1);
      }
      exit (0);
      return (1);
    }