/*  %Src=tu-Toks.c %Dir=\rlt\c\stuff\  %Date=94/07/21  */

#ifndef TU_TOKS_C
#define TU_TOKS_C

#ifndef XCI_H
#include "..\xci\xci.h"
#endif

/*------------------------------------------*/
   void Init_tu_toks (void)
/*-----------------------------------------------------------
 | Note:  Tok[0] is not normally used in processing.
 |        Either it's a scratch string or a glue-together of
 |        the PREVIOUS Tok[]-array's contents -- supports RE-READ
 +------------------------------------------------------------------*/
{  int   i;

   TOK_quoted_literal_mode = 0;  /* Ie, treat "xxx" no differently */

   for (i = 0 ; i < TOK_ARRAY_DIM_MAX ; i++)
    { Tok [i] = (char *) malloc (TOK_SIZE);
      if (Tok[i] == NULL)
      {
         printf ("*** ERR: malloc (Tok[])\n");
         PAUSE;
         exit (-303);
      }

     *Tok     [i] = '\0';
      TOK_col [i] =    0;

#ifdef  DEBUGGING
      printf ("!<><> Allocating Tok[%d] at %x\n", i, Tok[i]);
#endif
    }

    GG_comment_mode =    0;
   *GG_comment      = '\0';

   Ntoks    = 0;
   GG_tok  = (char *) malloc (TOK_SIZE);
  *GG_tok  = '\0';
   TOK_cur_col  =    0;
   TOK_prev_col =    0;

   Junk_tok = (char *) malloc (STRING_SIZE);  *Junk_tok = '\0';
}



/*----------------------------------------*/
     void Init_toks (void)
/*----------------------------------------*/
/* Normally, you just call Clear_toks ()  */

{   int i, j;
    char *Xd;

    for ( i=1  ; i < TOK_MAX ; i++)
    {
      Xd = Tok[i];
      for (j=0 ; j<TOK_SIZE ; j++) *Xd++ = '\0';
      TOK_col [i] = 0;
    }

   Ntoks        =     0;
   Cur_TOK      =     0;
  *GG_tok       =  '\0';
   TOK_cur_col  =     0;
   TOK_prev_col =     0;
}



/*-------------------------------------------*/
   void Clear_toks (void)
/*-------------------------------------------*/
{
    int    i;
    char *Xd;

/** BEGIN **/

#ifdef DEBUGGING
       printf ("!<><><><>  Clear_toks ()  <><><><><><><><>\n");
       printf ("!    Tok[0..%] of %d chars each.\n", TOK_MAX, TOK_SIZE);
#endif

       strcpy (Tok[0], "");

       for ( i=1 ; i < TOK_MAX ; i++)
         {

         /*------- If we can, Glue_toks together and store in Tok[0] */
            if ((strlen (Tok[0]) + strlen(Tok[i]) +3) < TOK_SIZE)
            {
               strcat (Tok[0], Tok[i]);               /* This allows   */
               if (i < Ntoks) strcat (Tok[0], " ");   /* RE-READ-LAST  */
            }
          /*-------------------------- Clear the i-th Tok[] already ! */
            Xd          = Tok[i];
           *Xd          = '\0';
            TOK_col [i] = 0;
         }

       Ntoks    =    0;
      *GG_tok  = '\0';
       TOK_cur_col  =    0;
       TOK_prev_col =    0;
       Cur_TOK  =    0;
}


/*-------------------------------------------*/
    void Report_TOK_error (char *Msg)
/*-------------------------------------------*/
{  int i;

   printf ("!*** ERR: %s\n", Msg);
   printf ("!***    ");

   for (i = 0;  i <= Ntoks ; i++) printf (" %s",Tok[i]);

   printf ("\n");
}



/*-----------------------*/
   void Show_toks (void)
/*-----------------------*/
{  int   i;

   if (Ntoks == 0)
    {
       printf ("   ** No Toks are stored **\n");
       return;
    }

   printf ("!---------------------  There are %d toks stored.\n", Ntoks);

   for ( i=0 ; i <= Ntoks ; i++)
     printf (" Tok [%2d|@%3d] = '%s'\n", i, TOK_col[i], Tok[i]);

}



/*-----------------------------------------------*/
   char *Search_toks_alligned (char *Xkey)
/*-----------------------------------------------*/
/*  Search the TOK_data area the string MUST be at the start of an tok  */

     {  int   i, X9;
        char *Xj;

#ifdef DEBUGGING
   printf ("\n   *Search_toks_alligned\n");
#endif;
       X9 = 0;
       Xj = Xkey;
       while (*Xj != '\0') X9++;

       for (i = 1 ; i <= Ntoks ; i++)
        {
          if (strncmp (Tok[i], Xkey, X9) == 0)
             {
                return (Tok[i]);
             }
        }
       return (NULL);
     }


/*------------------------------------------*/
   void Upper_toks (void)
/*------------------------------------------*/
/* Note: If the FIRST char of an Tok[] is   */
/*       ' or "  the string is left alone   */

{  int         i;
   char *Xs, *Xj;

   for ( i=1 ; i < TOK_MAX ; i++)
     {  if ((*Tok[i] != '\'') && (*Tok[i] != '\"'))
           {  Xs = Tok[i];
              if (*Xs == '/')
                  {
                     Xj = Xs; Xj++;
                     if (*Xj != '/') Upper (Tok[i]);
                  }
              else Upper (Tok[i]);
           }
     }
}



/*-----------------------------------------------*/
   char *Glue_toks_together (int First_tok_num)
/*-----------------------------------------------*/
{
   char *Xs, *Xd;
   int         i;

/** BEGIN **/

   Xd = Junk_tok;
  *Xd = '\0';

   if (Ntoks <= 0)
     {
       *Xd++ = ' ';
       *Xd   = '\0';
       return Junk_tok;
     }

   for (i=First_tok_num ; i <= Ntoks; i++)
   {
     Xs = Tok[i];
     while (*Xs != '\0') *Xd++ = *Xs++;
    *Xd++ = ' ';
   }

   Xd--;     /* Remove extra last space */
  *Xd = '\0';

   return Junk_tok;
}



/*---------------------------*/
    short Tok_eq  (char *Xs)
/*---------------------------*/
{

/** BEGIN **/

   if (strcmp (Xs, Tok[Cur_TOK]) == 0) return 1;
   el                                  return 0;
}

/*---------------------------*/
    short Tok_EQ  (char *Xs)
/*---------------------------*/
{

/** BEGIN **/
   strcpy (Junk8, Xs);              Upper (Junk8);
   strcpy (Junk9, Tok[Cur_TOK]);    Upper (Junk9);

   if (strcmp (Junk8, Junk9) == 0) return 1;
   el                              return 0;
}



/*---------------------------*/
    short Tok_is_int (void)
/*---------------------------*/
{
  char            *Xs;

/** BEGIN **/

  Xs = Tok[Cur_TOK];

  if ((*Xs == '+') || (*Xs == '-'))
  {
      Xs++;
      if (*Xs == '\0') return -TOK_IS_PLUS_OR_MINUS_SIGN;  /* Error */
  }

  while (*Xs != '\0')
  {
     if (strchr ("0123456789", *Xs) == NULL) return NOT_OK;
     Xs++;
  }
  return OK;
}



/*-------------------------------*/
    short Tok_is_var_name (void)
/*-------------------------------*/
{
  char            *Xs;

/** BEGIN **/

  Xs = Tok[Cur_TOK];

  if (! (   (  (*Xs >= 'A') && (*Xs <= 'Z')  )
         || (  (*Xs >= 'a') && (*Xs <= 'z')  )
        )
     )    return 0;

  /* should check for C-keywords etc */

  return 1;
}



/*---------------------------*/
    short Skip_tok  (void)
/*---------------------------*/
{

/** BEGIN **/
  Cur_TOK++;
  if ((Cur_TOK > Ntoks) || (Ntoks == 0))
  {
     do
     {   /* VERY SIMPLISTICALLY thinks                                 */
         /* that NO comment can continue across a newline boundary !!! */

        GG_len = Pull_src_line (GG_xs);

        if (GG_len < 0) return EOF_FOUND;

        Trim (GG_xs);  GG_len = Length (GG_xs);

        Pull_toks (GG_xs);

     } while (Ntoks < 1);
     Cur_TOK = 1;
  }
  return OK;
}

/*---------------------------*/
    short Skip_C_tok  (void)
/*---------------------------*/
{

/** BEGIN **/
  Cur_TOK++;
  if ((Cur_TOK > Ntoks) || (Ntoks == 0))
  {
     do
     {   /* VERY SIMPLISTICALLY thinks                                 */
         /* that NO comment can continue across a newline boundary !!! */

        GG_len = Pull_src_line (GG_xs);

        if (GG_len < 0) return EOF_FOUND;

        Trim (GG_xs);  GG_len = Length (GG_xs);

        Pull_C_toks (GG_xs);

     } while (Ntoks < 1);
     Cur_TOK = 1;
  }
  return OK;
}



  /*-------------------------------------------*/
     void Pull_toks_as_args (char *Xs)
  /*-------------------------------------------*/
{
   char  *Xd;     /* Move-along pointer -> GG_tok (at init time */
                  /* as well as after saving previous tok        */

   char   Quote_char;

#ifdef DEBUGGING
    printf ("\n<><><><> Pull_toks_as_args ('%s') <><><>\n",Xs);
#endif
   Clear_toks ();

   Xd = GG_tok;   /* The macro SAVE_TOK uses Xd and these two steps are */
  *Xd = '\0';      /* performed in SAVE_TOK automatically when called    */

   TOK_cur_col  = 1;
   TOK_prev_col = 1;
   Cur_TOK  = 0;

   Quote_char = '\0';

   while ((*Xs != '\0') && (*Xs != '\04'))
    {
      if ( (TOK_quoted_literal_mode == 1) &&
          ((*Xs == '\'') || (*Xs == '\"'))) /* Are we in literal mode?? */
          {
            if (*GG_tok != '\0')  SAVE_TOK;

            Quote_char = *Xs;   *Xd++ = *Xs++;   TOK_cur_col++;

            while ((*Xs != '\0') && (*Xs != '\04') && (*Xs != Quote_char))
                 { if (*Xs == '\t')
                        CONVERT_TABS_XD_XS;
                   else { *Xd++ = *Xs++; TOK_cur_col++; }
                 }
           *Xd++ = *Xs++; TOK_cur_col++;
            SAVE_TOK;
            Quote_char = '\0';
          }

      else if ((*Xs == ' ') || (*Xs == '\t'))
               {
                 if (*GG_tok != '\0') SAVE_TOK;

                 while ((*Xs == ' ') || (*Xs == '\t'))
                    {  if (*Xs == '\t')
                            TOK_cur_col+=TAB_SETTING;
                       else TOK_cur_col++;
                       Xs++;
                    }
                 TOK_prev_col = TOK_cur_col;  /* Encountered a non-blank */
               }

      else { *Xd++ = *Xs++;  TOK_cur_col++; }

    }  /* end of while loop */

  if (*GG_tok != '\0') SAVE_TOK;

  Cur_TOK  = 1;

}



/*--------------------------------------------------*/
   void Pull_toks (char *Xs)
/*--------------------------------------------------*/
/* This is a sort of VANILLA puller                 */
/* See one of the ones after it for other varieties */
/* ESPECIALLY for:  Pull_C_toks()                   */
{
    char  *Xd;     /* Move-along pointer -> GG_tok (at init time */
                   /* as well as after saving previous tok        */
    char  *Xj;
    char   Quote_char;

    Clear_toks ();

    Xd = GG_tok;   /* The macro SAVE_TOK uses Xd and these two steps are */
   *Xd = '\0';      /* performed in SAVE_TOK automatically when called    */

    TOK_cur_col  = 1;
    TOK_prev_col = 1;
    Cur_TOK  = 0;

    Quote_char = '\0';

    while ((*Xs != '\0') && (*Xs != '\04'))
     {
       if ((*Xs == '\'') || (*Xs == '\"'))    /* Are we in literal mode?? */
           {
             if (*GG_tok != '\0') SAVE_TOK;

             Quote_char = *Xs;   *Xd++ = *Xs++;   TOK_cur_col++;

             while ((*Xs != '\0') && (*Xs != '\04') && (*Xs != Quote_char))
                  { if (*Xs == '\t')
                         CONVERT_TABS_XD_XS;
                    else { *Xd++ = *Xs++; TOK_cur_col++; }
                  }
            *Xd++ = *Xs++; TOK_cur_col++;
             SAVE_TOK;
             Quote_char = '\0';
           }

   else if (strchr ("{}[]:,&|%^();~+-*\\/=!", *Xs) != NULL)  /* sp char       */
           {  if (*GG_tok != '\0') SAVE_TOK ;                 /* Rmvd $#@ rlt */

              *Xd++ = *Xs++;    /* Save the char as a sep-token */
               TOK_cur_col++;

               SAVE_TOK;
            }

    else if ((*Xs == ' ') || (*Xs == '\t'))
            {
              if (*GG_tok != '\0') SAVE_TOK;

              while ((*Xs == ' ') || (*Xs == '\t'))
                 {  if (*Xs == '\t')
                         TOK_cur_col+=TAB_SETTING;
                    else TOK_cur_col++;
                    Xs++;
                 }
              TOK_prev_col = TOK_cur_col;  /* Encountered a non-blank */
            }

   else     { *Xd++ = *Xs++;  TOK_cur_col++; }

     }  /* end of while loop */

   if (*GG_tok != '\0') SAVE_TOK;
   if (GG_convert_toks_to_upper_by_default != 0)
      Upper_toks ();

   Cur_TOK  = 1;

}



/*--------------------------------------------------*/
   void Pull_BAS_toks (char *Xs)
/*--------------------------------------------------*/
/* This is a sort of VANILLA puller                 */
/* See one of the ones after it for other varieties */
/* ESPECIALLY for:  Pull_C_toks()                   */
{
    char  *Xd;     /* Move-along pointer -> GG_tok (at init time */
                   /* as well as after saving previous tok        */
    char  *Xj;
    char   Quote_char;

    Clear_toks ();

    Xd = GG_tok;   /* The macro SAVE_TOK uses Xd and these two steps are */
   *Xd = '\0';      /* performed in SAVE_TOK automatically when called    */

    TOK_cur_col  = 1;
    TOK_prev_col = 1;
    Cur_TOK  = 0;

    Quote_char = '\0';

    while ((*Xs != '\0') && (*Xs != '\04'))
     {
       if ((*Xs == '\'') || (*Xs == '\"'))    /* Are we in literal mode?? */
           {
             if (*GG_tok != '\0') SAVE_TOK;

             Quote_char = *Xs;   *Xd++ = *Xs++;   TOK_cur_col++;

             while ((*Xs != '\0') && (*Xs != '\04') && (*Xs != Quote_char))
                  { if (*Xs == '\t')
                         CONVERT_TABS_XD_XS;
                    else { *Xd++ = *Xs++; TOK_cur_col++; }
                  }
            *Xd++ = *Xs++; TOK_cur_col++;
             SAVE_TOK;
             Quote_char = '\0';
           }

   else if (strchr ("!", *Xs) != NULL)   /* check for  ! comment and != */
   {
      if (*GG_tok != '\0') SAVE_TOK;
      *Xd++ = *Xs++;
      if (*Xs == '=')
      {
         *Xd++ = *Xs++;
          SAVE_TOK;
      }
      el
      {
        /*-- Comment to end of line ---*/
          while (*Xs != '\0') *Xd++ = *Xs++;
          SAVE_TOK;
      }
   }

   else if (strchr ("><=/", *Xs) != NULL)  /*  =  ==  <  > <=  >=  <<<  >>>  */
   {                                       /*     //option                   */
      if (*GG_tok != '\0') SAVE_TOK;
      *Xd++ = *Xs++;
      while (strchr ("><=/", *Xs) != NULL)  *Xd++ = *Xs++;
      SAVE_TOK;
   }

   else if (strchr ("{}[]:,&|%^();~+-*\\", *Xs) != NULL)  /* sp char       */
           {  if (*GG_tok != '\0') SAVE_TOK ;                 /* Rmvd $#@ rlt */

              *Xd++ = *Xs++;    /* Save the char as a sep-token */
               TOK_cur_col++;

               SAVE_TOK;
            }

    else if ((*Xs == ' ') || (*Xs == '\t'))
            {
              if (*GG_tok != '\0') SAVE_TOK;

              while ((*Xs == ' ') || (*Xs == '\t'))
                 {  if (*Xs == '\t')
                         TOK_cur_col+=TAB_SETTING;
                    else TOK_cur_col++;
                    Xs++;
                 }
              TOK_prev_col = TOK_cur_col;  /* Encountered a non-blank */
            }

   else     { *Xd++ = *Xs++;  TOK_cur_col++; }

     }  /* end of while loop */

   if (*GG_tok != '\0') SAVE_TOK;
   if (GG_convert_toks_to_upper_by_default != 0)
      Upper_toks ();

   Cur_TOK  = 1;

}



/*--------------------------------------------------*/
   void Pull_C_toks (char *Xs)
/*--------------------------------------------------*/
/* This is a sort of VANILLA puller                 */
/* See one of the ones after it for other varieties */
{
    char  *Xd;     /* Move-along pointer -> GG_tok (at init time  */
                   /* as well as after saving previous tok        */
    char  *Xc;     /* Used for COMMENT macros                     */
    char  *Xj;
    char   Quote_char;

    Clear_toks ();

    Xd = GG_tok;   /* The macro SAVE_TOK uses Xd and these two steps are */
   *Xd = '\0';      /* performed in SAVE_TOK automatically when called    */

    TOK_cur_col  = 1;
    TOK_prev_col = 1;
    Cur_TOK  = 0;

    Quote_char = '\0';

    while ((*Xs != '\0') && (*Xs != '\04'))
     {
       if ((*Xs == '\'') || (*Xs == '\"'))    /* Are we in literal mode?? */
           {
             if (*GG_tok != '\0') SAVE_TOK;

             Quote_char = *Xs;   *Xd++ = *Xs++;   TOK_cur_col++;

             while ((*Xs != '\0') && (*Xs != '\04') && (*Xs != Quote_char))
                  { if (*Xs == '\t')
                         CONVERT_TABS_XD_XS;
                    else { *Xd++ = *Xs++; TOK_cur_col++; }
                  }
            *Xd++ = *Xs++; TOK_cur_col++;
             SAVE_TOK;
             Quote_char = '\0';
           }

   else if (strchr ("{}[]:,&|@#$%^();~+-*\\/=!", *Xs) != NULL)  /* sp char */
           {  if (*GG_tok != '\0') SAVE_TOK ;

              /*--- check for // or /* as comments ------*/
               Xj = Xs; Xj++;
               if (    (*Xs == '/')
                    && ((*Xj == '/') || (*Xj == '*'))
                  )
                      EXTRACT_GG_COMMENT;
               el    /*---- this must just be a unary operator (eg, '/' ) */
                  {
                     *Xd++ = *Xs++;    /* Save the char as a sep-token */
                      TOK_cur_col++;
                      SAVE_TOK;
                  }
            }

    else if ((*Xs == ' ') || (*Xs == '\t'))
            {
              if (*GG_tok != '\0') SAVE_TOK;

              while ((*Xs == ' ') || (*Xs == '\t'))
                 {  if (*Xs == '\t')
                         TOK_cur_col+=TAB_SETTING;
                    else TOK_cur_col++;
                    Xs++;
                 }
              TOK_prev_col = TOK_cur_col;  /* Encountered a non-blank */
            }

   else     { *Xd++ = *Xs++;  TOK_cur_col++; }

     }  /* end of while loop */

   if (*GG_tok != '\0') SAVE_TOK;
   if (GG_convert_toks_to_upper_by_default != 0)
      Upper_toks ();

   Cur_TOK  = 1;

}



/*----------------------------------------------------------------*/
   void Pull_pdl_toks (char *Xs)
/*----------------------------------------------------------------*/
/* In this case the DECIMAL point is NOT stripped if in a numeric */
/* The ! indicates the start of a COMMENT-to-END-of-line          */

{
    char  *Xd;     /* Move-along pointer -> GG_tok (at init time */
                   /* as well as after saving previous tok        */
    char  *Xj;
    char   Quote_char, Xchar;

    Clear_toks ();

    Xd = GG_tok;   /* The macro SAVE_TOK uses Xd and these two steps are */
   *Xd = '\0';      /* performed in SAVE_TOK automatically when called    */

    TOK_cur_col  = 1;
    TOK_prev_col = 1;
    Cur_TOK  = 0;


    Quote_char = '\0';

    while ((*Xs != '\0') && (*Xs != '\04'))
     {
       if ((*Xs == '\'') || (*Xs == '\"'))    /* Are we in literal mode?? */
           {
             if (*GG_tok != '\0') SAVE_TOK;

             Quote_char = *Xs;   *Xd++ = *Xs++;   TOK_cur_col++;

             while ((*Xs != '\0') && (*Xs != '\04') && (*Xs != Quote_char))
                  { if (*Xs == '\t')
                         CONVERT_TABS_XD_XS;
                    else { *Xd++ = *Xs++; TOK_cur_col++; }
                  }
            *Xd++ = *Xs++; TOK_cur_col++;
             SAVE_TOK;
             Quote_char = '\0';
           }

   /*  (more else ifs on the next page .../...)    */



  /*--- The following elseif decodes the DECIMAL POINT (dot)  ---------------*/
  /*    Note that similar treatement should be put here if you want things   */
  /*    like 1,344  to be stored as a NUMBER -- right now, the "," would     */
  /*    cause there to be THREE tokens:  1  ,  344   The code below handles  */
  /*    things like  0.134  .14  but NOT 1.3E-5  etc                         */
  /*    This checks JUST the first char of the token stored so far (GG_tok  */

   else if (*Xs == '.')    /* Either   x.y  or   1.34 */
       { if (     (strchr ("0123456789", *GG_tok) != NULL)   /* eg 1.45    */
              ||  (Xd == GG_tok)                             /* eg  .123   */
            )
           {  *Xd++ = *Xs++;  /* Append the decimal point to the number */
           }
         else {  if (*GG_tok != '\0') SAVE_TOK ;
                *Xd++ = *Xs++;               /* Save the "." as a sep-token */
                 TOK_cur_col++;
                 SAVE_TOK;
              }
       }


  /* Note that $ is treated like ;   So that $0a (hex) becomes TWO toks $ 0a */

   else if (strchr ("{}[]:,&|@#$%^();~+-*/=", *Xs) != NULL)  /* sp char */
           {  if (*GG_tok != '\0') SAVE_TOK ;

              *Xd++ = *Xs++;    /* Save the char as a sep-token */
               TOK_cur_col++;

               SAVE_TOK;
            }


  /*-- Just to illustate what is involved, I will code the \x processor ---*/
  /*   Otherwise it should be coded up there with the other special chars  */
  /*   Note that EACH digraph is stored as a SEP token                     */

   else if (*Xs == '\\')
       { if (*GG_tok != '\0') SAVE_TOK ;

        *Xd++ = *Xs++;    /* copy the first \  */

         Xchar = *Xs++;
        *Xd++  = Xchar;    /* And copy the 2nd (very next) char too */

         if ((Xchar == 'x') || (Xchar == 'X'))    /*  \xb3fc  (hex) */
           {
                   while (strchr ("0123456789ABCDEFabcdef", *Xs) != NULL)
                         *Xd++ = *Xs++;
                   SAVE_TOK;
           }
         else if (strchr ("0123456789", Xchar) != NULL)  /* octal/dec */
                {
                   while (strchr ("0123456789", *Xs) != NULL)
                         *Xd++ = *Xs++;
                   SAVE_TOK;
                }
         else  /* should be just a digraph  \t \\ \r \n  etc */
             { SAVE_TOK;   /* We have already stored the 2nd char */
             }
       }


  /* Finally the comment-to-end of line.  Actually The ONLY difference here */
  /* between !xxxx and //xxx (c++) is the char we look at and then the fact */
  /* That a single / is just a divides sign.                                */

   else if (*Xs == '!')                         /* ! Comment to end of line */
            { if (*GG_tok != '\0') SAVE_TOK;
              *Xd++ = *Xs++;   /* Store the ! first */
               TOK_cur_col++;
               while ((*Xs != '\0') && (*Xs != '\04'))
                     *Xd++ = *Xs++;
               SAVE_TOK;
            }

    else if ((*Xs == ' ') || (*Xs == '\t'))
            {
              if (*GG_tok != '\0') SAVE_TOK;

              while ((*Xs == ' ') || (*Xs == '\t'))   /* Skip blanks & tabs */
                 {  if (*Xs == '\t')
                         TOK_cur_col+=TAB_SETTING;     /* But keep track of COL */
                    else TOK_cur_col++;
                    Xs++;
                 }
              TOK_prev_col = TOK_cur_col;  /* Encountered a non-blank */
            }

   else     { *Xd++ = *Xs++;  TOK_cur_col++; }

 }  /* end of while loop */

   if (*GG_tok != '\0') SAVE_TOK;
   if (GG_convert_toks_to_upper_by_default != 0)
      Upper_toks ();

   Cur_TOK  = 1;
}



/*--------------------------------------------------*/
   void Shuffle_toks_up (short Tok_num)
/*---------------------------------------------------------
 | This proc PHYSICALLY shuffles the toks, up; viz:
 |
 |       Tok[N] -> Tok[1],  Tok[N+1] -> Tok[2] etc
 |
 | Alternatively, you could just use:  Tok [Cur_TOK];
 +--------------------------------------------------------*/
{
   short   i, i_src, i_dest, Last_tok_to_mod;
   char                                  *Xd;

/** BEGIN **/

   if (Tok_num <= 0)  return;                  /** Shuffle nothing  RETURN **/

   if (Tok_num >= Ntoks)    /* Shuffle them all */
   {
      Clear_toks();
      return;                                     /** Ok do all  RETURN **/
   }

   Last_tok_to_mod = Ntoks - Tok_num;

   i_dest = 1;
   i_src  = Tok_num+1;    /*   +1  since we are to move down  BY  Tok_num  */

   for (i=1 ; i <= Last_tok_to_mod ; i++)
   {
       strcpy (Tok[i_dest], Tok[i_src]);
       i_src++;
       i_dest++;
    }

/*--------------------------------------------  Zero out remaining ones  ----*/
    for (i= (Last_tok_to_mod +1) ; i < MAX_TOKS ; i++)
    {
       Xd = Tok [i];
      *Xd = '\0';
    }
    Ntoks -= Tok_num;
}



/*--------------------------------------------------*/
   void Shuffle_gap_into_toks_at (short Tok_num)
/*---------------------------------------------------------
 | This proc PHYSICALLY shuffles the toks, up; viz:
 |
 |       Tok[N] -> Tok[N+1],  Tok[N+1] -> Tok[N+2] etc
 |       Note that Tok[N] is left intact.
 +--------------------------------------------------------*/
{
   short  i;

/** BEGIN **/

   if (   (Tok_num <= 0)
       || (Tok_num >= MAX_TOKS))  return;     /** no action RETURN **/

   for (i=Ntoks; i >= Tok_num; i--)    /* copy backwards */
       strcpy (Tok[i+1], Tok[i]);

}

#endif   /* ifndef TU_TOKS_C  */