Title: Implementation of a program which identifies the tokens

Theory: token or lexeme identification is a process of identifying the nature of the word
which are in an input string. It point out which word is keyword which is identifier and similarly
for other operation used in a string.

Requirements: A computer system with Borland or any other C compiler installed.

Code:
The C code for token generator is

#include<stdio.h>
#include<conio.h>
#include<string.h>
#include<ctype.h>
void ts(void);
void pt(void);
char str[50],st[50][50],at[50][50];
int f1=0;
void main()
{
clrscr();
printf("\nEnter the source program : ");
gets(str);
ts();
pt();
getch();
}
void ts(void)
{
int isk,l,i=0,j=0;
char *kw[]={"void","int","main","char","float","printf"};
char *op[]={"+","_","*","/","^","<",">","=","!="};
char pun[]="{}()[];:";
char token[40];
int k;
l=strlen(str);
while(i<l)
 {
if(isalpha(str[i]))
{
while(isalpha(str[i])||isdigit(str[i]))
{
token[j++]=str[i];
i++;
}
token[j++]='\0';
strcpy(st[f1],token);
isk=0;
for(k=0;k<6;k++)
{
if(strcmp(kw[k],token)==0)
{
isk=1;
strcpy(at[f1++],"keyword");
break;
}
}
if(isk==0)
strcpy(at[f1++],"Identifier");
j=0;
}
if(str[i]=='"')
{ //while(str[++i]!='=')
token[j++]=str[i];
token[j]='\0';
strcpy(st[f1],token);
strcpy(at[f1++],"liternal");
}
j=0;
if(isdigit(str[i]))
{
while(isdigit(str[i]||str[1])==',')
token[j++]=str[i++];
token[j]='\0';
strcpy(st[f1],token);
strcpy(at[f1++],"constant"); }
j=0;
token[j++]=str[i];
 token[j++]='\0';
for(k=0;k<11;k++)
{
if(strcmp(op[k],token)==0)
{
strcpy(st[f1],token);
strcpy(at[f1++],"Operator");
break;
}
}
for(k=0;k<8;k++)
{
if(pun[k]==str[i])
{
strcpy(st[f1],token);
strcpy(at[f1++],"Punctuation");
break;
}
}
j=0;
i++;
}
}
void pt(void)
{
int i;
for(i=0;i<f1;i++)
printf("\n%s\t%s",st[i],at[i]);
}
Output:

The above output describes that when the user enter the input string the program divide it into
tokens and also mention their type like word ‘’void’’ in the string is a Keyword.