Skip to content

Commit

Permalink
New BASIC tokenizer
Browse files Browse the repository at this point in the history
Fixes issues #34, #62, #63, #65, #101
  • Loading branch information
mungre committed Aug 23, 2024
1 parent f0a0209 commit 5fdc831
Show file tree
Hide file tree
Showing 12 changed files with 769 additions and 910 deletions.
829 changes: 0 additions & 829 deletions src/BASIC.cpp

This file was deleted.

39 changes: 0 additions & 39 deletions src/BASIC.h

This file was deleted.

6 changes: 4 additions & 2 deletions src/VS2010/BeebAsm.vcxproj
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
<ItemGroup>
<ClCompile Include="..\asmexception.cpp" />
<ClCompile Include="..\assemble.cpp" />
<ClCompile Include="..\BASIC.cpp" />
<ClCompile Include="..\basic_keywords.cpp" />
<ClCompile Include="..\commands.cpp" />
<ClCompile Include="..\discimage.cpp" />
<ClCompile Include="..\expression.cpp" />
Expand All @@ -95,10 +95,11 @@
<ClCompile Include="..\sourcefile.cpp" />
<ClCompile Include="..\stringutils.cpp" />
<ClCompile Include="..\symboltable.cpp" />
<ClCompile Include="..\basic_tokenize.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\asmexception.h" />
<ClInclude Include="..\BASIC.h" />
<ClInclude Include="..\basic_keywords.h" />
<ClInclude Include="..\constants.h" />
<ClInclude Include="..\discimage.h" />
<ClInclude Include="..\globaldata.h" />
Expand All @@ -113,6 +114,7 @@
<ClInclude Include="..\sourcefile.h" />
<ClInclude Include="..\stringutils.h" />
<ClInclude Include="..\symboltable.h" />
<ClInclude Include="..\basic_tokenize.h" />
<ClInclude Include="..\value.h" />
<ClInclude Include="..\version.h" />
</ItemGroup>
Expand Down
18 changes: 12 additions & 6 deletions src/VS2010/BeebAsm.vcxproj.filters
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@
<ClCompile Include="..\symboltable.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\BASIC.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\macro.cpp">
<Filter>Source Files</Filter>
</ClCompile>
Expand All @@ -66,6 +63,12 @@
<ClCompile Include="..\literals.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\basic_keywords.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\basic_tokenize.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\asmexception.h">
Expand Down Expand Up @@ -95,9 +98,6 @@
<ClInclude Include="..\symboltable.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\BASIC.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\macro.h">
<Filter>Header Files</Filter>
</ClInclude>
Expand All @@ -122,6 +122,12 @@
<ClInclude Include="..\version.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\basic_keywords.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\basic_tokenize.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<ResourceCompile Include="..\beebasm.rc">
Expand Down
143 changes: 143 additions & 0 deletions src/basic_keywords.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
/*************************************************************************************************
basic_keywords.cpp - a table of BBC BASIC keywords
*************************************************************************************************/

#include "basic_keywords.h"

// Return name and length of name
#define NL(N) N, sizeof(N)-1

// The order is important because it determines which abbreviations get precedence.
// For example, "P." is PRINT because PRINT is first in the P section.
const keyword keyword_list[] = {
{ NL("AND"), 0x80, 0 },
{ NL("ABS"), 0x94, 0 },
{ NL("ACS"), 0x95, 0 },
{ NL("ADVAL"), 0x96, 0 },
{ NL("ASC"), 0x97, 0 },
{ NL("ASN"), 0x98, 0 },
{ NL("ATN"), 0x99, 0 },
{ NL("AUTO"), 0xC6, kw_L_flag },
{ NL("BGET"), 0x9A, kw_C_flag },
{ NL("BPUT"), 0xD5, kw_M_flag | kw_C_flag },
{ NL("COLOUR"), 0xFB, kw_M_flag },
{ NL("CALL"), 0xD6, kw_M_flag },
{ NL("CHAIN"), 0xD7, kw_M_flag },
{ NL("CHR$"), 0xBD, 0 },
{ NL("CLEAR"), 0xD8, kw_C_flag },
{ NL("CLOSE"), 0xD9, kw_M_flag | kw_C_flag },
{ NL("CLG"), 0xDA, kw_C_flag },
{ NL("CLS"), 0xDB, kw_C_flag },
{ NL("COS"), 0x9B, 0 },
{ NL("COUNT"), 0x9C, kw_C_flag },
{ NL("DATA"), 0xDC, kw_R_flag },
{ NL("DEG"), 0x9D, 0 },
{ NL("DEF"), 0xDD, 0 },
{ NL("DELETE"), 0xC7, kw_L_flag },
{ NL("DIV"), 0x81, 0 },
{ NL("DIM"), 0xDE, kw_M_flag },
{ NL("DRAW"), 0xDF, kw_M_flag },
{ NL("ENDPROC"), 0xE1, kw_C_flag },
{ NL("END"), 0xE0, kw_C_flag },
{ NL("ENVELOPE"), 0xE2, kw_M_flag },
{ NL("ELSE"), 0x8B, kw_L_flag | kw_S_flag },
{ NL("EVAL"), 0xA0, 0 },
{ NL("ERL"), 0x9E, kw_C_flag },
{ NL("ERROR"), 0x85, kw_S_flag },
{ NL("EOF"), 0xC5, kw_C_flag },
{ NL("EOR"), 0x82, 0 },
{ NL("ERR"), 0x9F, kw_C_flag },
{ NL("EXP"), 0xA1, 0 },
{ NL("EXT"), 0xA2, kw_C_flag },
{ NL("FOR"), 0xE3, kw_M_flag },
{ NL("FALSE"), 0xA3, kw_C_flag },
{ NL("FN"), 0xA4, kw_F_flag },
{ NL("GOTO"), 0xE5, kw_L_flag | kw_M_flag },
{ NL("GET$"), 0xBE, 0 },
{ NL("GET"), 0xA5, 0 },
{ NL("GOSUB"), 0xE4, kw_L_flag | kw_M_flag },
{ NL("GCOL"), 0xE6, kw_M_flag },
{ NL("HIMEM"), 0x93, kw_P_flag | kw_M_flag | kw_C_flag },
{ NL("INPUT"), 0xE8, kw_M_flag },
{ NL("IF"), 0xE7, kw_M_flag },
{ NL("INKEY$"), 0xBF, 0 },
{ NL("INKEY"), 0xA6, 0 },
{ NL("INT"), 0xA8, 0 },
{ NL("INSTR("), 0xA7, 0 },
{ NL("LIST"), 0xC9, kw_L_flag },
{ NL("LINE"), 0x86, 0 },
{ NL("LOAD"), 0xC8, kw_M_flag },
{ NL("LOMEM"), 0x92, kw_P_flag | kw_M_flag | kw_C_flag },
{ NL("LOCAL"), 0xEA, kw_M_flag },
{ NL("LEFT$("), 0xC0, 0 },
{ NL("LEN"), 0xA9, 0 },
{ NL("LET"), 0xE9, kw_S_flag },
{ NL("LOG"), 0xAB, 0 },
{ NL("LN"), 0xAA, 0 },
{ NL("MID$("), 0xC1, 0 },
{ NL("MODE"), 0xEB, kw_M_flag },
{ NL("MOD"), 0x83, 0 },
{ NL("MOVE"), 0xEC, kw_M_flag },
{ NL("NEXT"), 0xED, kw_M_flag },
{ NL("NEW"), 0xCA, kw_C_flag },
{ NL("NOT"), 0xAC, 0 },
{ NL("OLD"), 0xCB, kw_C_flag },
{ NL("ON"), 0xEE, kw_M_flag },
{ NL("OFF"), 0x87, 0 },
{ NL("OR"), 0x84, 0 },
{ NL("OPENIN"), 0x8E, 0 },
{ NL("OPENOUT"), 0xAE, 0 },
{ NL("OPENUP"), 0xAD, 0 },
{ NL("OSCLI"), 0xFF, kw_M_flag },
{ NL("PRINT"), 0xF1, kw_M_flag },
{ NL("PAGE"), 0x90, kw_P_flag | kw_M_flag | kw_C_flag },
{ NL("PTR"), 0x8F, kw_P_flag | kw_M_flag | kw_C_flag },
{ NL("PI"), 0xAF, kw_C_flag },
{ NL("PLOT"), 0xF0, kw_M_flag },
{ NL("POINT("), 0xB0, 0 },
{ NL("PROC"), 0xF2, kw_F_flag | kw_M_flag },
{ NL("POS"), 0xB1, kw_C_flag },
{ NL("RETURN"), 0xF8, kw_C_flag },
{ NL("REPEAT"), 0xF5, 0 },
{ NL("REPORT"), 0xF6, kw_C_flag },
{ NL("READ"), 0xF3, kw_M_flag },
{ NL("REM"), 0xF4, kw_R_flag },
{ NL("RUN"), 0xF9, kw_C_flag },
{ NL("RAD"), 0xB2, 0 },
{ NL("RESTORE"), 0xF7, kw_L_flag | kw_M_flag },
{ NL("RIGHT$("), 0xC2, 0 },
{ NL("RND"), 0xB3, kw_C_flag },
{ NL("RENUMBER"), 0xCC, kw_L_flag },
{ NL("STEP"), 0x88, 0 },
{ NL("SAVE"), 0xCD, kw_M_flag },
{ NL("SGN"), 0xB4, 0 },
{ NL("SIN"), 0xB5, 0 },
{ NL("SQR"), 0xB6, 0 },
{ NL("SPC"), 0x89, 0 },
{ NL("STR$"), 0xC3, 0 },
{ NL("STRING$("), 0xC4, 0 },
{ NL("SOUND"), 0xD4, kw_M_flag },
{ NL("STOP"), 0xFA, kw_C_flag },
{ NL("TAN"), 0xB7, 0 },
{ NL("THEN"), 0x8C, kw_L_flag | kw_S_flag },
{ NL("TO"), 0xB8, 0 },
{ NL("TAB("), 0x8A, 0 },
{ NL("TRACE"), 0xFC, kw_L_flag | kw_M_flag },
{ NL("TIME"), 0x91, kw_P_flag | kw_M_flag | kw_C_flag },
{ NL("TRUE"), 0xB9, kw_C_flag },
{ NL("UNTIL"), 0xFD, kw_M_flag },
{ NL("USR"), 0xBA, 0 },
{ NL("VDU"), 0xEF, kw_M_flag },
{ NL("VAL"), 0xBB, 0 },
{ NL("VPOS"), 0xBC, kw_C_flag },
{ NL("WIDTH"), 0xFE, kw_M_flag },
{ NL("PAGE"), 0xD0, 0 },
{ NL("PTR"), 0xCF, 0 },
{ NL("TIME"), 0xD1, 0 },
{ NL("LOMEM"), 0xD2, 0 },
{ NL("HIMEM"), 0xD3, 0 },
};

int keyword_list_length = ARRAY_LENGTH(keyword_list);
36 changes: 36 additions & 0 deletions src/basic_keywords.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*************************************************************************************************
basic_keywords.h - a table of BBC BASIC keywords
*************************************************************************************************/

#ifndef BASIC_KEYWORDS_H_
#define BASIC_KEYWORDS_H_

#define ARRAY_LENGTH(a) (sizeof(a)/sizeof(a[0]))

typedef unsigned char byte;

enum keyword_flags
{
kw_C_flag = 0x01,
kw_M_flag = 0x02,
kw_S_flag = 0x04,
kw_F_flag = 0x08,
kw_L_flag = 0x10,
kw_R_flag = 0x20,
kw_P_flag = 0x40
};

struct keyword
{
const char* name;
byte length;
byte token;
byte flags;
};

extern const keyword keyword_list[];
extern int keyword_list_length;

#endif // BASIC_KEYWORDS_H_
Loading

0 comments on commit 5fdc831

Please sign in to comment.