Skip to content

Commit

Permalink
New BASIC tokenizer and additional tests
Browse files Browse the repository at this point in the history 
Fixes issues stardot#34, stardot#62, stardot#63, stardot#65
  • Loading branch information
@mungre
mungre committed Oct 16, 2022
1 parent 4492a69 commit 21cc7fd
Show file tree
Hide file tree
Showing 27 changed files with 3,575 additions and 910 deletions.
829 changes: 0 additions & 829 deletions src/BASIC.cpp
View file

This file was deleted.

39 changes: 0 additions & 39 deletions src/BASIC.h
View file

This file was deleted.

6 changes: 4 additions & 2 deletions src/VS2010/BeebAsm.vcxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@
<ItemGroup>
<ClCompile Include="..\asmexception.cpp" />
<ClCompile Include="..\assemble.cpp" />
<ClCompile Include="..\BASIC.cpp" />
<ClCompile Include="..\basic_tokens.cpp" />
<ClCompile Include="..\commands.cpp" />
<ClCompile Include="..\discimage.cpp" />
<ClCompile Include="..\expression.cpp" />
Expand All @@ -95,10 +95,11 @@
<ClCompile Include="..\sourcefile.cpp" />
<ClCompile Include="..\stringutils.cpp" />
<ClCompile Include="..\symboltable.cpp" />
<ClCompile Include="..\tokenize.cpp" />
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\asmexception.h" />
<ClInclude Include="..\BASIC.h" />
<ClInclude Include="..\basic_tokens.h" />
<ClInclude Include="..\constants.h" />
<ClInclude Include="..\discimage.h" />
<ClInclude Include="..\globaldata.h" />
Expand All @@ -112,6 +113,7 @@
<ClInclude Include="..\sourcefile.h" />
<ClInclude Include="..\stringutils.h" />
<ClInclude Include="..\symboltable.h" />
<ClInclude Include="..\tokenize.h" />
<ClInclude Include="..\value.h" />
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
Expand Down
18 changes: 12 additions & 6 deletions src/VS2010/BeebAsm.vcxproj.filters
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,6 @@
<ClCompile Include="..\symboltable.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\BASIC.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\macro.cpp">
<Filter>Source Files</Filter>
</ClCompile>
Expand All @@ -66,6 +63,12 @@
<ClCompile Include="..\literals.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\basic_tokens.cpp">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\tokenize.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\asmexception.h">
Expand Down Expand Up @@ -95,9 +98,6 @@
<ClInclude Include="..\symboltable.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\BASIC.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\macro.h">
<Filter>Header Files</Filter>
</ClInclude>
Expand All @@ -116,5 +116,11 @@
<ClInclude Include="..\literals.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\basic_tokens.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="..\tokenize.h">
<Filter>Header Files</Filter>
</ClInclude>
</ItemGroup>
</Project>
141 changes: 141 additions & 0 deletions src/basic_tokens.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
/*************************************************************************************************
basic_tokens.cpp - a table of BBC BASIC keywords
*************************************************************************************************/

#include "basic_tokens.h"

// Return name and length of name
#define NL(N) N, sizeof(N)-1

static const token token_list[] = {
{ NL("AND"), 0x80, 0 },
{ NL("ABS"), 0x94, 0 },
{ NL("ACS"), 0x95, 0 },
{ NL("ADVAL"), 0x96, 0 },
{ NL("ASC"), 0x97, 0 },
{ NL("ASN"), 0x98, 0 },
{ NL("ATN"), 0x99, 0 },
{ NL("AUTO"), 0xC6, token_L_flag },
{ NL("BGET"), 0x9A, token_C_flag },
{ NL("BPUT"), 0xD5, token_M_flag | token_C_flag },
{ NL("COLOUR"), 0xFB, token_M_flag },
{ NL("CALL"), 0xD6, token_M_flag },
{ NL("CHAIN"), 0xD7, token_M_flag },
{ NL("CHR$"), 0xBD, 0 },
{ NL("CLEAR"), 0xD8, token_C_flag },
{ NL("CLOSE"), 0xD9, token_M_flag | token_C_flag },
{ NL("CLG"), 0xDA, token_C_flag },
{ NL("CLS"), 0xDB, token_C_flag },
{ NL("COS"), 0x9B, 0 },
{ NL("COUNT"), 0x9C, token_C_flag },
{ NL("DATA"), 0xDC, token_R_flag },
{ NL("DEG"), 0x9D, 0 },
{ NL("DEF"), 0xDD, 0 },
{ NL("DELETE"), 0xC7, token_L_flag },
{ NL("DIV"), 0x81, 0 },
{ NL("DIM"), 0xDE, token_M_flag },
{ NL("DRAW"), 0xDF, token_M_flag },
{ NL("ENDPROC"), 0xE1, token_C_flag },
{ NL("END"), 0xE0, token_C_flag },
{ NL("ENVELOPE"), 0xE2, token_M_flag },
{ NL("ELSE"), 0x8B, token_L_flag | token_S_flag },
{ NL("EVAL"), 0xA0, 0 },
{ NL("ERL"), 0x9E, token_C_flag },
{ NL("ERROR"), 0x85, token_S_flag },
{ NL("EOF"), 0xC5, token_C_flag },
{ NL("EOR"), 0x82, 0 },
{ NL("ERR"), 0x9F, token_C_flag },
{ NL("EXP"), 0xA1, 0 },
{ NL("EXT"), 0xA2, token_C_flag },
{ NL("FOR"), 0xE3, token_M_flag },
{ NL("FALSE"), 0xA3, token_C_flag },
{ NL("FN"), 0xA4, token_F_flag },
{ NL("GOTO"), 0xE5, token_L_flag | token_M_flag },
{ NL("GET$"), 0xBE, 0 },
{ NL("GET"), 0xA5, 0 },
{ NL("GOSUB"), 0xE4, token_L_flag | token_M_flag },
{ NL("GCOL"), 0xE6, token_M_flag },
{ NL("HIMEM"), 0x93, token_P_flag | token_M_flag | token_C_flag },
{ NL("INPUT"), 0xE8, token_M_flag },
{ NL("IF"), 0xE7, token_M_flag },
{ NL("INKEY$"), 0xBF, 0 },
{ NL("INKEY"), 0xA6, 0 },
{ NL("INT"), 0xA8, 0 },
{ NL("INSTR("), 0xA7, 0 },
{ NL("LIST"), 0xC9, token_L_flag },
{ NL("LINE"), 0x86, 0 },
{ NL("LOAD"), 0xC8, token_M_flag },
{ NL("LOMEM"), 0x92, token_P_flag | token_M_flag | token_C_flag },
{ NL("LOCAL"), 0xEA, token_M_flag },
{ NL("LEFT$("), 0xC0, 0 },
{ NL("LEN"), 0xA9, 0 },
{ NL("LET"), 0xE9, token_S_flag },
{ NL("LOG"), 0xAB, 0 },
{ NL("LN"), 0xAA, 0 },
{ NL("MID$("), 0xC1, 0 },
{ NL("MODE"), 0xEB, token_M_flag },
{ NL("MOD"), 0x83, 0 },
{ NL("MOVE"), 0xEC, token_M_flag },
{ NL("NEXT"), 0xED, token_M_flag },
{ NL("NEW"), 0xCA, token_C_flag },
{ NL("NOT"), 0xAC, 0 },
{ NL("OLD"), 0xCB, token_C_flag },
{ NL("ON"), 0xEE, token_M_flag },
{ NL("OFF"), 0x87, 0 },
{ NL("OR"), 0x84, 0 },
{ NL("OPENIN"), 0x8E, 0 },
{ NL("OPENOUT"), 0xAE, 0 },
{ NL("OPENUP"), 0xAD, 0 },
{ NL("OSCLI"), 0xFF, token_M_flag },
{ NL("PRINT"), 0xF1, token_M_flag },
{ NL("PAGE"), 0x90, token_P_flag | token_M_flag | token_C_flag },
{ NL("PTR"), 0x8F, token_P_flag | token_M_flag | token_C_flag },
{ NL("PI"), 0xAF, token_C_flag },
{ NL("PLOT"), 0xF0, token_M_flag },
{ NL("POINT("), 0xB0, 0 },
{ NL("PROC"), 0xF2, token_F_flag | token_M_flag },
{ NL("POS"), 0xB1, token_C_flag },
{ NL("RETURN"), 0xF8, token_C_flag },
{ NL("REPEAT"), 0xF5, 0 },
{ NL("REPORT"), 0xF6, token_C_flag },
{ NL("READ"), 0xF3, token_M_flag },
{ NL("REM"), 0xF4, token_R_flag },
{ NL("RUN"), 0xF9, token_C_flag },
{ NL("RAD"), 0xB2, 0 },
{ NL("RESTORE"), 0xF7, token_L_flag | token_M_flag },
{ NL("RIGHT$("), 0xC2, 0 },
{ NL("RND"), 0xB3, token_C_flag },
{ NL("RENUMBER"), 0xCC, token_L_flag },
{ NL("STEP"), 0x88, 0 },
{ NL("SAVE"), 0xCD, token_M_flag },
{ NL("SGN"), 0xB4, 0 },
{ NL("SIN"), 0xB5, 0 },
{ NL("SQR"), 0xB6, 0 },
{ NL("SPC"), 0x89, 0 },
{ NL("STR$"), 0xC3, 0 },
{ NL("STRING$("), 0xC4, 0 },
{ NL("SOUND"), 0xD4, token_M_flag },
{ NL("STOP"), 0xFA, token_C_flag },
{ NL("TAN"), 0xB7, 0 },
{ NL("THEN"), 0x8C, token_L_flag | token_S_flag },
{ NL("TO"), 0xB8, 0 },
{ NL("TAB("), 0x8A, 0 },
{ NL("TRACE"), 0xFC, token_L_flag | token_M_flag },
{ NL("TIME"), 0x91, token_P_flag | token_M_flag | token_C_flag },
{ NL("TRUE"), 0xB9, token_C_flag },
{ NL("UNTIL"), 0xFD, token_M_flag },
{ NL("USR"), 0xBA, 0 },
{ NL("VDU"), 0xEF, token_M_flag },
{ NL("VAL"), 0xBB, 0 },
{ NL("VPOS"), 0xBC, token_C_flag },
{ NL("WIDTH"), 0xFE, token_M_flag },
{ NL("PAGE"), 0xD0, 0 },
{ NL("PTR"), 0xCF, 0 },
{ NL("TIME"), 0xD1, 0 },
{ NL("LOMEM"), 0xD2, 0 },
{ NL("HIMEM"), 0xD3, 0 },
};

int token_list_length = ARRAY_LENGTH(token_list);
36 changes: 36 additions & 0 deletions src/basic_tokens.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*************************************************************************************************
basic_tokens.h - a table of BBC BASIC keywords
*************************************************************************************************/

#ifndef _TABLES_H_
#define _TABLES_H_

#define ARRAY_LENGTH(a) (sizeof(a)/sizeof(a[0]))

typedef unsigned char byte;

enum token_flags
{
token_C_flag = 0x01,
token_M_flag = 0x02,
token_S_flag = 0x04,
token_F_flag = 0x08,
token_L_flag = 0x10,
token_R_flag = 0x20,
token_P_flag = 0x40
};

struct token
{
const char* name;
byte length;
byte code;
byte flags;
};

extern const token token_list[];
extern int token_list_length;

#endif // _TABLES_H
40 changes: 18 additions & 22 deletions src/commands.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
#include "sourcefile.h"
#include "asmexception.h"
#include "discimage.h"
#include "BASIC.h"
#include "tokenize.h"
#include "random.h"


Expand Down Expand Up @@ -1625,34 +1625,30 @@ void LineParser::HandlePutBasic()
if ( GlobalData::Instance().IsSecondPass() &&
GlobalData::Instance().UsesDiscImage() )
{
Uint8* buffer = new Uint8[ 0x10000 ];
int fileSize;
bool bSuccess = ImportBASIC( hostFilename.c_str(), buffer, &fileSize );

if (!bSuccess)
FILE* basic_file = fopen(hostFilename.c_str(), "rb");
if (!basic_file)
{
if (GetBASICErrorNum() == 2)
{
AsmException_AssembleError_FileOpen e;
e.SetString( m_line );
e.SetColumn( m_column );
throw e;
}
else
{
std::string message = hostFilename + ": " + GetBASICError();
throw AsmException_UserError( m_line, m_column, message );
}
AsmException_AssembleError_FileOpen e;
e.SetString( m_line );
e.SetColumn( m_column );
throw e;
}
std::vector<unsigned char> tokenized;
TokenizeError err = tokenize_file(basic_file, tokenized);
fclose(basic_file);
if (err.IsError())
{
std::stringstream message;
message << hostFilename << ":" << err.lineNumber << ": " << err.messageText;
throw AsmException_UserError( m_line, m_column, message.str() );
}

// disc image version of the save
GlobalData::Instance().GetDiscImage()->AddFile( beebFilename.c_str(),
reinterpret_cast< unsigned char* >( buffer ),
tokenized.data(),
0xFFFF1900,
0xFFFF8023,
fileSize );

delete [] buffer;
tokenized.size() );
}

}
Expand Down
2 changes: 0 additions & 2 deletions src/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
#include "objectcode.h"
#include "symboltable.h"
#include "discimage.h"
#include "BASIC.h"
#include "macro.h"
#include "random.h"

Expand Down Expand Up @@ -313,7 +312,6 @@ int main( int argc, char* argv[] )

ObjectCode::Create();
MacroTable::Create();
SetupBASICTables();

time_t randomSeed = time( NULL );

Expand Down
Loading

0 comments on commit 21cc7fd

Please sign in to comment.