🎨 (basm bmbuilder) replaced tokenizer

pkg/basm/asmparser.go

@@ -3,33 +3,25 @@ package basm
 import (
 	"errors"
 	"fmt"
-	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/BondMachineHQ/BondMachine/pkg/bmline"
 )
 
-// TODO Horrific and temporary code, a proper parser/lexer is desireable
-func idiotParser(s string) ([]string, int) {
-	// Convert tabs into spaces
-	tab := regexp.MustCompile(`\t`)
-	st := tab.ReplaceAllString(s, " ")
-	// Strip away all duplicates whitspace characters and comments
-	comment := regexp.MustCompile(`;.*`)
-	//space := regexp.MustCompile(`\s+`)
-	//stripped := strings.TrimSpace(space.ReplaceAllString(comment.ReplaceAllString(s, ""), " "))
-	stripped := strings.TrimSpace(comment.ReplaceAllString(st, ""))
-
-	// Splitting the line using spaces
-	splitted := strings.Split(stripped, " ")
-
-	return splitted, len(splitted)
+// tokenizeLine splits a source line into whitespace-separated operands while ignoring comments.
+func tokenizeLine(s string) ([]string, int) {
+	if idx := strings.Index(s, ";"); idx >= 0 {
+		s = s[:idx]
+	}
+
+	tokens := strings.Fields(s)
+	return tokens, len(tokens)
 }
 
 func basmParser(bi *BasmInstance, s string, lineNo uint32) error {
 	line := strconv.Itoa(int(lineNo))
-	argS, argN := idiotParser(s)
+	argS, argN := tokenizeLine(s)
 
 	if bi.debug {
 		fmt.Print("\t" + green(lineNo))

pkg/bmbuilder/builderparser.go

@@ -3,33 +3,25 @@ package bmbuilder
 import (
 	"errors"
 	"fmt"
-	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/BondMachineHQ/BondMachine/pkg/bmline"
 )
 
-// TODO Horrific and temporary code, a proper parser/lexer is desireable
-func idiotParser(s string) ([]string, int) {
-	// Convert tabs into spaces
-	tab := regexp.MustCompile(`\t`)
-	st := tab.ReplaceAllString(s, " ")
-	// Strip away all duplicates whitspace characters and comments
-	comment := regexp.MustCompile(`;.*`)
-	//space := regexp.MustCompile(`\s+`)
-	//stripped := strings.TrimSpace(space.ReplaceAllString(comment.ReplaceAllString(s, ""), " "))
-	stripped := strings.TrimSpace(comment.ReplaceAllString(st, ""))
-
-	// Splitting the line using spaces
-	splitted := strings.Split(stripped, " ")
-
-	return splitted, len(splitted)
+// tokenizeLine splits a source line into whitespace-separated operands while ignoring comments.
+func tokenizeLine(s string) ([]string, int) {
+	if idx := strings.Index(s, ";"); idx >= 0 {
+		s = s[:idx]
+	}
+
+	tokens := strings.Fields(s)
+	return tokens, len(tokens)
 }
 
 func basmParser(bi *BMBuilder, s string, lineNo uint32) error {
 	line := strconv.Itoa(int(lineNo))
-	argS, argN := idiotParser(s)
+	argS, argN := tokenizeLine(s)
 
 	if bi.debug {
 		fmt.Print("\t" + green(lineNo))

pkg/bmbuilder/builderparser_test.go

@@ -0,0 +1,29 @@
+package bmbuilder
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestTokenizeLine(t *testing.T) {
+	tokens, count := tokenizeLine("\tadd\t   r1, r2 ; comment")
+	want := []string{"add", "r1,", "r2"}
+
+	if count != len(want) {
+		t.Fatalf("token count = %d, want %d", count, len(want))
+	}
+
+	if !reflect.DeepEqual(tokens, want) {
+		t.Fatalf("tokens = %v, want %v", tokens, want)
+	}
+}
+
+func TestTokenizeLineEmpty(t *testing.T) {
+	tokens, count := tokenizeLine("   ; comment only")
+	if count != 0 {
+		t.Fatalf("token count = %d, want 0", count)
+	}
+	if len(tokens) != 0 {
+		t.Fatalf("tokens = %v, want empty slice", tokens)
+	}
+}