better (but slower) line counting

README.md

@@ -34,6 +34,7 @@ All funds that are donated to this project will be donated to charity. A full lo
 - get rid of `verbose` flag and introduced `debug` instead
 - the version command now also shows some build variables for more info
 - switched to another pkcs12 library to support p12s generated with openssl3 that use SHA256 HMAC
+- use a more accurate line counter which is also slower as before
 
 ## 3.6
 

libgobuster/helpers.go

@@ -68,7 +68,7 @@ func (set *Set[T]) Stringify() string {
  return strings.Join(values, ",")
 }
 
-func lineCounter(r io.Reader) (int, error) {
+func lineCounter_old(r io.Reader) (int, error) {
  buf := make([]byte, 32*1024)
  count := 1
  lineSep := []byte{'\n'}
@@ -87,6 +87,24 @@ func lineCounter(r io.Reader) (int, error) {
  }
 }
 
+func lineCounter(r io.Reader) (int, error) {
+ scanner := bufio.NewScanner(r)
+ scanner.Split(bufio.ScanLines)
+ var count int
+ for scanner.Scan() {
+ w := scanner.Text()
+ if w == "" || strings.HasPrefix(w, "#") {
+ continue
+ }
+
+ count++
+ }
+ if err := scanner.Err(); err != nil {
+ return -1, err
+ }
+ return count, nil
+}
+
 // DefaultUserAgent returns the default user agent to use in HTTP requests
 func DefaultUserAgent() string {
  return fmt.Sprintf("gobuster/%s", VERSION)

libgobuster/helpers_test.go

@@ -164,9 +164,11 @@ func TestLineCounter(t *testing.T) {
  }{
  {"One Line", "test", 1},
  {"3 Lines", "TestString\nTest\n1234", 3},
- {"Trailing newline", "TestString\nTest\n1234\n", 4},
+ {"Trailing newline", "TestString\nTest\n1234\n", 3},
+ {"Trailing newline with comment", "TestString\n# Test\n1234\n", 2},
  {"3 Lines cr lf", "TestString\r\nTest\r\n1234", 3},
- {"Empty", "", 1},
+ {"3 Lines cr lf with comment", "TestString\r\n# Test\r\n1234", 2},
+ {"Empty", "", 0},
  }
  for _, x := range tt {
  x := x // NOTE: https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables