Refactor vuls2 db fetch

config/config.go

@@ -47,6 +47,7 @@ type Config struct {
 	Metasploit MetasploitConf `json:"metasploit,omitempty"`
 	KEVuln     KEVulnConf     `json:"kevuln,omitempty"`
 	Cti        CtiConf        `json:"cti,omitempty"`
+	Vuls2      Vuls2DictConf  `json:"vuls2Dict,omitempty"`
 
 	Slack      SlackConf      `json:"-"`
 	EMail      SMTPConf       `json:"-"`

config/vulnDictConf.go

@@ -328,3 +328,9 @@ func (cnf *CtiConf) Init() {
 	cnf.setDefault("go-cti.sqlite3")
 	cnf.DebugSQL = Conf.DebugSQL
 }
+
+type Vuls2DictConf struct {
+	Repository string
+	Path       string
+	SkipUpdate bool
+}

detector/detector.go

@@ -15,6 +15,7 @@ import (
 	"github.com/future-architect/vuls/constant"
 	"github.com/future-architect/vuls/contrib/owasp-dependency-check/parser"
 	"github.com/future-architect/vuls/cwe"
+	"github.com/future-architect/vuls/detector/vuls2"
 	"github.com/future-architect/vuls/gost"
 	"github.com/future-architect/vuls/logging"
 	"github.com/future-architect/vuls/models"
@@ -49,7 +50,7 @@ func Detect(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
 			return nil, xerrors.Errorf("Failed to fill with Library dependency: %w", err)
 		}
 
-		if err := DetectPkgCves(&r, config.Conf.OvalDict, config.Conf.Gost, config.Conf.LogOpts); err != nil {
+		if err := DetectPkgCves(&r, config.Conf.OvalDict, config.Conf.Gost, config.Conf.Vuls2, config.Conf.LogOpts, config.Conf.NoProgress); err != nil {
 			return nil, xerrors.Errorf("Failed to detect Pkg CVE: %w", err)
 		}
 
@@ -317,14 +318,19 @@ func Detect(rs []models.ScanResult, dir string) ([]models.ScanResult, error) {
 
 // DetectPkgCves detects OS pkg cves
 // pass 2 configs
-func DetectPkgCves(r *models.ScanResult, ovalCnf config.GovalDictConf, gostCnf config.GostConf, logOpts logging.LogOpts) error {
+func DetectPkgCves(r *models.ScanResult, ovalCnf config.GovalDictConf, gostCnf config.GostConf, vuls2Cnf config.Vuls2DictConf, logOpts logging.LogOpts, noProgress bool) error {
 	// Pkg Scan
 	if isPkgCvesDetactable(r) {
 		// OVAL, gost(Debian Security Tracker) does not support Package for Raspbian, so skip it.
 		if r.Family == constant.Raspbian {
 			r = r.RemoveRaspbianPackFromResult()
 		}
 
+		// Vuls2
+		if err := vuls2.Detect(r, vuls2Cnf, noProgress); err != nil {
+			return xerrors.Errorf("Failed to detect CVE with Vuls2: %w", err)
+		}
+
 		// OVAL
 		if err := detectPkgsCvesWithOval(ovalCnf, r, logOpts); err != nil {
 			return xerrors.Errorf("Failed to detect CVE with OVAL: %w", err)
@@ -537,6 +543,9 @@ func detectPkgsCvesWithOval(cnf config.GovalDictConf, r *models.ScanResult, logO
 		logging.Log.Infof("Skip OVAL and Scan with gost alone.")
 		logging.Log.Infof("%s: %d CVEs are detected with OVAL", r.FormatServerName(), 0)
 		return nil
+	case constant.RedHat, constant.CentOS, constant.Alma, constant.Rocky:
+		logging.Log.Debugf("Skip OVAL and Scan by Vuls2")
+		return nil
 	case constant.Windows, constant.MacOSX, constant.MacOSXServer, constant.MacOS, constant.MacOSServer, constant.FreeBSD, constant.ServerTypePseudo:
 		return nil
 	default:

detector/vuls2/db.go

@@ -0,0 +1,97 @@
+package vuls2
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/future-architect/vuls/config"
+	"github.com/future-architect/vuls/logging"
+	"github.com/pkg/errors"
+	"golang.org/x/xerrors"
+
+	db "github.com/MaineK00n/vuls2/pkg/db/common"
+	"github.com/MaineK00n/vuls2/pkg/db/fetch"
+)
+
+var (
+	// DefaultGHCRRepository is GitHub Container Registry for vuls2 db
+	DefaultGHCRRepository = fmt.Sprintf("%s:%d", "ghcr.io/vulsio/vuls-nightly-db", db.SchemaVersion)
+
+	// DefaultPath is the path for vuls2 db file
+	DefaultPath = func() string {
+		wd, _ := os.Getwd()
+		return filepath.Join(wd, "vuls.db")
+	}()
+)
+
+func newDBConnection(vuls2Cnf config.Vuls2DictConf, noProgress bool) (db.DB, error) {
+	now := time.Now()
+	stale, err := isStale(vuls2Cnf, now)
+	if err != nil {
+		return nil, xerrors.Errorf("Failed to needsDownload. err: %w", err)
+	}
+
+	if stale {
+		logging.Log.Infof("Downloading vuls2 db. repository: %s", vuls2Cnf.Repository)
+		if err := fetch.Fetch(fetch.WithRepository(vuls2Cnf.Repository), fetch.WithDBPath(vuls2Cnf.Path), fetch.WithNoProgress(noProgress)); err != nil {
+			return nil, xerrors.Errorf("Failed to fetch vuls2 db. err: %w", err)
+		}
+	}
+
+	vuls2Config := db.Config{
+		Type: "boltdb",
+		Path: vuls2Cnf.Path,
+	}
+
+	dbc, err := vuls2Config.New()
+	if err != nil {
+		return nil, xerrors.Errorf("Failed to new vuls2 db. err: %w", err)
+	}
+
+	return dbc, nil
+}
+
+func isStale(vuls2Cnf config.Vuls2DictConf, now time.Time) (bool, error) {
+	if _, err := os.Stat(vuls2Cnf.Path); errors.Is(err, os.ErrNotExist) {
+		if vuls2Cnf.SkipUpdate {
+			return false, xerrors.New("Vuls2 db not found, cannot skip update")
+		}
+		return true, nil
+	}
+
+	vuls2Config := db.Config{
+		Type: "boltdb",
+		Path: vuls2Cnf.Path,
+	}
+
+	dbc, err := vuls2Config.New()
+	if err != nil {
+		return false, xerrors.Errorf("Failed to new vuls2 db. path: %s, err: %w", vuls2Cnf.Path, err)
+	}
+
+	if err := dbc.Open(); err != nil {
+		return false, xerrors.Errorf("Failed to open vuls2 db. path: %s, err: %w", vuls2Cnf.Path, err)
+	}
+	defer func() {
+		_ = dbc.Close()
+	}()
+
+	metadata, err := dbc.GetMetadata()
+	if err != nil {
+		return false, xerrors.Errorf("Failed to get vuls2 db metadata. path: %s, err: %w", vuls2Cnf.Path, err)
+	}
+	if metadata == nil {
+		return false, xerrors.Errorf("Unexpected Vuls2 db metadata. metadata: nil,. path: %s", vuls2Cnf.Path)
+	}
+
+	if metadata.SchemaVersion != db.SchemaVersion {
+		return false, xerrors.Errorf("Unexpected schema version. expected: %d, actual: %d", db.SchemaVersion, metadata.SchemaVersion)
+	}
+
+	if metadata.Downloaded != nil && now.Before((*metadata.Downloaded).Add(1*time.Hour)) {
+		return false, nil
+	}
+	return metadata.LastModified.Add(6 * time.Hour).Before(now), nil
+}

detector/vuls2/db_test.go

@@ -0,0 +1,139 @@
+package vuls2
+
+import (
+	"path/filepath"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/MaineK00n/vuls2/pkg/db/common"
+	"github.com/MaineK00n/vuls2/pkg/db/common/types"
+	"github.com/future-architect/vuls/config"
+)
+
+func Test_isStale(t *testing.T) {
+	type args struct {
+		vuls2Cnf config.Vuls2DictConf
+		now      time.Time
+	}
+	tests := []struct {
+		name     string
+		args     args
+		metadata *types.Metadata
+		want     bool
+		wantErr  bool
+	}{
+		{
+			name: "no db file",
+			args: args{
+				vuls2Cnf: config.Vuls2DictConf{},
+				now:      *parse("2024-01-02T00:00:00Z"),
+			},
+			want: true,
+		},
+		{
+			name: "just created",
+			args: args{
+				vuls2Cnf: config.Vuls2DictConf{},
+				now:      *parse("2024-01-02T00:00:00Z"),
+			},
+			metadata: &types.Metadata{
+				LastModified:  *parse("2024-01-02T00:00:00Z"),
+				Downloaded:    parse("2024-01-02T00:00:00Z"),
+				SchemaVersion: common.SchemaVersion,
+			},
+			want: false,
+		},
+		{
+			name: "8 hours old",
+			args: args{
+				vuls2Cnf: config.Vuls2DictConf{},
+				now:      *parse("2024-01-02T08:00:00Z"),
+			},
+			metadata: &types.Metadata{
+				LastModified:  *parse("2024-01-02T00:00:00Z"),
+				Downloaded:    parse("2024-01-02T00:00:00Z"),
+				SchemaVersion: common.SchemaVersion,
+			},
+			want: true,
+		},
+		{
+			name: "8 hours old, but download recently",
+			args: args{
+				vuls2Cnf: config.Vuls2DictConf{},
+				now:      *parse("2024-01-02T08:00:00Z"),
+			},
+			metadata: &types.Metadata{
+				LastModified:  *parse("2024-01-02T00:00:00Z"),
+				Downloaded:    parse("2024-01-02T07:30:00Z"),
+				SchemaVersion: common.SchemaVersion,
+			},
+			want: false,
+		},
+		{
+			name: "wrong schema version",
+			args: args{
+				vuls2Cnf: config.Vuls2DictConf{},
+				now:      *parse("2024-01-02T08:00:00Z"),
+			},
+			metadata: &types.Metadata{
+				LastModified:  *parse("2024-01-02T00:00:00Z"),
+				Downloaded:    parse("2024-01-02T07:30:00Z"),
+				SchemaVersion: common.SchemaVersion + 42,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := t.TempDir()
+			tt.args.vuls2Cnf.Path = filepath.Join(d, "vuls.db")
+			if tt.metadata != nil {
+				if err := putMetadata(t, *tt.metadata, tt.args.vuls2Cnf.Path); err != nil {
+					t.Errorf("putMetadata() err = %v", err)
+					return
+				}
+			}
+			got, err := isStale(tt.args.vuls2Cnf, tt.args.now)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("isStale() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("isStale() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+
+}
+
+func putMetadata(t *testing.T, metadata types.Metadata, path string) error {
+	c := common.Config{
+		Type: "boltdb",
+		Path: path,
+	}
+	dbc, err := c.New()
+	if err != nil {
+		t.Errorf("c.New() err = %v", err)
+		return err
+	}
+	if err := dbc.Open(); err != nil {
+		t.Errorf("dbc.Open() err = %v", err)
+		return err
+	}
+	defer dbc.Close()
+	if err := dbc.Initialize(); err != nil {
+		t.Errorf("dbc.Initialize() err = %v", err)
+		return err
+	}
+	if err := dbc.PutMetadata(metadata); err != nil {
+		t.Errorf("dbc.PutMetadata() err = %v", err)
+		return err
+	}
+	return nil
+}
+
+func parse(date string) *time.Time {
+	t, _ := time.Parse(time.RFC3339, date)
+	return &t
+}