diff --git a/Dockerfile b/Dockerfile index 1487a05..4c00d39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:latest AS builder ENV APP_NAME ngtd -ENV NGT_VERSION 1.7.0 +ENV NGT_VERSION 1.7.1 ENV DEBIAN_FRONTEND noninteractive ENV INITRD No @@ -41,6 +41,7 @@ RUN CGO_ENABLED=1 \ GOARCH=$(go env GOARCH) \ GO111MODULE=on \ go build --ldflags '-s -w -linkmode "external" -extldflags "-static -fPIC -m64 -pthread -fopenmp -std=c++17 -lstdc++ -lm"' -a -tags "cgo netgo" -installsuffix "cgo netgo" -o ${APP_NAME} \ + # && upx -9 -o /usr/bin/${APP_NAME} ${APP_NAME} && upx --best --ultra-brute -o /usr/bin/${APP_NAME} ${APP_NAME} # Start From Scratch For Running Environment diff --git a/Makefile b/Makefile index 9ce9e65..13df03a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,23 @@ VERSION := v0.0.1 GO_VERSION:=$(shell go version) REVISION := $(shell git rev-parse --short HEAD) -.PHONY: build +.PHONY: build clean + +clean: + go clean ./... + go clean -modcache + rm -rf ./*.log + rm -rf ./*.svg + rm -rf ./go.mod + rm -rf ./go.sum + rm -rf bench + rm -rf pprof + rm -rf vendor + +init: + GO111MODULE=on go mod init + GO111MODULE=on go mod vendor + sleep 3 deps: curl -LO https://github.com/yahoojapan/NGT/archive/v$(NGT_VERSION).tar.gz @@ -18,5 +34,16 @@ proto/ngtd.pb.go: proto/ngtd.proto build: proto/ngtd.pb.go GO111MODULE=on go build -ldflags="-w -s" -test: proto/ngtd.pb.go - GO111MODULE=on go test -v ./... +test: clean init proto/ngtd.pb.go + GO111MODULE=on go test --race -v ./... + +docker-build: + docker build --pull=true --file=Dockerfile -t yahoojapan/ngtd:latest . + +docker-push: + docker push yahoojapan/ngtd:latest + +coverage: + go test -v -race -covermode=atomic -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html + rm -f coverage.out diff --git a/cmd/ngtd/main.go b/cmd/ngtd/main.go index 90361cb..442baa6 100644 --- a/cmd/ngtd/main.go +++ b/cmd/ngtd/main.go @@ -17,14 +17,12 @@ package main import ( - "database/sql" "fmt" "os" "runtime" "time" "github.com/kpango/glg" - _ "github.com/mattn/go-sqlite3" "github.com/yahoojapan/gongt" "github.com/yahoojapan/ngtd" "github.com/yahoojapan/ngtd/cmd/ngtd/build" @@ -79,13 +77,13 @@ func main() { cli.StringFlag{ Name: "database-type, t", Value: "", - Usage: "ngtd inner kvs type(redis, golevel, bolt or sqlite)", + Usage: "ngtd inner kvs type(redis, golevel, bolt)", Destination: &dbType, }, cli.StringFlag{ Name: "database-path, p", Value: "/usr/share/ngtd/db/kvs.db", - Usage: "ngtd inner kvs path(for golevel, bolt and sqlite)", + Usage: "ngtd inner kvs path(for golevel, bolt)", }, cli.StringFlag{ Name: "redis-host", @@ -142,12 +140,6 @@ func main() { return kvs.NewBoltDB(p) case "golevel": return kvs.NewGoLevel(p) - case "sqlite": - s, err := sql.Open("sqlite3", p) - if err != nil { - return nil, err - } - return kvs.NewSQL(s) default: return nil, fmt.Errorf("unsupported database type: %v", dbType) } diff --git a/go.mod b/go.mod index c7a4ee9..ec9665b 100644 --- a/go.mod +++ b/go.mod @@ -6,13 +6,12 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/go-redis/redis v6.15.2+incompatible github.com/golang/protobuf v1.3.1 - github.com/gorilla/mux v1.7.0 - github.com/kpango/glg v1.3.1 - github.com/mattn/go-sqlite3 v1.10.0 + github.com/gorilla/mux v1.7.1 + github.com/kpango/glg v1.3.2 github.com/syndtr/goleveldb v1.0.0 github.com/yahoojapan/gongt v0.0.0-20190329111937-add7bd9fc281 - golang.org/x/net v0.0.0-20190328230028-74de082e2cca - golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 - google.golang.org/grpc v1.19.1 + golang.org/x/net v0.0.0-20190424024845-afe8014c977f + golang.org/x/sync v0.0.0-20190423024810-112230192c58 + google.golang.org/grpc v1.20.1 gopkg.in/urfave/cli.v1 v1.20.0 ) diff --git a/go.sum b/go.sum index 46ed63b..c7f4400 100644 --- a/go.sum +++ b/go.sum @@ -13,50 +13,53 @@ github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U= -github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU= +github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/kpango/fastime v1.0.8 h1:Wif5eocdsIXmMG+8HHfRP/jD6UUl+/OVTJ+sMzvA1+E= github.com/kpango/fastime v1.0.8/go.mod h1:Y5XY5bLG5yc7g2XmMUzc22XYV1XaH+KgUOHkDvLp4SA= github.com/kpango/glg v1.3.0/go.mod h1:7zzaAoMqvngad+sagWLjr00EQMJaqyGONdg0WYBAO3M= -github.com/kpango/glg v1.3.1 h1:tOwfDQWkmy8WuubZGBz1kcYO2+tJj8WnAan0WpBrb0g= -github.com/kpango/glg v1.3.1/go.mod h1:7zzaAoMqvngad+sagWLjr00EQMJaqyGONdg0WYBAO3M= -github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/kpango/glg v1.3.2 h1:daEbLGtRspX5MYne4ppkl2StRN/hc8hE+ntnu38tsOI= +github.com/kpango/glg v1.3.2/go.mod h1:7zzaAoMqvngad+sagWLjr00EQMJaqyGONdg0WYBAO3M= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/yahoojapan/gongt v0.0.0-20190329111937-add7bd9fc281 h1:ACmE1iAPgPUsdu1HUkVbpc1duQGoZerecJEFIMtRg/Q= github.com/yahoojapan/gongt v0.0.0-20190329111937-add7bd9fc281/go.mod h1:t+18dG3a4an14F0wj/fkSdROjODEN7405sla9IPjuUo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1CtlaD1KilQ6iudt1aI= -golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190424024845-afe8014c977f h1:uALRiwYevCJtciRa4mKKFkrs5jY4F2OTf1D2sfi1swY= +golang.org/x/net v0.0.0-20190424024845-afe8014c977f/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= gonum.org/v1/hdf5 v0.0.0-20190227001252-83207889d689/go.mod h1:g+PDU5ogjIKcc3Cg4ALAK7X4c8bBQvPzPKWNW5NB7I0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/grpc v1.19.1 h1:TrBcJ1yqAl1G++wO39nD/qtgpsW9/1+QGrluyMGEYgM= -google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0= gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/handler/helper_test.go b/handler/helper_test.go index 7204742..7f86f1c 100644 --- a/handler/helper_test.go +++ b/handler/helper_test.go @@ -18,7 +18,7 @@ package handler import ( "testing" - + "github.com/yahoojapan/ngtd/ngtdtest" "github.com/yahoojapan/ngtd/service" ) @@ -30,4 +30,3 @@ func SetupWithTeardown(t *testing.T) func() { ngtdtest.DeleteDB(t) } } - diff --git a/kvs/sql.go b/kvs/sql.go deleted file mode 100644 index 1925ea3..0000000 --- a/kvs/sql.go +++ /dev/null @@ -1,104 +0,0 @@ -// -// Copyright (C) 2018 Yahoo Japan Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package kvs - -import ( - "database/sql" -) - -type SQL struct { - db *sql.DB -} - -func NewSQL(db *sql.DB) (*SQL, error) { - _, err := db.Exec("create table if not exists kvs (key varchar(256) not null primary key, val integer not null unique)") - if err != nil { - return nil, err - } - return &SQL{ - db: db, - }, nil -} - -func (s *SQL) GetKey(val uint) ([]byte, error) { - stmt, err := s.db.Prepare("select key from kvs where val = ? limit 1") - if err != nil { - return nil, err - } - defer stmt.Close() - - var key string - if err := stmt.QueryRow(val).Scan(&key); err != nil { - return nil, err - } - - return []byte(key), nil -} - -// GetKeys wraps multiple calls GetKey -func (s *SQL) GetKeys(vals []uint) ([][]byte, error) { - ret := make([][]byte, len(vals)) - for i, val := range vals { - k, err := s.GetKey(val) - if err != nil { - return nil, err - } - ret[i] = k - } - return ret, nil -} - -func (s *SQL) GetVal(key []byte) (uint, error) { - stmt, err := s.db.Prepare("select val from kvs where key = ? limit 1") - if err != nil { - return 0, err - } - defer stmt.Close() - - var val uint - if err := stmt.QueryRow(string(key)).Scan(&val); err != nil { - return 0, err - } - - return val, nil -} - -func (s *SQL) Set(key []byte, val uint) error { - stmt, err := s.db.Prepare("insert into kvs (key, val) values(?, ?)") - if err != nil { - return err - } - defer stmt.Close() - _, err = stmt.Exec(string(key), val) - return err -} - -func (s *SQL) Delete(key []byte) error { - stmt, err := s.db.Prepare("delete from kvs where key = ?") - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec(string(key)) - return err - -} - -func (s *SQL) Close() error { - return s.db.Close() -} diff --git a/kvs/sql_test.go b/kvs/sql_test.go deleted file mode 100644 index 2f95bf0..0000000 --- a/kvs/sql_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// -// Copyright (C) 2018 Yahoo Japan Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package kvs - -import ( - "os" - "testing" - - "database/sql" - _ "github.com/mattn/go-sqlite3" -) - -func initSQL(t *testing.T) *SQL { - db, err := sql.Open("sqlite3", ":memory:") - if err != nil { - t.Fatalf("Unexpected Error: initSQL(): %v", err) - } - s, err := NewSQL(db) - if err != nil { - t.Fatalf("Unexpected Error: initSQL(): %v", err) - } - return s -} - -func TestSQL(t *testing.T) { - t.Parallel() - t.Run("TestGetKey", func(t *testing.T) { - s := initSQL(t) - defer SetupWithTeardown(s, t)() - GetKey(s, t) - }) - - t.Run("TestGetKeys", func(t *testing.T) { - s := initSQL(t) - defer SetupWithTeardown(s, t)() - GetKeys(s, t) - }) - - t.Run("TestGetVal", func(t *testing.T) { - s := initSQL(t) - defer SetupWithTeardown(s, t)() - GetVal(s, t) - }) - - t.Run("TestSet", func(t *testing.T) { - s := initSQL(t) - Set(s, t) - s.Close() - os.RemoveAll("test") - }) - - t.Run("TestDelete", func(t *testing.T) { - s := initSQL(t) - defer SetupWithTeardown(s, t)() - Delete(s, t) - }) - - t.Run("TestClose", func(t *testing.T) { - s := initSQL(t) - defer SetupWithTeardown(s, t)() - Close(s, t) - }) -} diff --git a/proto/ngtd.pb.go b/proto/ngtd.pb.go index 424f831..25080af 100644 --- a/proto/ngtd.pb.go +++ b/proto/ngtd.pb.go @@ -7,10 +7,11 @@ import ( context "context" encoding_binary "encoding/binary" fmt "fmt" - proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" io "io" math "math" + + proto "github.com/golang/protobuf/proto" + grpc "google.golang.org/grpc" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/service/service.go b/service/service.go index afeaf0f..02d07d6 100644 --- a/service/service.go +++ b/service/service.go @@ -17,6 +17,7 @@ package service import ( + "errors" "fmt" "sync" @@ -123,6 +124,10 @@ func Insert(vector []float64, id []byte) error { } func (s *Service) Insert(vector []float64, id []byte) error { + i, _ := s.db.GetVal(id) + if i != 0 { + return errors.New("ID already exists") + } in, err := gongt.StrictInsert(vector) if err != nil { return err