From 951e6dedf532c83f133c8725cbd43bba1ed6df45 Mon Sep 17 00:00:00 2001 From: Alexander Simmerl Date: Mon, 21 Aug 2017 14:16:20 +0200 Subject: [PATCH] Implement reaction multi count Since the introduction of reactions one of the biggest impacts on response times for feed/list requests has been the constant overhead of countless network requests in order to populate counters for posts. This is due the nature how we constructed the service interfaces. In this change-set we introduce a specialised method to get all counts for a list of posts at once. After we verify that this does impact the response times significantly we can mirror the same idea in the cache layer. --- core/feed.go | 2 +- core/post.go | 102 ++++++---------------------- core/reaction.go | 10 --- handler/http/post.go | 12 ++-- service/reaction/cache.go | 6 +- service/reaction/helper_test.go | 48 +++++++++++++ service/reaction/instrumentation.go | 10 +++ service/reaction/logging.go | 20 ++++++ service/reaction/mem.go | 39 +++++++++++ service/reaction/mem_test.go | 4 ++ service/reaction/postgres.go | 96 +++++++++++++++++++++++++- service/reaction/postgres_test.go | 4 ++ service/reaction/reaction.go | 18 ++++- service/reaction/sourcing.go | 4 ++ 14 files changed, 272 insertions(+), 103 deletions(-) diff --git a/core/feed.go b/core/feed.go index a98ac8e..a93a11c 100644 --- a/core/feed.go +++ b/core/feed.go @@ -1286,7 +1286,7 @@ func sourceReactions( Type: fmt.Sprintf( reactionEventFmt, event.TypeReaction, - reaction.TypeToIdenitifier[r.Type], + reaction.TypeToIdentifier[r.Type], ), UserID: r.OwnerID, Visibility: event.VisibilityPrivate, diff --git a/core/post.go b/core/post.go index bb42a1c..0805d8a 100644 --- a/core/post.go +++ b/core/post.go @@ -37,7 +37,7 @@ type Post struct { type PostCounts struct { Comments int Likes int - ReactionCounts ReactionCounts + ReactionCounts reaction.Counts } // PostFeed is the composite answer for post list methods. @@ -96,6 +96,16 @@ func (ps PostList) OwnerIDs() []uint64 { return ids } +func (ps PostList) objectIDs() []uint64 { + ids := []uint64{} + + for _, p := range ps { + ids = append(ids, p.ObjectID) + } + + return ids +} + func postsFromObjects(os object.List) PostList { ps := PostList{} @@ -545,6 +555,14 @@ func enrichCounts( currentApp *app.App, ps PostList, ) error { + countsMap, err := reactions.CountMulti(currentApp.Namespace(), reaction.QueryOptions{ + Deleted: &defaultDeleted, + ObjectIDs: ps.objectIDs(), + }) + if err != nil { + return err + } + for _, p := range ps { comments, err := objects.Count(currentApp.Namespace(), object.QueryOptions{ ObjectIDs: []uint64{ @@ -558,89 +576,9 @@ func enrichCounts( return err } - reactionCounts := ReactionCounts{} - - reactionCounts.Angry, err = reactions.Count(currentApp.Namespace(), reaction.QueryOptions{ - Deleted: &defaultDeleted, - ObjectIDs: []uint64{ - p.ID, - }, - Types: []reaction.Type{ - reaction.TypeAngry, - }, - }) - if err != nil { - return err - } - - reactionCounts.Haha, err = reactions.Count(currentApp.Namespace(), reaction.QueryOptions{ - Deleted: &defaultDeleted, - ObjectIDs: []uint64{ - p.ID, - }, - Types: []reaction.Type{ - reaction.TypeHaha, - }, - }) - if err != nil { - return err - } - - reactionCounts.Like, err = reactions.Count(currentApp.Namespace(), reaction.QueryOptions{ - Deleted: &defaultDeleted, - ObjectIDs: []uint64{ - p.ID, - }, - Types: []reaction.Type{ - reaction.TypeLike, - }, - }) - if err != nil { - return err - } - - reactionCounts.Love, err = reactions.Count(currentApp.Namespace(), reaction.QueryOptions{ - Deleted: &defaultDeleted, - ObjectIDs: []uint64{ - p.ID, - }, - Types: []reaction.Type{ - reaction.TypeLove, - }, - }) - if err != nil { - return err - } - - reactionCounts.Sad, err = reactions.Count(currentApp.Namespace(), reaction.QueryOptions{ - Deleted: &defaultDeleted, - ObjectIDs: []uint64{ - p.ID, - }, - Types: []reaction.Type{ - reaction.TypeSad, - }, - }) - if err != nil { - return err - } - - reactionCounts.Wow, err = reactions.Count(currentApp.Namespace(), reaction.QueryOptions{ - Deleted: &defaultDeleted, - ObjectIDs: []uint64{ - p.ID, - }, - Types: []reaction.Type{ - reaction.TypeWow, - }, - }) - if err != nil { - return err - } - p.Counts = PostCounts{ Comments: comments, - ReactionCounts: reactionCounts, + ReactionCounts: countsMap[p.ObjectID], } } diff --git a/core/reaction.go b/core/reaction.go index 2623774..59617d0 100644 --- a/core/reaction.go +++ b/core/reaction.go @@ -212,13 +212,3 @@ type ReactionFeed struct { PostMap PostMap UserMap user.Map } - -// ReactionCounts bundles all Reaction counts by type. -type ReactionCounts struct { - Angry uint - Haha uint - Like uint - Love uint - Sad uint - Wow uint -} diff --git a/handler/http/post.go b/handler/http/post.go index 6b86ba2..7e9b842 100644 --- a/handler/http/post.go +++ b/handler/http/post.go @@ -451,12 +451,12 @@ type postCounts struct { } type reactionCounts struct { - Angry uint `json:"angry"` - Haha uint `json:"haha"` - Like uint `json:"like"` - Love uint `json:"love"` - Sad uint `json:"sad"` - Wow uint `json:"wow"` + Angry uint64 `json:"angry"` + Haha uint64 `json:"haha"` + Like uint64 `json:"like"` + Love uint64 `json:"love"` + Sad uint64 `json:"sad"` + Wow uint64 `json:"wow"` } type postFields struct { diff --git a/service/reaction/cache.go b/service/reaction/cache.go index 49f8df7..b8b21a6 100644 --- a/service/reaction/cache.go +++ b/service/reaction/cache.go @@ -50,6 +50,10 @@ func (s *cacheService) Count(ns string, opts QueryOptions) (uint, error) { return aCount, err } +func (s *cacheService) CountMulti(ns string, opts QueryOptions) (CountsMap, error) { + return nil, fmt.Errorf("cacheService.CountMulti not implemented") +} + func (s *cacheService) Put(ns string, input *Reaction) (*Reaction, error) { key := cacheCountKey(QueryOptions{ ObjectIDs: []uint64{ @@ -104,7 +108,7 @@ func cacheCountKey(opts QueryOptions) string { } if len(opts.Types) == 1 { - ps = append(ps, TypeToIdenitifier[opts.Types[0]]) + ps = append(ps, TypeToIdentifier[opts.Types[0]]) } if len(opts.ObjectIDs) == 1 { diff --git a/service/reaction/helper_test.go b/service/reaction/helper_test.go index 63cd97c..8424bc6 100644 --- a/service/reaction/helper_test.go +++ b/service/reaction/helper_test.go @@ -51,6 +51,54 @@ func testServiceCount(p prepareFunc, t *testing.T) { } } +func testServiceCountMulti(p prepareFunc, t *testing.T) { + var ( + objectIDs = []uint64{ + uint64(rand.Int63()), + uint64(rand.Int63()), + uint64(rand.Int63()), + } + ownerID = uint64(rand.Int63()) + namespace = "service_count_multi" + service = p(t, namespace) + ) + + for _, oid := range objectIDs { + for _, r := range testList(oid, ownerID) { + r.ObjectID = oid + + _, err := service.Put(namespace, r) + if err != nil { + t.Fatal(err) + } + } + } + + want := CountsMap{} + + for _, oid := range objectIDs { + want[oid] = Counts{ + Angry: 5, + Haha: 3, + Like: 21, + Love: 9, + Sad: 1, + Wow: 7, + } + } + + have, err := service.CountMulti(namespace, QueryOptions{ + ObjectIDs: objectIDs, + }) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(have, want) { + t.Errorf("\nhave %v\nwant %v", have, want) + } +} + func testServicePut(p prepareFunc, t *testing.T) { var ( deleted = true diff --git a/service/reaction/instrumentation.go b/service/reaction/instrumentation.go index a332c99..d187e5d 100644 --- a/service/reaction/instrumentation.go +++ b/service/reaction/instrumentation.go @@ -50,6 +50,14 @@ func (s *instrumentService) Count( return s.next.Count(ns, opts) } +func (s *instrumentService) CountMulti(ns string, opts QueryOptions) (m CountsMap, err error) { + defer func(begin time.Time) { + s.track("CountMulti", ns, begin, err) + }(time.Now()) + + return s.next.CountMulti(ns, opts) +} + func (s *instrumentService) Put( ns string, input *Reaction, @@ -132,6 +140,8 @@ type instrumentSource struct { store string } +// InstrumentSourceMiddleware observes key apsects of Source operations and exposes +// Prometheus metrics. func InstrumentSourceMiddleware( component, store string, errCount kitmetrics.Counter, diff --git a/service/reaction/logging.go b/service/reaction/logging.go index a65f1b3..3609518 100644 --- a/service/reaction/logging.go +++ b/service/reaction/logging.go @@ -134,6 +134,26 @@ func (s *logService) Count(ns string, opts QueryOptions) (count uint, err error) return s.next.Count(ns, opts) } +func (s *logService) CountMulti(ns string, opts QueryOptions) (m CountsMap, err error) { + defer func(begin time.Time) { + ps := []interface{}{ + "duration_ns", time.Since(begin).Nanoseconds(), + "keys_count", len(m), + "method", "CountMulti", + "namespace", ns, + "opts", opts, + } + + if err != nil { + ps = append(ps, "err", err) + } + + _ = s.logger.Log(ps...) + }(time.Now()) + + return s.next.CountMulti(ns, opts) +} + func (s *logService) Put(ns string, input *Reaction) (output *Reaction, err error) { defer func(begin time.Time) { ps := []interface{}{ diff --git a/service/reaction/mem.go b/service/reaction/mem.go index 603bfba..e5f9188 100644 --- a/service/reaction/mem.go +++ b/service/reaction/mem.go @@ -26,6 +26,45 @@ func (s *memService) Count(ns string, opts QueryOptions) (uint, error) { return uint(len(filterList(s.reactions[ns].ToList(), opts))), nil } +func (s *memService) CountMulti(ns string, opts QueryOptions) (CountsMap, error) { + if err := s.Setup(ns); err != nil { + return nil, err + } + + countsMap := CountsMap{} + + for _, oid := range opts.ObjectIDs { + counts := Counts{} + + for _, r := range s.reactions[ns] { + if r.Deleted { + continue + } + + if r.ObjectID == oid { + switch r.Type { + case TypeAngry: + counts.Angry++ + case TypeHaha: + counts.Haha++ + case TypeLike: + counts.Like++ + case TypeLove: + counts.Love++ + case TypeSad: + counts.Sad++ + case TypeWow: + counts.Wow++ + } + } + } + + countsMap[oid] = counts + } + + return countsMap, nil +} + func (s *memService) Put(ns string, input *Reaction) (*Reaction, error) { if err := s.Setup(ns); err != nil { return nil, err diff --git a/service/reaction/mem_test.go b/service/reaction/mem_test.go index aa7dcff..bfcd33c 100644 --- a/service/reaction/mem_test.go +++ b/service/reaction/mem_test.go @@ -6,6 +6,10 @@ func TestMemCount(t *testing.T) { testServiceCount(prepareMem, t) } +func TestMemCountMulti(t *testing.T) { + testServiceCountMulti(prepareMem, t) +} + func TestMemPut(t *testing.T) { testServicePut(prepareMem, t) } diff --git a/service/reaction/postgres.go b/service/reaction/postgres.go index d70bdf2..6a90855 100644 --- a/service/reaction/postgres.go +++ b/service/reaction/postgres.go @@ -1,6 +1,7 @@ package reaction import ( + "database/sql" "fmt" "time" @@ -26,8 +27,21 @@ const ( WHERE id = $1` - pgCountReactions = `SELECT count(*) FROM %s.reactions %s` - pgListReactions = ` + pgCountReactions = `SELECT count(*) FROM %s.reactions %s` + pgCountReactionsMulti = ` + SELECT + type, + count(*) + FROM + %s.reactions + WHERE + deleted = false + AND object_id = $1 + GROUP BY + type + ORDER BY + type` + pgListReactions = ` SELECT deleted, id, object_id, owner_id, type, created_at, updated_at FROM @@ -94,6 +108,84 @@ func (s *pgService) Count(ns string, opts QueryOptions) (uint, error) { return s.countReactions(ns, where, params...) } +func (s *pgService) CountMulti(ns string, opts QueryOptions) (m CountsMap, err error) { + tx, err := s.db.Begin() + if err != nil { + return nil, err + } + + defer func(tx *sql.Tx) { + if err != nil { + _ = tx.Rollback() + } + }(tx) + + countsMap := CountsMap{} + + for _, oid := range opts.ObjectIDs { + var ( + counts = Counts{} + params = []interface{}{oid} + ) + + query := fmt.Sprintf(pgCountReactionsMulti, ns) + + rows, err := s.db.Query(query, params...) + if err != nil { + if rerr := tx.Rollback(); rerr != nil { + return nil, err + } + + return nil, err + } + defer rows.Close() + + for rows.Next() { + var ( + t int + count uint64 + ) + + err := rows.Scan(&t, &count) + if err != nil { + return nil, err + } + + switch Type(t) { + case TypeAngry: + counts.Angry = count + case TypeHaha: + counts.Haha = count + case TypeLike: + counts.Like = count + case TypeLove: + counts.Love = count + case TypeSad: + counts.Sad = count + case TypeWow: + counts.Wow = count + } + } + + if err := rows.Err(); err != nil { + if rerr := tx.Rollback(); rerr != nil { + return nil, err + } + + return nil, err + } + + countsMap[oid] = counts + } + + err = tx.Commit() + if err != nil { + return nil, err + } + + return countsMap, nil +} + func (s *pgService) Put(ns string, r *Reaction) (*Reaction, error) { if err := r.Validate(); err != nil { return nil, err diff --git a/service/reaction/postgres_test.go b/service/reaction/postgres_test.go index 256d110..ee9fe87 100644 --- a/service/reaction/postgres_test.go +++ b/service/reaction/postgres_test.go @@ -18,6 +18,10 @@ func TestPostgresCount(t *testing.T) { testServiceCount(preparePostgres, t) } +func TestPostgresCountMulti(t *testing.T) { + testServiceCountMulti(preparePostgres, t) +} + func TestPostgresPut(t *testing.T) { testServicePut(preparePostgres, t) } diff --git a/service/reaction/reaction.go b/service/reaction/reaction.go index ba1097a..e8d40e3 100644 --- a/service/reaction/reaction.go +++ b/service/reaction/reaction.go @@ -20,7 +20,9 @@ const ( TypeAngry ) -var TypeToIdenitifier = map[Type]string{ +// TypeToIdentifier is the lookup of a reaction type to human readable +// idenitfier. +var TypeToIdentifier = map[Type]string{ TypeLike: "like", TypeLove: "love", TypeHaha: "haha", @@ -34,6 +36,19 @@ type Consumer interface { Consume() (*StateChange, error) } +// Counts bundles all Reaction counts by type. +type Counts struct { + Angry uint64 + Haha uint64 + Like uint64 + Love uint64 + Sad uint64 + Wow uint64 +} + +// CountsMap is the association of an object id to Counts. +type CountsMap map[uint64]Counts + // List is a collection of Reaction. type List []*Reaction @@ -153,6 +168,7 @@ type Service interface { service.Lifecycle Count(namespace string, opts QueryOptions) (uint, error) + CountMulti(namespace string, opts QueryOptions) (CountsMap, error) Put(namespace string, reaction *Reaction) (*Reaction, error) Query(namespace string, opts QueryOptions) (List, error) } diff --git a/service/reaction/sourcing.go b/service/reaction/sourcing.go index a82bb20..866654a 100644 --- a/service/reaction/sourcing.go +++ b/service/reaction/sourcing.go @@ -18,6 +18,10 @@ func (s *sourcingService) Count(ns string, opts QueryOptions) (uint, error) { return s.service.Count(ns, opts) } +func (s *sourcingService) CountMulti(ns string, opts QueryOptions) (CountsMap, error) { + return s.service.CountMulti(ns, opts) +} + func (s *sourcingService) Put(ns string, input *Reaction) (new *Reaction, err error) { var old *Reaction