Merge branch 'develop' into 'master'

Develop

See merge request alpinefresh/tcr-party/tcrpartybot!58

api/server.go

@@ -628,14 +628,36 @@ func (server *Server) syncList(w http.ResponseWriter, r *http.Request) {
 	w.Write([]byte("Ok"))
 }
 
+type listShowResult struct {
+	Applications []*models.RegistryListing              `json:"applications"`
+	Challenges   []*models.RegistryListingWithChallenge `json:"challenges"`
+	Whitelisted  []*models.RegistryListing              `json:"whitelisted"`
+}
+
 func (server *Server) showList(w http.ResponseWriter, r *http.Request) {
-	listings, err := models.FindWhitelistedRegistryListings()
+	applications, err := models.FindUnchallengedApplications()
+	if err != nil {
+		renderServerError(err, w)
+		return
+	}
+
+	whitelisted, err := models.FindUnchallengedWhitelistedListings()
+	if err != nil {
+		renderServerError(err, w)
+		return
+	}
+
+	challenges, err := models.FindChallengedRegistryListings()
 	if err != nil {
 		renderServerError(err, w)
 		return
 	}
 
-	jsonText, err := json.Marshal(listings)
+	jsonText, err := json.Marshal(listShowResult{
+		Applications: applications,
+		Challenges:   challenges,
+		Whitelisted:  whitelisted,
+	})
 	if err != nil {
 		renderServerError(err, w)
 		return

events/eth_listener.go

@@ -191,6 +191,10 @@ func ETHListener(ethEvents chan<- *ETHEvent, errChan chan<- error, generateContr
 			syncing = true
 		}
 
+		if blockCursor.Cmp(toBlock) == 0 {
+			continue
+		}
+
 		// The filter is inclusive, therefore we should add 1 to the last seen block
 		query.FromBlock = blockCursor.Add(blockCursor, big.NewInt(1))
 		query.ToBlock = toBlock
@@ -205,6 +209,13 @@ func ETHListener(ethEvents chan<- *ETHEvent, errChan chan<- error, generateContr
 		// Create a cache for block header information, allowing us to share
 		// block timestamp data between multiple events within the same block
 		blockInfo := map[uint64]*types.Header{}
+
+		// Certain operations in this process are prone to fail (namely any
+		// call to the blockchain). Instead of firing off ETHEvents as we
+		// discover them, we batch them together and only fire them off if
+		// there were no errors while preparing the events.
+		events := []*ETHEvent{}
+		needsRetry := false
 		for _, ethLog := range logs {
 			// Get the block timestamp
 			var blockTime *time.Time
@@ -221,17 +232,13 @@ func ETHListener(ethEvents chan<- *ETHEvent, errChan chan<- error, generateContr
 				header, err := client.HeaderByNumber(context.Background(), block)
 				if err != nil {
 					errChan <- errors.Wrap(err)
+					needsRetry = true
+					break
 				}
 
-				if header != nil {
-					time := time.Unix(header.Time.Int64(), 0)
-					blockTime = &time
-					blockInfo[ethLog.BlockNumber] = header
-				} else {
-					// If INFURA's API fails us let's just mark the block time
-					// as nil and be on our merry way
-					blockTime = nil
-				}
+				time := time.Unix(header.Time.Int64(), 0)
+				blockTime = &time
+				blockInfo[ethLog.BlockNumber] = header
 			}
 
 			// Look for a topic that we're interested in
@@ -252,11 +259,20 @@ func ETHListener(ethEvents chan<- *ETHEvent, errChan chan<- error, generateContr
 					Data:        ethLog.Data,
 					Topics:      ethLog.Topics,
 				}
-
-				ethEvents <- event
+				events = append(events, event)
 			}
 		}
 
+		if needsRetry {
+			log.Println("Synced events were marked for retry due to error (see above in logs)")
+			continue
+		}
+
+		// Send 'em out
+		for _, event := range events {
+			ethEvents <- event
+		}
+
 		err = models.SetKey(lastSyncedKey, toBlock.String())
 		if err != nil {
 			errChan <- errors.Wrap(err)

events/registry_generator.go

@@ -3,7 +3,6 @@ package events
 import (
 	"database/sql"
 	"fmt"
-	"log"
 	"strconv"
 	"time"
 
@@ -43,8 +42,6 @@ func generateRegistry() error {
 		return nil
 	}
 
-	log.Printf("Updating registry since %d", intID)
-	// @TODO Start a transaction
 	for _, event := range events {
 		switch event.EventType {
 		case ETHEventNewTCRApplication:

models/registry_challenge.go

@@ -7,18 +7,20 @@ import (
 
 // RegistryChallenge represents a challenge on a RegistryListing
 type RegistryChallenge struct {
-	PollID              int64         `db:"poll_id"`
-	ListingHash         string        `db:"listing_hash"`
-	ListingID           string        `db:"listing_id"`
-	ChallengerAccountID sql.NullInt64 `db:"challenger_account_id"`
-	ChallengerAddress   string        `db:"challenger_address"`
-	CreatedAt           *time.Time    `db:"created_at"`
-	CommitEndsAt        *time.Time    `db:"commit_ends_at"`
-	RevealEndsAt        *time.Time    `db:"reveal_ends_at"`
-	SucceededAt         *time.Time    `db:"succeeded_at"`
-	FailedAt            *time.Time    `db:"failed_at"`
+	PollID              int64         `db:"poll_id" json:"poll_id"`
+	ListingHash         string        `db:"listing_hash" json:"-"`
+	ListingID           string        `db:"listing_id" json:"-"`
+	ChallengerAccountID sql.NullInt64 `db:"challenger_account_id" json:"-"`
+	ChallengerAddress   string        `db:"challenger_address" json:"-"`
+	CreatedAt           *time.Time    `db:"created_at" json:"created_at"`
+	CommitEndsAt        *time.Time    `db:"commit_ends_at" json:"commit_ends_at"`
+	RevealEndsAt        *time.Time    `db:"reveal_ends_at" json:"reveal_ends_at"`
+	SucceededAt         *time.Time    `db:"succeeded_at" json:"-"`
+	FailedAt            *time.Time    `db:"failed_at" json:"-"`
 }
 
+// FindRegistryChallengeByPollID finds a challenge given its poll ID (also
+// known as a challenge ID)
 func FindRegistryChallengeByPollID(pollID int64) (*RegistryChallenge, error) {
 	db := GetDBSession()
 
@@ -36,6 +38,18 @@ func FindRegistryChallengeByPollID(pollID int64) (*RegistryChallenge, error) {
 	return &challenge, nil
 }
 
+// FindActiveChallenges returns a list of challenges current in progress
+func FindActiveChallenges() ([]*RegistryChallenge, error) {
+	db := GetDBSession()
+
+	challenges := []*RegistryChallenge{}
+	err := db.Select(&challenges, `
+		SELECT * FROM registry_challenges WHERE succeeded_at IS NULL AND failed_at IS NULL
+	`)
+
+	return challenges, err
+}
+
 // Create inserts the registry challenge into the database
 func (challenge *RegistryChallenge) Create() error {
 	db := GetDBSession()

models/registry_listing.go

@@ -10,14 +10,19 @@ import (
 // RegistryListing represents an application for a listing in the registry
 // (which may later be whitelisted)
 type RegistryListing struct {
-	ID                   string     `db:"id"`
-	ListingHash          string     `db:"listing_hash"`
-	TwitterHandle        string     `db:"twitter_handle"`
-	Deposit              string     `db:"deposit"`
-	ApplicationCreatedAt *time.Time `db:"application_created_at"`
-	ApplicationEndedAt   *time.Time `db:"application_ended_at"`
-	WhitelistedAt        *time.Time `db:"whitelisted_at"`
-	RemovedAt            *time.Time `db:"removed_at"`
+	ID                   string     `db:"id" json:"-"`
+	ListingHash          string     `db:"listing_hash" json:"listing_hash"`
+	TwitterHandle        string     `db:"twitter_handle" json:"twitter_handle"`
+	Deposit              string     `db:"deposit" json:"deposit"`
+	ApplicationCreatedAt *time.Time `db:"application_created_at" json:"application_created_at"`
+	ApplicationEndedAt   *time.Time `db:"application_ended_at" json:"application_ended_at"`
+	WhitelistedAt        *time.Time `db:"whitelisted_at" json:"whitelisted_at"`
+	RemovedAt            *time.Time `db:"removed_at" json:"-"`
+}
+
+type RegistryListingWithChallenge struct {
+	RegistryListing
+	RegistryChallenge `db:"registry_challenge" json:"challenge"`
 }
 
 // FindLatestRegistryListingByHash finds a registry listing by its listing hash
@@ -54,6 +59,77 @@ func FindWhitelistedRegistryListings() ([]*RegistryListing, error) {
 	return listings, err
 }
 
+// FindUnchallengedWhitelistedListings returns a list of registry listings that
+// have been whitelisted but do not have an active challenge against them
+func FindUnchallengedWhitelistedListings() ([]*RegistryListing, error) {
+	db := GetDBSession()
+
+	listings := []*RegistryListing{}
+	err := db.Select(&listings, `
+		SELECT
+			registry_listings.*
+		FROM
+			registry_listings
+		LEFT OUTER JOIN registry_challenges ON
+			registry_challenges.listing_id = registry_listings.id
+		WHERE
+			registry_listings.whitelisted_at IS NOT NULL AND
+			registry_listings.removed_at IS NULL AND
+			registry_challenges.succeeded_at IS NULL AND
+			registry_challenges.failed_at IS NULL
+	`)
+
+	return listings, err
+}
+
+// FindChallengedRegistryListings returns a list of registry listings (in the
+// application and whitelisted stages) that are under active challenge
+func FindChallengedRegistryListings() ([]*RegistryListingWithChallenge, error) {
+	db := GetDBSession()
+
+	listings := []*RegistryListingWithChallenge{}
+	err := db.Select(&listings, `
+		SELECT
+			registry_listings.*,
+			registry_challenges.poll_id AS "registry_challenge.poll_id",
+			registry_challenges.created_at AS "registry_challenge.created_at",
+			registry_challenges.commit_ends_at AS "registry_challenge.commit_ends_at",
+			registry_challenges.reveal_ends_at AS "registry_challenge.reveal_ends_at"
+		FROM
+			registry_listings
+		INNER JOIN registry_challenges ON
+			registry_challenges.listing_id = registry_listings.id
+		WHERE
+			registry_listings.removed_at IS NULL AND
+			registry_challenges.succeeded_at IS NULL AND
+			registry_challenges.failed_at IS NULL
+	`)
+
+	return listings, err
+}
+
+// FindUnchallengedApplications returns a list of listings that are currently
+// applying to be on the TCR and do not have an active challenge against them.
+func FindUnchallengedApplications() ([]*RegistryListing, error) {
+	db := GetDBSession()
+
+	listings := []*RegistryListing{}
+	err := db.Select(&listings, `
+		SELECT
+			registry_listings.*
+		FROM
+			registry_listings
+		LEFT OUTER JOIN registry_challenges ON
+			registry_challenges.listing_id = registry_listings.id
+		WHERE
+			registry_listings.whitelisted_at IS NULL AND
+			registry_listings.removed_at IS NULL AND
+			registry_challenges.poll_id IS NULL
+	`)
+
+	return listings, err
+}
+
 // Create inserts the registry challenge into the database
 func (listing *RegistryListing) Create() error {
 	db := GetDBSession()
@@ -87,6 +163,7 @@ func (listing *RegistryListing) Create() error {
 	return err
 }
 
+// Save updates the registry listing in the database based on its current state
 func (listing *RegistryListing) Save() error {
 	db := GetDBSession()
 	_, err := db.NamedExec(`

tcrpartybot/main.go

@@ -35,25 +35,34 @@ func main() {
 		log.Printf("Credentials for vip bot not found. Please authenticate!")
 	}
 
-	// Set up channels
-	ethEvents := make(chan *events.ETHEvent)
-	twitterEventChan := make(chan *events.TwitterEvent)
 	errChan := make(chan error)
 
 	// Listen for and process any incoming twitter events
 	go twitter.MonitorRatelimit()
+
+	twitterEventChan := make(chan *events.TwitterEvent)
 	go api.StartServer(twitterEventChan, errChan)
 	go events.ProcessTwitterEvents(twitterEventChan, errChan)
 
-	// Listen on Twitter and ETH to determine who/when to retweet
-	go events.ListenAndRetweet(ethEvents, errChan)
-
 	// Look for any existing applications/challenges that may need to be updated
 	go events.ScheduleUpdates(errChan)
 
 	// Start listening for relevant events on the blockchain
+	ethEvents := make(chan *events.ETHEvent)
+	twitterETHChan := make(chan *events.ETHEvent, 10)
+	processETHChan := make(chan *events.ETHEvent, 10)
+
+	go func() {
+		// Fan events out to each listner
+		for event := range ethEvents {
+			twitterETHChan <- event
+			processETHChan <- event
+		}
+	}()
+
 	go events.StartBotListener(ethEvents, errChan)
-	go events.ProcessETHEvents(ethEvents, errChan)
+	go events.ListenAndRetweet(twitterETHChan, errChan)
+	go events.ProcessETHEvents(processETHChan, errChan)
 
 	startRepl := flag.Bool("repl", false, "Starts the debug REPL")
 	flag.Parse()