close #2 (remove_duplicates optimization)

handlers/remove_duplicates.py

@@ -17,13 +17,26 @@ def handle_game(self, game):
  game.hash = hasher.hexdigest()
  game.put_async()
 
+ def get(self):
+ GameHistory(json_string='a').put()
+ GameHistory(json_string='a').put()
+ GameHistory(json_string='b').put()
+ GameHistory(json_string='c').put()
+ GameHistory(json_string='c').put()
+ GameHistory(json_string='c').put()
+
  @ndb.toplevel
  def post(self):
  stage = self.request.get('stage')
+ c = ndb.Cursor(urlsafe=self.request.get('cursor'))
  if stage == 'hash':
- GameHistory.query().map(self.handle_game)
+ games, curs, more = GameHistory.query().fetch_page(10, start_cursor=c)
+ map(self.handle_game, games)
+ if len(games) > 0 and more and curs:
+ taskqueue.add(url='/remove_duplicates',
+ params={'stage': 'hash', 'cursor': curs.urlsafe()},
+ queue_name='fast')
  elif stage == 'mark':
- c = ndb.Cursor(urlsafe=self.request.get('cursor'))
  game, curs, more = GameHistory.query().fetch_page(1, start_cursor=c)
  game = game[0]
  if game is None:
@@ -35,7 +48,7 @@ def post(self):
  for el in duplicates:
  el.ignored = True
  ndb.put_multi(duplicates)
- if more:
+ if more and curs:
  taskqueue.add(url='/remove_duplicates',
  params={'stage': 'mark', 'cursor': curs.urlsafe()},
  queue_name='fast')

handlers/statistics/word_statistics_handler.py

@@ -21,13 +21,13 @@ def get(self, *args, **kwargs):
  entity, games, top, bottom, rand = None, None, None, None, None
  if word:
  entity = ndb.Key(GlobalDictionaryWord, word).get()
+ if entity:
  games = []
  for id in entity.used_games:
  games.append(ndb.Key('GameLog', id).urlsafe())
  for id in entity.used_legacy_games:
  games.append(ndb.Key('GameHistory', id).urlsafe())
-
- if not entity:
+ else:
  top = memcache.get("words_top")
  if not top:
  top = GlobalDictionaryWord.query(projection=[GlobalDictionaryWord.E, GlobalDictionaryWord.word]).\
@@ -46,6 +46,7 @@ def get(self, *args, **kwargs):
  memcache.set("used_words_count", c)
  if c >= 10:
  rand = q.fetch(limit=10, offset=randint(0, c-10))
+
  self.draw_page('statistics/word_statistic', word=word, word_entity=entity, games=games,
  top=top, bottom=bottom, rand=rand)
 

templates/statistics/word_statistic.html

@@ -9,7 +9,7 @@
  {% for word in list %}
  <tr>
  <td>{{ loop.index }}</td>
- <td><a href="statistics/word_statistics?word={{ word.word }}">{{ word.word }}</a></td>
+ <td><a href="word_statistics?word={{ word.word }}">{{ word.word }}</a></td>
  <td>{{ word.E|round(1) }}</td>
  </tr>
  {% endfor %}
@@ -31,7 +31,7 @@
 <div class="alert alert-danger">К сожалению, в нашем словаре нет слова "{{ word }}"</div>
 {% endif %}
 <h2>Выберите, статистику по какому слову вы хотите посмотреть:</h2>
-<form style="margin-left: 30px" class="form-inline" method="get" action="statistics/word_statistics">
+<form style="margin-left: 30px" class="form-inline" method="get" action="word_statistics">
  <div class="form-group">
  <input type="search" class="form-control" name="word" placeholder="Слово">
  </div>
@@ -58,7 +58,7 @@ <h2>Выберите, статистику по какому слову вы х
 <div class="row">
 <div class="col-md-6"><p class="lead">Показана статистика по слову "{{ word_entity.word }}"</p></div>
 <div class="col-md-6">
- <form class="form-inline" method="get" action="statistics/word_statistics">
+ <form class="form-inline" method="get" action="word_statistics">
  <div class="form-group">
  <input type="search" class="form-control" name="word" placeholder="Слово">
  </div>