master: Add an option to janitor to delete logs for specific builders

common/code_spelling_ignore_words.txt

@@ -1467,6 +1467,7 @@ thursday
 tid
 timedelta
 timestamp
+timestamps
 timezone
 timezones
 tld

master/buildbot/test/fakedb/logs.py

@@ -138,7 +138,7 @@ def finishLog(self, logid):
  def compressLog(self, logid, force=False):
  return defer.succeed(None)
 
- def deleteOldLogChunks(self, older_than_timestamp):
+ def deleteOldLogChunks(self, older_than_timestamp=None, horizon_per_builder=None):
  # not implemented
  self._deleted = older_than_timestamp
  return defer.succeed(1)

master/buildbot/test/unit/db/test_build_data.py

@@ -13,6 +13,9 @@
 #
 # Copyright Buildbot Team Members
 
+from datetime import datetime
+from datetime import timedelta
+
 from parameterized import parameterized
 
 from twisted.internet import defer
@@ -23,6 +26,7 @@
 from buildbot.test.util import connector_component
 from buildbot.test.util import interfaces
 from buildbot.test.util import validation
+from buildbot.util import datetime2epoch
 
 
 class Tests(interfaces.InterfaceTests):
@@ -206,7 +210,8 @@ def test_remove_old_build_data(self, older_than_timestamp, exp_num_deleted,
  fakedb.BuildData(id=96, buildid=53, name='name6', value=b'value6', source='src6'),
  ])
 
- num_deleted = yield self.db.build_data.deleteOldBuildData(older_than_timestamp)
+ num_deleted = yield self.db.build_data.deleteOldBuildData(
+ older_than_timestamp=older_than_timestamp)
  self.assertEqual(num_deleted, exp_num_deleted)
 
  remaining_names = []
@@ -217,6 +222,64 @@ def test_remove_old_build_data(self, older_than_timestamp, exp_num_deleted,
  self.assertEqual(sorted(remaining_names), sorted(exp_remaining_names))
 
 
+class RealTests(Tests):
+ @parameterized.expand([
+ (
+ {
+ "b1": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ },
+ "b2": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }, 2),
+ (
+ {
+ "b%": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }, 2),
+ (
+ {
+ "b1": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ },
+ "b7": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }, 1),
+ ])
+ @defer.inlineCallbacks
+ def test_remove_old_build_data_horizon_per_builder(self, config, exp_deleted):
+ yield self.insertTestData(self.common_data + [
+ fakedb.Build(id=50, buildrequestid=41, number=17, masterid=88,
+ builderid=88, workerid=47, complete_at=None),
+ fakedb.Build(id=51, buildrequestid=42, number=18, masterid=88,
+ builderid=88, workerid=47,
+ complete_at=datetime2epoch(datetime.now() - timedelta(weeks=1))),
+ fakedb.Build(id=52, buildrequestid=42, number=19, masterid=88,
+ builderid=88, workerid=47,
+ complete_at=datetime2epoch(datetime.now() - timedelta(weeks=2))),
+ fakedb.Build(id=53, buildrequestid=43, number=12, masterid=88,
+ builderid=89, workerid=47,
+ complete_at=datetime2epoch(datetime.now() - timedelta(weeks=2))),
+ fakedb.BuildData(id=91, buildid=50, name='name1', value=b'value1', source='src1'),
+ fakedb.BuildData(id=92, buildid=50, name='name2', value=b'value2', source='src2'),
+ fakedb.BuildData(id=93, buildid=51, name='name3', value=b'value3', source='src3'),
+ fakedb.BuildData(id=94, buildid=51, name='name4', value=b'value4', source='src4'),
+ fakedb.BuildData(id=95, buildid=52, name='name5', value=b'value5', source='src5'),
+ fakedb.BuildData(id=96, buildid=53, name='name6', value=b'value6', source='src6'),
+ ])
+
+ num_deleted = yield self.db.build_data.deleteOldBuildData(horizon_per_builder=config)
+ self.assertEqual(num_deleted, exp_deleted)
+
+
 class TestFakeDB(Tests, connector_component.FakeConnectorComponentMixin, unittest.TestCase):
 
  @defer.inlineCallbacks
@@ -226,7 +289,7 @@ def setUp(self):
 
 class TestRealDB(unittest.TestCase,
  connector_component.ConnectorComponentMixin,
- Tests):
+ RealTests):
 
  @defer.inlineCallbacks
  def setUp(self):

master/buildbot/test/unit/db/test_logs.py

@@ -13,13 +13,15 @@
 #
 # Copyright Buildbot Team Members
 
-
 import base64
 import bz2
 import textwrap
 import zlib
+from datetime import datetime
+from datetime import timedelta
 
 import sqlalchemy as sa
+from parameterized import parameterized
 
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -30,6 +32,7 @@
 from buildbot.test.util import interfaces
 from buildbot.test.util import validation
 from buildbot.util import bytes2unicode
+from buildbot.util import datetime2epoch
 from buildbot.util import unicode2bytes
 
 
@@ -141,7 +144,7 @@ def compressLog(self, logid, force=False):
 
  def test_signature_deleteOldLogChunks(self):
  @self.assertArgSpecMatches(self.db.logs.deleteOldLogChunks)
- def deleteOldLogChunks(self, older_than_timestamp):
+ def deleteOldLogChunks(self, older_than_timestamp=None, horizon_per_builder=None):
  pass
 
  # method tests
@@ -571,6 +574,66 @@ def test_deleteOldLogChunks_basic(self):
  lines = yield self.db.logs.getLogLines(logid, 0, logdict['num_lines'])
  self.assertEqual(lines, '')
 
+ @parameterized.expand([
+ (
+ {
+ "b1": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ },
+ "b7": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }, 30),
+ (
+ {
+ "b2": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ },
+ "b7": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }, 10),
+ (
+ {
+ "b%": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ },
+ "b7": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }, 40)
+ ])
+ @defer.inlineCallbacks
+ def test_deleteOldLogChunks_horizon_per_builder(self, config, exp_deleted):
+ yield self.insertTestData(self.backgroundData + [
+ fakedb.Builder(id=89, name='b2'),
+ fakedb.Build(id=31, buildrequestid=41, number=7,
+ masterid=88, builderid=89, workerid=47),
+ fakedb.Step(id=107, buildid=30, number=3, name='three',
+ started_at=datetime2epoch(datetime.now() - timedelta(weeks=2))),
+ fakedb.Step(id=108, buildid=31, number=4, name='four',
+ started_at=datetime2epoch(datetime.now() - timedelta(weeks=2))),
+ fakedb.Step(id=109, buildid=30, number=5, name='five',
+ started_at=datetime2epoch(datetime.now())),
+ fakedb.Step(id=110, buildid=31, number=6, name='six',
+ started_at=datetime2epoch(datetime.now()))])
+ logids = []
+ for stepid in (101, 102, 107, 108, 109, 110):
+ for i in range(10):
+ logid = yield self.db.logs.addLog(
+ stepid=stepid, name='another' + str(i), slug='another' + str(i), type='s')
+ yield self.db.logs.appendLog(logid, 'xyz\n')
+ logids.append(logid)
+
+ num_deleted = yield self.db.logs.deleteOldLogChunks(horizon_per_builder=config)
+ self.assertEqual(num_deleted, exp_deleted)
+
 
 class TestFakeDB(unittest.TestCase, connector_component.FakeConnectorComponentMixin, Tests):
 

master/buildbot/test/unit/test_janitor_configurator.py

@@ -54,6 +54,11 @@ def test_nothing(self):
  ('logs_build_data', {'build_data_horizon': timedelta(weeks=1),
  'logHorizon': timedelta(weeks=1)},
  [LogChunksJanitor, BuildDataJanitor]),
+ ('horizon_per_builder', {'horizon_per_builder': {
+ 'b1': {
+ 'logHorizon': timedelta(weeks=1),
+ 'buildDataHorizon': timedelta(weeks=1)}}},
+ [LogChunksJanitor, BuildDataJanitor]),
  ])
  def test_steps(self, name, configuration, exp_steps):
  self.setupConfigurator(**configuration)
@@ -64,6 +69,26 @@ def test_steps(self, name, configuration, exp_steps):
  self.expectNoConfigError()
 
 
+class JanitorConfiguratorErrorsTests(TestBuildStepMixin,
+ configmixin.ConfigErrorsMixin,
+ TestReactorMixin,
+ unittest.TestCase):
+
+ def test_BuildDataJanitor_InvalidConfig(self):
+ horizon_per_builder = {
+ "b1": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }
+ with self.assertRaisesConfigError("JanitorConfigurator: horizon_per_builder only " +
+ "possible without logHorizon and build_data_horizon set."):
+ self.setup_step(JanitorConfigurator(logHorizon=timedelta(weeks=1),
+ horizon_per_builder=horizon_per_builder))
+ self.setup_step(JanitorConfigurator(build_data_horizon=timedelta(weeks=1),
+ horizon_per_builder=horizon_per_builder))
+
+
 class LogChunksJanitorTests(TestBuildStepMixin,
  configmixin.ConfigErrorsMixin,
  TestReactorMixin,
@@ -87,7 +112,24 @@ def test_basic(self):
  state_string="deleted 3 logchunks")
  yield self.run_step()
  expected_timestamp = datetime2epoch(datetime.datetime(year=2016, month=12, day=25))
- self.master.db.logs.deleteOldLogChunks.assert_called_with(expected_timestamp)
+ self.master.db.logs.deleteOldLogChunks.assert_called_with(
+ older_than_timestamp=expected_timestamp)
+
+ @defer.inlineCallbacks
+ def test_LogChunksJanitorHorizon_PerBuilder(self):
+ config = {
+ "b1": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }
+ self.setup_step(LogChunksJanitor(horizon_per_builder=config))
+ self.master.db.logs.deleteOldLogChunks = mock.Mock(return_value=3)
+ self.expect_outcome(result=SUCCESS,
+ state_string="deleted 3 logchunks")
+ yield self.run_step()
+ self.master.db.logs.deleteOldLogChunks.assert_called_with(
+ horizon_per_builder=config)
 
  @defer.inlineCallbacks
  def test_build_data(self):
@@ -96,4 +138,20 @@ def test_build_data(self):
  self.expect_outcome(result=SUCCESS, state_string="deleted 4 build data key-value pairs")
  yield self.run_step()
  expected_timestamp = datetime2epoch(datetime.datetime(year=2016, month=12, day=25))
- self.master.db.build_data.deleteOldBuildData.assert_called_with(expected_timestamp)
+ self.master.db.build_data.deleteOldBuildData.assert_called_with(
+ older_than_timestamp=expected_timestamp)
+
+ @defer.inlineCallbacks
+ def test_BuildDataJanitor_horizon_per_builder(self):
+ config = {
+ "b1": {
+ "logHorizon": timedelta(weeks=1),
+ "buildDataHorizon": timedelta(weeks=1)
+ }
+ }
+ self.setup_step(BuildDataJanitor(horizon_per_builder=config))
+ self.master.db.build_data.deleteOldBuildData = mock.Mock(return_value=4)
+ self.expect_outcome(result=SUCCESS, state_string="deleted 4 build data key-value pairs")
+ yield self.run_step()
+ self.master.db.build_data.deleteOldBuildData.assert_called_with(
+ horizon_per_builder=config)