diff options
| -rw-r--r-- | patches/bb-clean-db.patch | 176 | ||||
| -rw-r--r-- | patches/bb-gitlab.patch | 22 | ||||
| -rw-r--r-- | patches/bb-props_master_fix.patch | 2 |
3 files changed, 67 insertions, 133 deletions
diff --git a/patches/bb-clean-db.patch b/patches/bb-clean-db.patch index 8d370a4..c9eecda 100644 --- a/patches/bb-clean-db.patch +++ b/patches/bb-clean-db.patch @@ -1,60 +1,56 @@ --- a/buildbot/db/build_data.py 2023-09-19 23:24:04.714268581 +0200 +++ b/buildbot/db/build_data.py 2023-09-24 22:28:21.778583084 +0200 -@@ -159,6 +159,15 @@ class BuildDataConnectorComponent(base.D - res = yield self.db.pool.do(thd) - return res +@@ -185,6 +185,16 @@ -+ @defer.inlineCallbacks -+ def pruneBuildData(self, buildid): -+ def thd(conn): + return self.db.pool.do(thd) + ++ def pruneBuildData(self, buildid: int) -> defer.Deferred[None]: ++ def thd(conn) -> None: + tbl = self.db.model.build_data + q = tbl.delete() + q = q.where(tbl.c.buildid == buildid) -+ conn.execute(q) -+ yield self.db.pool.do(thd) ++ res = conn.execute(q) ++ conn.commit() ++ res.close() ++ return self.db.pool.do(thd) + - def _row2dict(self, conn, row): - return BuildDataDict(buildid=row.buildid, - name=row.name, + def _model_from_row(self, row, value: bytes | None): + return BuildDataModel( + buildid=row.buildid, --- a/buildbot/db/builds.py 2023-09-19 23:24:04.714268581 +0200 +++ b/buildbot/db/builds.py 2023-09-24 15:31:20.598023751 +0200 -@@ -206,6 +206,15 @@ class BuildsConnectorComponent(base.DBCo - results=results) - return self.db.pool.do(thd) +@@ -335,6 +335,26 @@ -+ @defer.inlineCallbacks -+ def pruneBuild(self, buildid): -+ def thd(conn): + yield self.db.pool.do_with_transaction(thd) + ++ def pruneBuild(self, buildid: int) -> defer.Deferred[None]: ++ def thd(conn) -> None: + tbl = self.db.model.builds + q = tbl.delete() + q = q.where(tbl.c.id == buildid) -+ conn.execute(q) -+ yield self.db.pool.do(thd) ++ res = conn.execute(q) ++ conn.commit() ++ res.close() ++ return self.db.pool.do(thd) + - # returns a Deferred that returns a value - def getBuildProperties(self, bid, resultSpec=None): - def thd(conn): -@@ -251,6 +260,15 @@ class BuildsConnectorComponent(base.DBCo - {"value": value_js, "source": source}) - yield self.db.pool.do(thd) - -+ @defer.inlineCallbacks -+ def pruneBuildProperties(self, buildid): -+ def thd(conn): ++ def pruneBuildProperties(self, buildid: int) -> defer.Deferred[None]: ++ def thd(conn) -> None: + bp_tbl = self.db.model.build_properties + q = bp_tbl.delete() + q = q.where(bp_tbl.c.buildid == buildid) -+ conn.execute(q) -+ yield self.db.pool.do(thd) ++ res = conn.execute(q) ++ conn.commit() ++ res.close() ++ return self.db.pool.do(thd) + - def _builddictFromRow(self, row): - return { - "id": row.id, + def _model_from_row(self, row): + return BuildModel( + id=row.id, --- a/buildbot/db/changes.py 2023-09-19 23:24:04.714268581 +0200 +++ b/buildbot/db/changes.py 2023-09-24 13:53:28.166432187 +0200 -@@ -333,6 +334,40 @@ class ChangesConnectorComponent(base.DBC - table.delete(table.c.changeid.in_(batch))) - yield self.db.pool.do(thd) +@@ -333,6 +334,40 @@ class ChangesConnectorComponent(base. + + yield self.db.pool.do_with_transaction(thd) + @defer.inlineCallbacks + def pruneChangesId(self, revision): @@ -87,93 +83,31 @@ + table = self.db.model.metadata.tables[table_name] + conn.execute( + table.delete(table.c.changeid.in_(batch))) -+ yield self.db.pool.do(thd) ++ yield self.db.pool.do_with_transaction(thd) + + - def _chdict_from_change_row_thd(self, conn, ch_row): - # This method must be run in a db.pool thread, and returns a chdict - # given a row from the 'changes' table - + def _thd_model_from_row(self, conn, ch_row) -> ChangeModel: + # This method must be run in a db.pool thread + change_files_tbl = self.db.model.change_files --- a/buildbot/db/logs.py 2022-04-02 11:10:34.892310594 +0200 +++ b/buildbot/db/logs.py 2023-06-26 23:06:24.611959431 +0200 -@@ -410,3 +410,80 @@ - rv = dict(row) - rv['complete'] = bool(rv['complete']) - return rv -+ -+ # returns a Deferred that returns a value -+ def deleteLogChunks(self, buildid): -+ model = self.db.model -+ horizon_per_builder = False -+ -+ def countLogchunks(conn): -+ res = conn.execute(sa.select([sa.func.count(model.logchunks.c.logid)])) -+ count = res.fetchone()[0] -+ res.close() -+ return count -+ -+ # find the steps.id at the upper bound of steps -+ def getStepidMax(conn, buildid): -+ # N.B.: we utilize the fact that steps.id is auto-increment, thus steps.started_at -+ # times are effectively sorted and we only need to find the steps.id at the upper -+ # bound of steps to update. -+ -+ # SELECT steps.id from steps WHERE steps.buildid = buildid ORDER BY -+ # steps.id DESC LIMIT 1; -+ res = conn.execute( -+ sa.select([model.steps.c.id]) -+ .where(model.steps.c.buildid == buildid) -+ .order_by(model.steps.c.id.desc()) -+ .limit(1) -+ ) -+ res_list = res.fetchone() -+ stepid_max = None -+ if res_list: -+ stepid_max = res_list[0] +@@ -502,3 +502,19 @@ + num_lines=row.num_lines, + type=row.type, + ) ++ def deleteLogChunks(self, logid: int) -> defer.Deferred[None]: ++ def thd(conn) -> None: ++ tbl = self.db.model.logchunks ++ res = conn.execute(tbl.delete().where(tbl.c.logid == logid)) + res.close() -+ return stepid_max -+ -+ # query all logs with type 'd' and delete their chunks. -+ def deleteLogsWithTypeD(conn): -+ if self.db._engine.dialect.name == 'sqlite': -+ # sqlite does not support delete with a join, so for this case we use a subquery, -+ # which is much slower -+ q = sa.select([model.logs.c.id]) -+ q = q.select_from(model.logs) -+ q = q.where(model.logs.c.type == 'd') -+ -+ # delete their logchunks -+ q = model.logchunks.delete().where(model.logchunks.c.logid.in_(q)) -+ else: -+ q = model.logchunks.delete() -+ q = q.where(model.logs.c.id == model.logchunks.c.logid) -+ q = q.where(model.logs.c.type == 'd') -+ -+ res = conn.execute(q) ++ return self.db.pool.do(thd) ++ ++ def deleteLog(self, stepid: int) -> defer.Deferred[None]: ++ def thd(conn) -> None: ++ tbl = self.db.model.logs ++ q = tbl.update() ++ q = q.where(tbl.c.id == stepid) ++ q = q.where(tbl.c.type != 'd') ++ res = conn.execute(q.values(type='d')) + res.close() -+ -+ def thddeleteLogs(conn): -+ count_before = countLogchunks(conn) -+ -+ # update log types that match buildid -+ # we do it first to avoid having UI discrepancy -+ -+ stepid_max = getStepidMax(conn, buildid) -+ if stepid_max: -+ # UPDATE logs SET logs.type = 'd' -+ # WHERE logs.stepid <= stepid_max AND type != 'd'; -+ res = conn.execute( -+ model.logs.update() -+ .where(sa.and_(model.logs.c.stepid <= stepid_max, -+ model.logs.c.type != 'd')) -+ .values(type='d') -+ ) -+ res.close() -+ -+ deleteLogsWithTypeD(conn) -+ -+ count_after = countLogchunks(conn) -+ count = count_before - count_after -+ -+ return count if count > 0 else 0 -+ return self.db.pool.do(thddeleteLogs) ++ return self.db.pool.do(thd) diff --git a/patches/bb-gitlab.patch b/patches/bb-gitlab.patch index 251991a..fe185c3 100644 --- a/patches/bb-gitlab.patch +++ b/patches/bb-gitlab.patch @@ -37,9 +37,9 @@ index 8f2f80a83..40de0273e 100644 +import gitlab as python_gitlab + + from dateutil.parser import parse as dateparse from twisted.internet.defer import inlineCallbacks from twisted.python import log - @@ -28,6 +30,7 @@ from buildbot.www.hooks.base import BaseHookHandler _HEADER_EVENT = b'X-Gitlab-Event' @@ -84,14 +84,14 @@ index 8f2f80a83..40de0273e 100644 """ Consumes the merge_request JSON as a python object and turn it into a buildbot change. @@ -126,7 +157,7 @@ class GitLabHandler(BaseHookHandler): - - changes = [{ - 'author': f"{commit['author']['name']} <{commit['author']['email']}>", -- 'files': [], # @todo use rest API -+ 'files': self._getFiles(attrs), - 'comments': f"MR#{attrs['iid']}: {attrs['title']}\n\n{attrs['description']}", - 'revision': commit['id'], - 'when_timestamp': when_timestamp, + changes = [ + { + 'author': f"{commit['author']['name']} <{commit['author']['email']}>", +- 'files': [], # @todo use rest API ++ 'files': self._getFiles(attrs), + 'comments': f"MR#{attrs['iid']}: {attrs['title']}\n\n{attrs['description']}", + 'revision': commit['id'], + 'when_timestamp': when_timestamp, @@ -220,17 +251,21 @@ class GitLabHandler(BaseHookHandler): request the http request object @@ -122,6 +122,6 @@ index 8f2f80a83..40de0273e 100644 payload, user, repo, repo_url, event_type, codebase=codebase) elif event_type == 'merge_request': + self.gl = self._configGitlabRest(expected_token_value, baseURL=baseUrl_value) - changes = self._process_merge_request_change( - payload, event_type, codebase=codebase) + changes = self._process_merge_request_change(payload, event_type, codebase=codebase) elif event_type == 'note': + changes = self._process_note_addition_to_merge_request( diff --git a/patches/bb-props_master_fix.patch b/patches/bb-props_master_fix.patch index f31d4ad..2cf7b64 100644 --- a/patches/bb-props_master_fix.patch +++ b/patches/bb-props_master_fix.patch @@ -5,7 +5,7 @@ index df8f9c014..c16e4f64e 100644 @@ -288,6 +288,10 @@ class Builder(util_service.ReconfigurableServiceMixin, # The instance type of the worker may depend on the properties of # the build that substantiated it. - props = setupPropsIfNeeded(props) + props = yield self._setup_props_if_needed(props, workerforbuilder, buildrequest) + # check if we have props.master when we call renderWorkerProps later + # set props.master if needed + if props.master is None: |