Add package metadata support (email)

Signed-off-by: Magnus Granberg <zorry@gentoo.org>

4 files changed, 335 insertions, 9 deletions

diff --git a/buildbot_gentoo_ci/config/buildfactorys.py b/buildbot_gentoo_ci/config/buildfactorys.py
index efccc93..2ca2ca9 100644
--- a/buildbot_gentoo_ci/config/buildfactorys.py
+++ b/buildbot_gentoo_ci/config/buildfactorys.py
@@ -58,13 +58,14 @@ def update_db_cpv():
     f.addStep(category.CheckC())
     # if package in db
     #   return package_data
-    #   add check package path step at end
     # else
     #   add package to db step
     #   return package_data
     f.addStep(package.CheckP())
     # Trigger update_db_v
     f.addStep(package.TriggerCheckForV())
+    # update metadata if needed
+    f.addStep(package.CheckMetadataPackagePath())
     return f
 
 def update_db_v():
diff --git a/buildbot_gentoo_ci/db/model.py b/buildbot_gentoo_ci/db/model.py
index 4373ebb..6e13ad7 100644
--- a/buildbot_gentoo_ci/db/model.py
+++ b/buildbot_gentoo_ci/db/model.py
@@ -287,6 +287,34 @@ class Model(base.DBConnectorComponent):
         sa.Column('deleted_at', sa.Integer, nullable=True),
     )
 
+    emails = sautils.Table(
+        "emails", metadata,
+        # unique uuid per keyword
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('email', sa.String(255), nullable=False),
+    )
+
+    packages_emails = sautils.Table(
+        "packages_emails", metadata,
+        # unique uuid per keyword
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('package_uuid', sa.String(36),
+                  sa.ForeignKey('packages.uuid', ondelete='CASCADE')),
+        sa.Column('email_id', sa.Integer,
+                  sa.ForeignKey('emails.id', ondelete='CASCADE')),
+        sa.Column('mail_type', sa.Enum('project', 'person', 'unknown',), nullable=False, default='unknown'),
+        sa.Column('proxied', sa.Enum('yes', 'no', 'proxy',), nullable=False, default='no'),
+    )
+
+    packages_metadata = sautils.Table(
+        "packages_metadata", metadata,
+        # unique uuid per keyword
+        sa.Column('id', sa.Integer, primary_key=True),
+        sa.Column('package_uuid', sa.String(36),
+                  sa.ForeignKey('packages.uuid', ondelete='CASCADE')),
+        sa.Column('sha256', sa.String(255), nullable=False, default='0'),
+    )
+
     versions = sautils.Table(
         "versions", metadata,
         sa.Column('uuid', sa.String(36), primary_key=True,
diff --git a/buildbot_gentoo_ci/db/packages.py b/buildbot_gentoo_ci/db/packages.py
index a22b9d8..79ae685 100644
--- a/buildbot_gentoo_ci/db/packages.py
+++ b/buildbot_gentoo_ci/db/packages.py
@@ -65,7 +65,10 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
                 r = conn.execute(q, dict(name=name,
                                          repository_uuid=repository_uuid,
                                          category_uuid=category_uuid))
-            except (sa.exc.IntegrityError, sa.exc.ProgrammingError):
+            except Exception as e:
+                print(type(e))
+                print(e.args)
+                print(e)
                 uuid = None
             else:
                 uuid = r.inserted_primary_key[0]
@@ -80,3 +83,145 @@ class PackagesConnectorComponent(base.DBConnectorComponent):
             repository_uuid=row.repository_uuid,
             category_uuid=row.category_uuid
             )
+
+    @defer.inlineCallbacks
+    def getPackageMetadataByPackageUuid(self, package_uuid):
+        def thd(conn):
+            tbl = self.db.model.packages_metadata
+            q = tbl.select()
+            q = q.where(tbl.c.package_uuid == package_uuid)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_metadata(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def addMetadata(self, package_uuid, sha256):
+        def thd(conn, no_recurse=False):
+            try:
+                tbl = self.db.model.packages_metadata
+                q = tbl.insert()
+                r = conn.execute(q, dict(package_uuid=package_uuid,
+                                         sha256=sha256
+                                         ))
+            except Exception as e:
+                print(type(e))
+                print(e.args)
+                print(e)
+                res = None
+            else:
+                res = r.inserted_primary_key[0]
+            return res
+        res = yield self.db.pool.do(thd)
+        return res
+
+    def _row2dict_metadata(self, conn, row):
+        return dict(
+            id=row.id,
+            package_uuid=row.package_uuid,
+            sha256=row.sha256,
+            )
+
+    @defer.inlineCallbacks
+    def addEmail(self, email):
+        def thd(conn, no_recurse=False):
+            try:
+                tbl = self.db.model.emails
+                q = tbl.insert()
+                r = conn.execute(q, dict(email=email))
+            except Exception as e:
+                print(type(e))
+                print(e.args)
+                print(e)
+                res = None
+            else:
+                res = r.inserted_primary_key[0]
+            return res
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def addPackageEmail(self, email_id, package_uuid, mail_type, proxied):
+        def thd(conn, no_recurse=False):
+            try:
+                tbl = self.db.model.packages_emails
+                q = tbl.insert()
+                r = conn.execute(q, dict(email_id=email_id,
+                                         package_uuid=package_uuid,
+                                         mail_type=mail_type,
+                                         proxied=proxied
+                                         ))
+            except Exception as e:
+                print(type(e))
+                print(e.args)
+                print(e)
+                res = None
+            else:
+                res = r.inserted_primary_key[0]
+            return res
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getEmailIdByEmail(self, email):
+        def thd(conn):
+            tbl = self.db.model.emails
+            q = tbl.select()
+            q = q.where(tbl.c.email == email)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_email(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getEmailByEmailId(self, email_id):
+        def thd(conn):
+            tbl = self.db.model.emails
+            q = tbl.select()
+            q = q.where(tbl.c.id == email_id)
+            res = conn.execute(q)
+            row = res.fetchone()
+            if not row:
+                return None
+            return self._row2dict_email(conn, row)
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def getEmailsIdsByPackageUuid(self, package_uuid):
+        def thd(conn):
+            tbl = self.db.model.packages_emails
+            q = tbl.select()
+            q = q.where(tbl.c.package_uuid == package_uuid)
+            return [self._row2dict_package_email(conn, row)
+                for row in conn.execute(q).fetchall()]
+        res = yield self.db.pool.do(thd)
+        return res
+
+    @defer.inlineCallbacks
+    def delPackageEmail(self, package_uuid):
+        def thd(conn, no_recurse=False):
+            tbl = self.db.model.packages_emails
+            conn.execute(tbl.delete(
+                whereclause=((tbl.c.package_uuid == package_uuid))))
+        return self.db.pool.do(thd)
+
+    def _row2dict_email(self, conn, row):
+        return dict(
+            id=row.id,
+            email=row.email,
+            )
+    def _row2dict_package_email(self, conn, row):
+        return dict(
+            id=row.id,
+            email_id=row.email_id,
+            package_uuid=row.package_uuid,
+            type=row.type,
+            proxied=row.proxied,
+            )
diff --git a/buildbot_gentoo_ci/steps/package.py b/buildbot_gentoo_ci/steps/package.py
index 9fc59e8..13eafa2 100644
--- a/buildbot_gentoo_ci/steps/package.py
+++ b/buildbot_gentoo_ci/steps/package.py
@@ -4,9 +4,7 @@
 import re
 import os
 
-from portage.xml.metadata import MetaDataXML
-from portage.checksum import perform_checksum
-from portage.versions import catpkgsplit
+import xml.etree.ElementTree as etree
 
 from twisted.internet import defer
 from twisted.python import log
@@ -14,6 +12,7 @@ from twisted.python import log
 from buildbot.process.buildstep import BuildStep
 from buildbot.process.results import SUCCESS
 from buildbot.process.results import FAILURE
+from buildbot.process import remotecommand
 from buildbot.plugins import steps
 
 class SetupPropertys(BuildStep):
@@ -67,6 +66,161 @@ class AddPackage(BuildStep):
         self.setProperty("package_data", self.package_data, 'package_data')
         return SUCCESS
 
+class UpdateEmails(BuildStep):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    name = 'UpdateEmails'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    @defer.inlineCallbacks
+    def run(self):
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        packages_emails = yield self.gentooci.db.packages.getEmailsIdsByPackageUuid(self.getProperty("package_data")['uuid'])
+        if packages_emails != []:
+            yield self.gentooci.db.packages.delPackageEmail(self.getProperty("package_data")['uuid'])
+        # update email
+        for maintainer in self.getProperty("maintainers"):
+            print(maintainer)
+            email_data = yield self.gentooci.db.packages.getEmailIdByEmail(maintainer['email'])
+            print(email_data)
+            if email_data is None:
+                email_id = yield self.gentooci.db.packages.addEmail(maintainer['email'])
+                print(email_id)
+            else:
+                email_id = email_data['id']
+            if maintainer['proxied'] is None:
+                proxied = 'no'
+            else:
+                proxied = maintainer['proxied']
+            res = yield self.gentooci.db.packages.addPackageEmail(
+                                                                email_id,
+                                                                self.getProperty("package_data")['uuid'],
+                                                                maintainer['type'],
+                                                                proxied
+                                                                )
+            print(res)
+        return SUCCESS
+
+class ReadMetadataXML(BuildStep):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    name = 'ReadMetadataXML'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    @defer.inlineCallbacks
+    def run(self):
+        self._xml_tree = etree.fromstring(self.getProperty("metadata_xml"))
+        # get maintainer
+        self.maintainers = []
+        for maintainer in self._xml_tree.findall('maintainer'):
+            self.maintainers.append(dict(
+                                        type=maintainer.get('type'),
+                                        proxied=maintainer.get('proxied'),
+                                        email=maintainer.find('email').text,
+                                        ))
+        print(self.maintainers)
+        self.setProperty("maintainers", self.maintainers, 'maintainers')
+        yield self.build.addStepsAfterCurrentStep([UpdateEmails()])
+        #   update email
+        return SUCCESS
+
+class CheckSha256Metadata(BuildStep):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    name = 'CheckSha256Metadata'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    @defer.inlineCallbacks
+    def run(self):
+        # if checksum diff update
+        self.gentooci = self.master.namedServices['services'].namedServices['gentooci']
+        sha256sum_metadata = self.getProperty("sha256sum_metadata").split()[0]
+        print(sha256sum_metadata)
+        package_metadata_data_updated = False
+        self.package_metadata_data = yield self.gentooci.db.packages.getPackageMetadataByPackageUuid(self.getProperty("package_data")['uuid'])
+        print(self.package_metadata_data)
+        if self.package_metadata_data is None:
+            res = yield self.gentooci.db.packages.addMetadata(
+                                                            self.getProperty("package_data")['uuid'],
+                                                            sha256sum_metadata
+                                                            )
+            print(res)
+            package_metadata_data_updated = True
+        elif self.package_metadata_data['sha256'] != sha256sum_metadata:
+            # update sha256
+            package_metadata_data_updated = True
+        if package_metadata_data_updated:
+            yield self.build.addStepsAfterCurrentStep([
+            steps.SetPropertyFromCommand(
+                                        name = 'CatMetadataFile',
+                                        haltOnFailure = True,
+                                        flunkOnFailure = True,
+                                        command=['cat', self.getProperty("metadata_file")],
+                                        strip=False,
+                                        property='metadata_xml'
+                                        ),
+            ReadMetadataXML(),
+            ])
+        return SUCCESS
+
+class CheckMetadataPackagePath(BuildStep):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    name = 'CheckMetadataPackagePath'
+    description = 'Running'
+    descriptionDone = 'Ran'
+    descriptionSuffix = None
+    haltOnFailure = True
+    flunkOnFailure = True
+
+    @defer.inlineCallbacks
+    def run(self):
+        # set metadata.xml path
+        self.repository_basedir = yield os.path.join(self.getProperty("worker_basedir"), self.getProperty('portage_repos_path'))
+        print(self.repository_basedir)
+        self.repository_path = yield os.path.join(self.repository_basedir, self.getProperty("repository_data")['name'])
+        print(self.repository_path)
+        self.cp_path = yield os.path.join(self.repository_path, self.getProperty("change_data")['cp'])
+        print(self.cp_path)
+        self.metadata_file = yield '/'.join([self.cp_path, 'metadata.xml'])
+        print(self.metadata_file)
+        # check file exist
+        cmd = remotecommand.RemoteCommand('stat', {'file': self.metadata_file})
+        yield self.runCommand(cmd)
+        if cmd.didFail():
+            return FAILURE
+        self.setProperty("metadata_file", self.metadata_file, 'metadata_file')
+        # checksum on file
+        yield self.build.addStepsAfterCurrentStep([
+            steps.SetPropertyFromCommand(
+                                        name = 'ChecksumMetadataPackage',
+                                        haltOnFailure = True,
+                                        flunkOnFailure = True,
+                                        command=['sha256sum', self.metadata_file],
+                                        workdir=self.repository_path,
+                                        strip=False,
+                                        property='sha256sum_metadata'
+                                        ),
+            CheckSha256Metadata(),
+            ])
+        return SUCCESS
+
 class CheckP(BuildStep):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -90,10 +244,8 @@ class CheckP(BuildStep):
         if self.package_data is None:
             self.setProperty("package", self.package, 'package')
             yield self.build.addStepsAfterCurrentStep([AddPackage()])
-            #yield self.build.addStepsAfterLastStep([AddMetadataPackage()])
-            return SUCCESS
-        self.setProperty("package_data", self.package_data, 'package_data')
-        #yield self.build.addStepsAfterLastStep([CheckPathPackage()])
+        else:
+            self.setProperty("package_data", self.package_data, 'package_data')
         return SUCCESS
 
 class TriggerCheckForV(BuildStep):